def update_table_provisioning(table_name, reads, writes): """""" table = get_table(table_name) try: table.update( throughput={ 'read': reads, 'write': writes }) logger.info( '{0} - Provisioning updated to {1} reads and {2} writes'.format( table_name, reads, writes)) except JSONResponseError as error: exception = error.body['__type'].split('#')[1] know_exceptions = [ 'LimitExceededException', 'ValidationException', 'ResourceInUseException'] if exception in know_exceptions: logger.warning('{0} - {1}: {2}'.format( table_name, exception, error.body['message'])) else: logger.error( ( '{0} - Unhandled exception: {1}: {2}. ' 'Please file a bug report at ' 'https://github.com/sebdah/dynamic-dynamodb/issues' ).format(table_name, exception, error.body['message']))
def get_tables_and_gsis(): """ Get a set of tables and gsis and their configuration keys :returns: set -- A set of tuples (table_name, table_conf_key) """ table_names = set() configured_tables = get_configured_tables() not_used_tables = set(configured_tables) # Add regexp table names for table_instance in list_tables(): for key_name in configured_tables: try: if re.match(key_name, table_instance.table_name): logger.debug("Table {0} match with config key {1}".format(table_instance.table_name, key_name)) table_names.add((table_instance.table_name, key_name)) not_used_tables.discard(key_name) else: logger.debug( "Table {0} did not match with config key {1}".format(table_instance.table_name, key_name) ) except re.error: logger.error('Invalid regular expression: "{0}"'.format(key_name)) sys.exit(1) if not_used_tables: logger.warning( "No tables matching the following configured " "tables found: {0}".format(", ".join(not_used_tables)) ) return sorted(table_names)
def get_tables_and_gsis(): """ Get a set of tables and gsis and their configuration keys :returns: set -- A set of tuples (table_name, table_conf_key) """ table_names = set() configured_tables = get_configured_tables() not_used_tables = set(configured_tables) # Add regexp table names for table_instance in list_tables(): for key_name in configured_tables: try: if re.match(key_name, table_instance.table_name): logger.debug("Table {0} match with config key {1}".format( table_instance.table_name, key_name)) table_names.add((table_instance.table_name, key_name)) not_used_tables.discard(key_name) else: logger.debug( "Table {0} did not match with config key {1}".format( table_instance.table_name, key_name)) except re.error: logger.error( 'Invalid regular expression: "{0}"'.format(key_name)) sys.exit(1) if not_used_tables: logger.warning('No tables matching the following configured ' 'tables found: {0}'.format(', '.join(not_used_tables))) return sorted(table_names)
def ensure_provisioning(table_name, key_name): """ Ensure that provisioning is correct :type table_name: str :param table_name: Name of the DynamoDB table :type key_name: str :param key_name: Configuration option key name """ if get_global_option('circuit_breaker_url'): if circuit_breaker.is_open(): logger.warning('Circuit breaker is OPEN!') return None read_update_needed, updated_read_units = __ensure_provisioning_reads( table_name, key_name) write_update_needed, updated_write_units = __ensure_provisioning_writes( table_name, key_name) # Handle throughput updates if read_update_needed or write_update_needed: logger.info( '{0} - Changing provisioning to {1:d} ' 'read units and {2:d} write units'.format( table_name, int(updated_read_units), int(updated_write_units))) __update_throughput( table_name, updated_read_units, updated_write_units, key_name) else: logger.info('{0} - No need to change provisioning'.format(table_name))
def ensure_provisioning(table_name, key_name): """ Ensure that provisioning is correct :type table_name: str :param table_name: Name of the DynamoDB table :type key_name: str :param key_name: Configuration option key name """ if get_global_option('circuit_breaker_url'): if __circuit_breaker_is_open(): logger.warning('Circuit breaker is OPEN!') return None read_update_needed, updated_read_units = __ensure_provisioning_reads( table_name, key_name) write_update_needed, updated_write_units = __ensure_provisioning_writes( table_name, key_name) # Handle throughput updates if read_update_needed or write_update_needed: logger.info( '{0} - Changing provisioning to {1:d} ' 'read units and {2:d} write units'.format( table_name, int(updated_read_units), int(updated_write_units))) update_throughput(table_name, updated_read_units, updated_write_units, key_name) else: logger.info('{0} - No need to change provisioning'.format(table_name))
def ensure_provisioning(table_name, key_name, num_consec_read_checks, num_consec_write_checks): """ Ensure that provisioning is correct :type table_name: str :param table_name: Name of the DynamoDB table :type key_name: str :param key_name: Configuration option key name :type num_consec_read_checks: int :param num_consec_read_checks: How many consecutive checks have we had :type num_consec_write_checks: int :param num_consec_write_checks: How many consecutive checks have we had :returns: (int, int) -- num_consec_read_checks, num_consec_write_checks """ if get_global_option('circuit_breaker_url') or get_table_option( key_name, 'circuit_breaker_url'): if circuit_breaker.is_open(table_name, key_name): logger.warning('Circuit breaker is OPEN!') return (0, 0) # Handle throughput alarm checks __ensure_provisioning_alarm(table_name, key_name) try: read_update_needed, updated_read_units, num_consec_read_checks = \ __ensure_provisioning_reads( table_name, key_name, num_consec_read_checks) write_update_needed, updated_write_units, num_consec_write_checks = \ __ensure_provisioning_writes( table_name, key_name, num_consec_write_checks) if read_update_needed: num_consec_read_checks = 0 if write_update_needed: num_consec_write_checks = 0 # Handle throughput updates if read_update_needed or write_update_needed: logger.info('{0} - Changing provisioning to {1:d} ' 'read units and {2:d} write units'.format( table_name, int(updated_read_units), int(updated_write_units))) __update_throughput(table_name, key_name, updated_read_units, updated_write_units) else: logger.info( '{0} - No need to change provisioning'.format(table_name)) except JSONResponseError: raise except BotoServerError: raise return num_consec_read_checks, num_consec_write_checks
def ensure_provisioning(table_name, table_key, gsi_name, gsi_key, num_consec_read_checks, num_consec_write_checks): """ Ensure that provisioning is correct for Global Secondary Indexes :type table_name: str :param table_name: Name of the DynamoDB table :type table_key: str :param table_key: Table configuration option key name :type gsi_name: str :param gsi_name: Name of the GSI :type gsi_key: str :param gsi_key: GSI configuration option key name :type num_consec_read_checks: int :param num_consec_read_checks: How many consecutive checks have we had :type num_consec_write_checks: int :param num_consec_write_checks: How many consecutive checks have we had :returns: (int, int) -- num_consec_read_checks, num_consec_write_checks """ if get_global_option("circuit_breaker_url"): if circuit_breaker.is_open(): logger.warning("Circuit breaker is OPEN!") return (0, 0) logger.info("{0} - Will ensure provisioning for global secondary index {1}".format(table_name, gsi_name)) # Handle throughput alarm checks __ensure_provisioning_alarm(table_name, table_key, gsi_name, gsi_key) try: read_update_needed, updated_read_units, num_consec_read_checks = __ensure_provisioning_reads( table_name, table_key, gsi_name, gsi_key, num_consec_read_checks ) write_update_needed, updated_write_units, num_consec_write_checks = __ensure_provisioning_writes( table_name, table_key, gsi_name, gsi_key, num_consec_write_checks ) if read_update_needed: num_consec_read_checks = 0 if write_update_needed: num_consec_write_checks = 0 # Handle throughput updates if read_update_needed or write_update_needed: logger.info( "{0} - GSI: {1} - Changing provisioning to {2:d} " "read units and {3:d} write units".format( table_name, gsi_name, int(updated_read_units), int(updated_write_units) ) ) __update_throughput(table_name, table_key, gsi_name, gsi_key, updated_read_units, updated_write_units) else: logger.info("{0} - GSI: {1} - No need to change provisioning".format(table_name, gsi_name)) except JSONResponseError: raise except BotoServerError: raise return num_consec_read_checks, num_consec_write_checks
def ensure_provisioning(table_name, table_key, gsi_name, gsi_key): """ Ensure that provisioning is correct for Global Secondary Indexes :type table_name: str :param table_name: Name of the DynamoDB table :type table_key: str :param table_key: Table configuration option key name :type gsi_name: str :param gsi_name: Name of the GSI :type gsi_key: str :param gsi_key: GSI configuration option key name """ if get_global_option('circuit_breaker_url'): if circuit_breaker.is_open(): logger.warning('Circuit breaker is OPEN!') return None logger.info( '{0} - Will ensure provisioning for global secondary index {1}'.format( table_name, gsi_name)) try: read_update_needed, updated_read_units = __ensure_provisioning_reads( table_name, table_key, gsi_name, gsi_key) write_update_needed, updated_write_units = __ensure_provisioning_writes( table_name, table_key, gsi_name, gsi_key) # Handle throughput updates if read_update_needed or write_update_needed: logger.info( '{0} - GSI: {1} - Changing provisioning to {2:d} ' 'read units and {3:d} write units'.format( table_name, gsi_name, int(updated_read_units), int(updated_write_units))) __update_throughput( table_name, table_key, gsi_name, gsi_key, updated_read_units, updated_write_units) else: logger.info( '{0} - GSI: {1} - No need to change provisioning'.format( table_name, gsi_name)) except JSONResponseError: raise
def ensure_provisioning( table_name, key_name, num_consec_read_checks, num_consec_write_checks): """ Ensure that provisioning is correct :type table_name: str :param table_name: Name of the DynamoDB table :type key_name: str :param key_name: Configuration option key name :type num_consec_read_checks: int :param num_consec_read_checks: How many consecutive checks have we had :type num_consec_write_checks: int :param num_consec_write_checks: How many consecutive checks have we had :returns: (int, int) -- num_consec_read_checks, num_consec_write_checks """ if get_global_option('circuit_breaker_url'): if circuit_breaker.is_open(): logger.warning('Circuit breaker is OPEN!') return (0, 0) try: read_update_needed, updated_read_units, num_consec_read_checks = \ __ensure_provisioning_reads( table_name, key_name, num_consec_read_checks) write_update_needed, updated_write_units, num_consec_write_checks = \ __ensure_provisioning_writes( table_name, key_name, num_consec_write_checks) # Handle throughput updates if read_update_needed or write_update_needed: logger.info( '{0} - Changing provisioning to {1:d} ' 'read units and {2:d} write units'.format( table_name, int(updated_read_units), int(updated_write_units))) __update_throughput( table_name, key_name, updated_read_units, updated_write_units) else: logger.info('{0} - No need to change provisioning'.format( table_name)) except JSONResponseError: raise except BotoServerError: raise return num_consec_read_checks, num_consec_write_checks
def __update_throughput(table_name, table_key, gsi_name, gsi_key, read_units, write_units): """ Update throughput on the GSI :type table_name: str :param table_name: Name of the DynamoDB table :type table_key: str :param table_key: Table configuration option key name :type gsi_name: str :param gsi_name: Name of the GSI :type gsi_key: str :param gsi_key: Configuration option key name :type read_units: int :param read_units: New read unit provisioning :type write_units: int :param write_units: New write unit provisioning """ try: current_ru = dynamodb.get_provisioned_gsi_read_units( table_name, gsi_name) current_wu = dynamodb.get_provisioned_gsi_write_units( table_name, gsi_name) except JSONResponseError: raise # Check table status try: gsi_status = dynamodb.get_gsi_status(table_name, gsi_name) except JSONResponseError: raise logger.debug('{0} - GSI: {1} - GSI status is {2}'.format( table_name, gsi_name, gsi_status)) if gsi_status != 'ACTIVE': logger.warning( '{0} - GSI: {1} - Not performing throughput changes when GSI ' 'status is {2}'.format(table_name, gsi_name, gsi_status)) return # If this setting is True, we will only scale down when # BOTH reads AND writes are low if get_gsi_option(table_key, gsi_key, 'always_decrease_rw_together'): read_units, write_units = __calculate_always_decrease_rw_values( table_name, gsi_name, read_units, current_ru, write_units, current_wu) if read_units == current_ru and write_units == current_wu: logger.info('{0} - GSI: {1} - No changes to perform'.format( table_name, gsi_name)) return dynamodb.update_gsi_provisioning(table_name, table_key, gsi_name, gsi_key, int(read_units), int(write_units))
def __update_throughput(table_name, key_name, read_units, write_units): """ Update throughput on the DynamoDB table :type table_name: str :param table_name: Name of the DynamoDB table :type key_name: str :param key_name: Configuration option key name :type read_units: int :param read_units: New read unit provisioning :type write_units: int :param write_units: New write unit provisioning """ try: current_ru = dynamodb.get_provisioned_table_read_units(table_name) current_wu = dynamodb.get_provisioned_table_write_units(table_name) except JSONResponseError: raise # Check table status try: table_status = dynamodb.get_table_status(table_name) except JSONResponseError: raise logger.debug('{0} - Table status is {1}'.format(table_name, table_status)) if table_status != 'ACTIVE': logger.warning( '{0} - Not performing throughput changes when table ' 'is {1}'.format(table_name, table_status)) return # If this setting is True, we will only scale down when # BOTH reads AND writes are low if get_table_option(key_name, 'always_decrease_rw_together'): read_units, write_units = __calculate_always_decrease_rw_values( table_name, read_units, current_ru, write_units, current_wu) if read_units == current_ru and write_units == current_wu: logger.info('{0} - No changes to perform'.format(table_name)) return dynamodb.update_table_provisioning( table_name, key_name, int(read_units), int(write_units))
def __update_throughput(table_name, table_key, gsi_name, gsi_key, read_units, write_units): """ Update throughput on the GSI :type table_name: str :param table_name: Name of the DynamoDB table :type table_key: str :param table_key: Table configuration option key name :type gsi_name: str :param gsi_name: Name of the GSI :type gsi_key: str :param gsi_key: Configuration option key name :type read_units: int :param read_units: New read unit provisioning :type write_units: int :param write_units: New write unit provisioning """ try: current_ru = dynamodb.get_provisioned_gsi_read_units(table_name, gsi_name) current_wu = dynamodb.get_provisioned_gsi_write_units(table_name, gsi_name) except JSONResponseError: raise # Check table status try: gsi_status = dynamodb.get_gsi_status(table_name, gsi_name) except JSONResponseError: raise logger.debug("{0} - GSI: {1} - GSI status is {2}".format(table_name, gsi_name, gsi_status)) if gsi_status != "ACTIVE": logger.warning( "{0} - GSI: {1} - Not performing throughput changes when GSI " "status is {2}".format(table_name, gsi_name, gsi_status) ) return # If this setting is True, we will only scale down when # BOTH reads AND writes are low if get_gsi_option(table_key, gsi_key, "always_decrease_rw_together"): read_units, write_units = __calculate_always_decrease_rw_values( table_name, gsi_name, read_units, current_ru, write_units, current_wu ) if read_units == current_ru and write_units == current_wu: logger.info("{0} - GSI: {1} - No changes to perform".format(table_name, gsi_name)) return dynamodb.update_gsi_provisioning(table_name, table_key, gsi_name, gsi_key, int(read_units), int(write_units))
def get_tables_and_gsis(): """ Get a set of tables and gsis and their configuration keys :returns: set -- A set of tuples (table_name, table_conf_key) """ table_names = set() configured_tables = get_configured_tables() not_used_tables = set(configured_tables) # Add regexp table names for table_instance in list_tables(): for key_name in configured_tables: try: if re.match(key_name, table_instance.table_name): logger.debug("Table {0} match with config key {1}".format( table_instance.table_name, key_name)) # Notify users about regexps that match multiple tables if table_instance.table_name in [x[0] for x in table_names]: logger.warning( 'Table {0} matches more than one regexp in config, ' 'skipping this match: "{1}"'.format( table_instance.table_name, key_name)) else: table_names.add( ( table_instance.table_name, key_name )) not_used_tables.discard(key_name) else: logger.debug( "Table {0} did not match with config key {1}".format( table_instance.table_name, key_name)) except re.error: logger.error('Invalid regular expression: "{0}"'.format( key_name)) sys.exit(1) if not_used_tables: logger.warning( 'No tables matching the following configured ' 'tables found: {0}'.format(', '.join(not_used_tables))) return sorted(table_names)
def update_gsi_provisioning(table_name, gsi_name, reads, writes): """ Update provisioning on a global secondary index :type table_name: str :param table_name: Name of the table :type gsi_name: str :param gsi_name: Name of the GSI :type reads: int :param reads: Number of reads to provision :type writes: int :param writes: Number of writes to provision """ try: DYNAMODB_CONNECTION.update_table( table_name=table_name, global_secondary_index_updates=[ { "Update": { "IndexName": gsi_name, "ProvisionedThroughput": { "ReadCapacityUnits": reads, "WriteCapacityUnits": writes } } } ]) except JSONResponseError as error: exception = error.body['__type'].split('#')[1] know_exceptions = ['LimitExceededException'] if exception in know_exceptions: logger.warning('{0} - GSI: {1} - {2}: {3}'.format( table_name, gsi_name, exception, error.body['message'])) else: logger.error( ( '{0} - GSI: {1} - Unhandled exception: {2}: {3}. ' 'Please file a bug report at ' 'https://github.com/sebdah/dynamic-dynamodb/issues' ).format( table_name, gsi_name, exception, error.body['message']))
def update_throughput(table_name, read_units, write_units, key_name): """ Update throughput on the DynamoDB table :type table_name: str :param table_name: Name of the DynamoDB table :type read_units: int :param read_units: New read unit provisioning :type write_units: int :param write_units: New write unit provisioning :type key_name: str :param key_name: Configuration option key name """ try: table = dynamodb.get_table(table_name) except DynamoDBResponseError: # Return if the table does not exist return None # Check that we are in the right time frame if get_table_option(key_name, 'maintenance_windows'): if (not __is_maintenance_window(table_name, get_table_option( key_name, 'maintenance_windows'))): logger.warning( '{0} - Current time is outside maintenance window'.format( table_name)) return else: logger.info( '{0} - Current time is within maintenance window'.format( table_name)) # Check table status if table.status != 'ACTIVE': logger.warning( '{0} - Not performing throughput changes when table ' 'is in {1} state'.format(table_name, table.status)) # If this setting is True, we will only scale down when # BOTH reads AND writes are low if get_table_option(key_name, 'always_decrease_rw_together'): if ((read_units < table.read_units) or (table.read_units == get_table_option( key_name, 'min_provisioned_reads'))): if ((write_units < table.write_units) or (table.write_units == get_table_option( key_name, 'min_provisioned_writes'))): logger.info( '{0} - Both reads and writes will be decreased'.format( table_name)) elif read_units < table.read_units: logger.info( '{0} - Will not decrease reads nor writes, waiting for ' 'both to become low before decrease'.format(table_name)) read_units = table.read_units elif write_units < table.write_units: logger.info( '{0} - Will not decrease reads nor writes, waiting for ' 'both to become low before decrease'.format(table_name)) write_units = table.write_units if read_units == table.read_units and write_units == table.write_units: logger.debug('{0} - No need to update provisioning') return if not get_global_option('dry_run'): try: table.update_throughput(int(read_units), int(write_units)) logger.info('Provisioning updated') except DynamoDBResponseError as error: dynamodb_error = error.body['__type'].rsplit('#', 1)[1] if dynamodb_error == 'LimitExceededException': logger.warning( '{0} - {1}'.format(table_name, error.body['message'])) if int(read_units) > table.read_units: logger.info('{0} - Scaling up reads to {1:d}'.format( table_name, int(read_units))) update_throughput( table_name, int(read_units), int(table.write_units), key_name) elif int(write_units) > table.write_units: logger.info('{0} - Scaling up writes to {1:d}'.format( table_name, int(write_units))) update_throughput( table_name, int(table.read_units), int(write_units), key_name) elif dynamodb_error == 'ValidationException': logger.warning('{0} - ValidationException: {1}'.format( table_name, error.body['message'])) elif dynamodb_error == 'ResourceInUseException': logger.warning('{0} - ResourceInUseException: {1}'.format( table_name, error.body['message'])) elif dynamodb_error == 'AccessDeniedException': logger.warning('{0} - AccessDeniedException: {1}'.format( table_name, error.body['message'])) else: logger.error( ( '{0} - Unhandled exception: {1}: {2}. ' 'Please file a bug report at ' 'https://github.com/sebdah/dynamic-dynamodb/issues' ).format( table_name, dynamodb_error, error.body['message']))
def is_open(table_name=None, table_key=None, gsi_name=None, gsi_key=None): """ Checks whether the circuit breaker is open :param table_name: Name of the table being checked :param table_key: Configuration key for table :param gsi_name: Name of the GSI being checked :param gsi_key: Configuration key for the GSI :returns: bool -- True if the circuit is open """ logger.debug('Checking circuit breaker status') # Parse the URL to make sure it is OK pattern = re.compile( r'^(?P<scheme>http(s)?://)' r'((?P<username>.+):(?P<password>.+)@){0,1}' r'(?P<url>.*)$' ) url = timeout = None if gsi_name: url = get_gsi_option(table_key, gsi_key, 'circuit_breaker_url') timeout = get_gsi_option(table_key, gsi_key, 'circuit_breaker_timeout') elif table_name: url = get_table_option(table_key, 'circuit_breaker_url') timeout = get_table_option(table_key, 'circuit_breaker_timeout') if not url: url = get_global_option('circuit_breaker_url') timeout = get_global_option('circuit_breaker_timeout') match = pattern.match(url) if not match: logger.error('Malformatted URL: {0}'.format(url)) sys.exit(1) use_basic_auth = False if match.group('username') and match.group('password'): use_basic_auth = True # Make the actual URL to call auth = () if use_basic_auth: url = '{scheme}{url}'.format( scheme=match.group('scheme'), url=match.group('url')) auth = (match.group('username'), match.group('password')) headers = {} if table_name: headers["x-table-name"] = table_name if gsi_name: headers["x-gsi-name"] = gsi_name # Make the actual request try: response = requests.get( url, auth=auth, timeout=timeout / 1000.00, headers=headers) if int(response.status_code) == 200: logger.info('Circuit breaker is closed') return False else: logger.warning( 'Circuit breaker returned with status code {0:d}'.format( response.status_code)) except requests.exceptions.SSLError as error: logger.warning('Circuit breaker: {0}'.format(error)) except requests.exceptions.Timeout as error: logger.warning('Circuit breaker: {0}'.format(error)) except requests.exceptions.ConnectionError as error: logger.warning('Circuit breaker: {0}'.format(error)) except requests.exceptions.HTTPError as error: logger.warning('Circuit breaker: {0}'.format(error)) except requests.exceptions.TooManyRedirects as error: logger.warning('Circuit breaker: {0}'.format(error)) except Exception as error: logger.error('Unhandled exception: {0}'.format(error)) logger.error( 'Please file a bug at ' 'https://github.com/sebdah/dynamic-dynamodb/issues') return True
def update_gsi_provisioning(table_name, table_key, gsi_name, gsi_key, reads, writes, retry_with_only_increase=False): """ Update provisioning on a global secondary index :type table_name: str :param table_name: Name of the DynamoDB table :type table_key: str :param table_key: Table configuration option key name :type gsi_name: str :param gsi_name: Name of the GSI :type gsi_key: str :param gsi_key: GSI configuration option key name :type reads: int :param reads: Number of reads to provision :type writes: int :param writes: Number of writes to provision :type retry_with_only_increase: bool :param retry_with_only_increase: Set to True to ensure only increases """ current_reads = int(get_provisioned_table_read_units(table_name)) current_writes = int(get_provisioned_table_write_units(table_name)) if retry_with_only_increase: # Ensure that we are only doing increases if current_reads > reads: reads = current_reads if current_writes > writes: writes = current_writes logger.info( '{0} - Retrying to update provisioning, excluding any decreases. ' 'Setting new reads to {1} and new writes to {2}'.format( table_name, reads, writes)) # Check that we are in the right time frame m_windows = get_gsi_option(table_key, gsi_key, 'maintenance_windows') if m_windows: if not __is_gsi_maintenance_window(table_name, gsi_name, m_windows): logger.warning( '{0} - GSI: {1} - We are outside a maintenace window. ' 'Will only perform up scaling activites'.format( table_name, gsi_name)) # Ensure that we are only doing increases if current_reads > reads: reads = current_reads if current_writes > writes: writes = current_writes # Return if we do not need to scale up if reads == current_reads and writes == current_writes: logger.info('{0} - GSI: {1} - ' 'No need to scale up reads nor writes'.format( table_name, gsi_name)) return else: logger.info('{0} - GSI: {1} - ' 'Current time is within maintenance window'.format( table_name, gsi_name)) logger.info('{0} - GSI: {1} - ' 'Updating provisioning to {2} reads and {3} writes'.format( table_name, gsi_name, reads, writes)) # Return if dry-run if get_global_option('dry_run'): return try: DYNAMODB_CONNECTION.update_table(table_name=table_name, global_secondary_index_updates=[{ "Update": { "IndexName": gsi_name, "ProvisionedThroughput": { "ReadCapacityUnits": reads, "WriteCapacityUnits": writes } } }]) message = ('{0} - GSI: {1} - Provisioning updated to ' '{2} reads and {3} writes').format(table_name, gsi_name, reads, writes) # See if we should send notifications for scale-down, scale-up or both sns_message_types = [] if current_reads > reads or current_writes > current_writes: sns_message_types.append('scale-down') if current_reads < reads or current_writes < current_writes: sns_message_types.append('scale-up') sns.publish_gsi_notification( table_key, gsi_key, message, sns_message_types, subject='Updated provisioning for GSI {0}'.format(gsi_name)) except JSONResponseError as error: exception = error.body['__type'].split('#')[1] know_exceptions = ['LimitExceededException'] if exception in know_exceptions: logger.warning('{0} - GSI: {1} - {2}: {3}'.format( table_name, gsi_name, exception, error.body['message'])) else: logger.error( ('{0} - GSI: {1} - Unhandled exception: {2}: {3}. ' 'Please file a bug report at ' 'https://github.com/sebdah/dynamic-dynamodb/issues').format( table_name, gsi_name, exception, error.body['message'])) if (not retry_with_only_increase and exception == 'LimitExceededException'): logger.info('{0} - GSI: {1} - Will retry to update provisioning ' 'with only increases'.format(table_name, gsi_name)) update_gsi_provisioning(table_name, table_key, gsi_name, gsi_key, reads, writes, retry_with_only_increase=True)
def update_table_provisioning(table_name, key_name, reads, writes, retry_with_only_increase=False): """ Update provisioning for a given table :type table_name: str :param table_name: Name of the table :type key_name: str :param key_name: Configuration option key name :type reads: int :param reads: New number of provisioned read units :type writes: int :param writes: New number of provisioned write units :type retry_with_only_increase: bool :param retry_with_only_increase: Set to True to ensure only increases """ table = get_table(table_name) current_reads = int(get_provisioned_table_read_units(table_name)) current_writes = int(get_provisioned_table_write_units(table_name)) if retry_with_only_increase: # Ensure that we are only doing increases if current_reads > reads: reads = current_reads if current_writes > writes: writes = current_writes logger.info( '{0} - Retrying to update provisioning, excluding any decreases. ' 'Setting new reads to {1} and new writes to {2}'.format( table_name, reads, writes)) # Check that we are in the right time frame maintenance_windows = get_table_option(key_name, 'maintenance_windows') if maintenance_windows: if not __is_table_maintenance_window(table_name, maintenance_windows): logger.warning( '{0} - We are outside a maintenace window. ' 'Will only perform up scaling activites'.format(table_name)) # Ensure that we are only doing increases if current_reads > reads: reads = current_reads if current_writes > writes: writes = current_writes # Return if we do not need to scale up if reads == current_reads and writes == current_writes: logger.info( '{0} - No need to scale up reads nor writes'.format( table_name)) return else: logger.info( '{0} - Current time is within maintenance window'.format( table_name)) logger.info( '{0} - Updating provisioning to {1} reads and {2} writes'.format( table_name, reads, writes)) # Return if dry-run if get_global_option('dry_run'): return try: table.update(throughput={'read': reads, 'write': writes}) # See if we should send notifications for scale-down, scale-up or both sns_message_types = [] if current_reads > reads or current_writes > current_writes: sns_message_types.append('scale-down') if current_reads < reads or current_writes < current_writes: sns_message_types.append('scale-up') message = ( '{0} - Provisioning updated to {1} reads and {2} writes').format( table_name, reads, writes) sns.publish_table_notification( key_name, message, sns_message_types, subject='Updated provisioning for table {0}'.format(table_name)) except JSONResponseError as error: exception = error.body['__type'].split('#')[1] know_exceptions = [ 'LimitExceededException', 'ValidationException', 'ResourceInUseException' ] if exception in know_exceptions: logger.warning('{0} - {1}: {2}'.format(table_name, exception, error.body['message'])) else: logger.error( ('{0} - Unhandled exception: {1}: {2}. ' 'Please file a bug report at ' 'https://github.com/sebdah/dynamic-dynamodb/issues').format( table_name, exception, error.body['message'])) if (not retry_with_only_increase and exception == 'LimitExceededException'): logger.info('{0} - Will retry to update provisioning ' 'with only increases'.format(table_name)) update_table_provisioning(table_name, key_name, reads, writes, retry_with_only_increase=True)
def update_gsi_provisioning(table_name, table_key, gsi_name, gsi_key, reads, writes, retry_with_only_increase=False): """ Update provisioning on a global secondary index :type table_name: str :param table_name: Name of the DynamoDB table :type table_key: str :param table_key: Table configuration option key name :type gsi_name: str :param gsi_name: Name of the GSI :type gsi_key: str :param gsi_key: GSI configuration option key name :type reads: int :param reads: Number of reads to provision :type writes: int :param writes: Number of writes to provision :type retry_with_only_increase: bool :param retry_with_only_increase: Set to True to ensure only increases """ current_reads = int(get_provisioned_gsi_read_units(table_name, gsi_name)) current_writes = int(get_provisioned_gsi_write_units(table_name, gsi_name)) if retry_with_only_increase: # Ensure that we are only doing increases if current_reads > reads: reads = current_reads if current_writes > writes: writes = current_writes # Return if we do not need to scale at all if reads == current_reads and writes == current_writes: logger.info("{0} - GSI: {1} - No need to scale up reads nor writes".format(table_name, gsi_name)) return logger.info( "{0} - GSI: {1} - Retrying to update provisioning, excluding any decreases. " "Setting new reads to {2} and new writes to {3}".format(table_name, gsi_name, reads, writes) ) # Check that we are in the right time frame m_windows = get_gsi_option(table_key, gsi_key, "maintenance_windows") if m_windows: if not __is_gsi_maintenance_window(table_name, gsi_name, m_windows): logger.warning( "{0} - GSI: {1} - We are outside a maintenace window. " "Will only perform up scaling activites".format(table_name, gsi_name) ) # Ensure that we are only doing increases if current_reads > reads: reads = current_reads if current_writes > writes: writes = current_writes # Return if we do not need to scale up if reads == current_reads and writes == current_writes: logger.info("{0} - GSI: {1} - " "No need to scale up reads nor writes".format(table_name, gsi_name)) return else: logger.info("{0} - GSI: {1} - " "Current time is within maintenance window".format(table_name, gsi_name)) logger.info( "{0} - GSI: {1} - " "Updating provisioning to {2} reads and {3} writes".format(table_name, gsi_name, reads, writes) ) # Return if dry-run if get_global_option("dry_run"): return try: DYNAMODB_CONNECTION.update_table( table_name=table_name, global_secondary_index_updates=[ { "Update": { "IndexName": gsi_name, "ProvisionedThroughput": {"ReadCapacityUnits": reads, "WriteCapacityUnits": writes}, } } ], ) message = ("{0} - GSI: {1} - Provisioning updated to " "{2} reads and {3} writes").format( table_name, gsi_name, reads, writes ) # See if we should send notifications for scale-down, scale-up or both sns_message_types = [] if current_reads > reads or current_writes > current_writes: sns_message_types.append("scale-down") if current_reads < reads or current_writes < current_writes: sns_message_types.append("scale-up") sns.publish_gsi_notification( table_key, gsi_key, message, sns_message_types, subject="Updated provisioning for GSI {0}".format(gsi_name) ) except JSONResponseError as error: exception = error.body["__type"].split("#")[1] know_exceptions = ["LimitExceededException"] if exception in know_exceptions: logger.warning("{0} - GSI: {1} - {2}: {3}".format(table_name, gsi_name, exception, error.body["message"])) else: logger.error( ( "{0} - GSI: {1} - Unhandled exception: {2}: {3}. " "Please file a bug report at " "https://github.com/sebdah/dynamic-dynamodb/issues" ).format(table_name, gsi_name, exception, error.body["message"]) ) if not retry_with_only_increase and exception == "LimitExceededException": logger.info( "{0} - GSI: {1} - Will retry to update provisioning " "with only increases".format(table_name, gsi_name) ) update_gsi_provisioning( table_name, table_key, gsi_name, gsi_key, reads, writes, retry_with_only_increase=True )
def __update_throughput(table_name, read_units, write_units, key_name): """ Update throughput on the DynamoDB table :type table_name: str :param table_name: Name of the DynamoDB table :type read_units: int :param read_units: New read unit provisioning :type write_units: int :param write_units: New write unit provisioning :type key_name: str :param key_name: Configuration option key name """ provisioned_reads = table_stats.get_provisioned_read_units(table_name) provisioned_writes = table_stats.get_provisioned_write_units(table_name) # Check that we are in the right time frame if get_table_option(key_name, 'maintenance_windows'): if (not __is_maintenance_window(table_name, get_table_option( key_name, 'maintenance_windows'))): logger.warning( '{0} - Current time is outside maintenance window'.format( table_name)) return else: logger.info( '{0} - Current time is within maintenance window'.format( table_name)) # Check table status table_status = dynamodb.get_table_status(table_name) logger.debug('{0} - Table status is {1}'.format(table_name, table_status)) if table_status != 'ACTIVE': logger.warning( '{0} - Not performing throughput changes when table ' 'is {1}'.format(table_name, table_status)) return # If this setting is True, we will only scale down when # BOTH reads AND writes are low if get_table_option(key_name, 'always_decrease_rw_together'): if read_units < provisioned_reads and write_units < provisioned_writes: logger.debug( '{0} - Both reads and writes will be decreased'.format( table_name)) elif read_units < provisioned_reads: logger.info( '{0} - Will not decrease reads nor writes, waiting for ' 'both to become low before decrease'.format(table_name)) return elif write_units < provisioned_writes: logger.info( '{0} - Will not decrease reads nor writes, waiting for ' 'both to become low before decrease'.format(table_name)) return if not get_global_option('dry_run'): dynamodb.update_table_provisioning( table_name, int(read_units), int(write_units))
def is_open(table_name=None, table_key=None, gsi_name=None, gsi_key=None): """ Checks whether the circuit breaker is open :param table_name: Name of the table being checked :param table_key: Configuration key for table :param gsi_name: Name of the GSI being checked :param gsi_key: Configuration key for the GSI :returns: bool -- True if the circuit is open """ logger.debug('Checking circuit breaker status') # Parse the URL to make sure it is OK pattern = re.compile(r'^(?P<scheme>http(s)?://)' r'((?P<username>.+):(?P<password>.+)@){0,1}' r'(?P<url>.*)$') url = timeout = None if gsi_name: url = get_gsi_option(table_key, gsi_key, 'circuit_breaker_url') timeout = get_gsi_option(table_key, gsi_key, 'circuit_breaker_timeout') elif table_name: url = get_table_option(table_key, 'circuit_breaker_url') timeout = get_table_option(table_key, 'circuit_breaker_timeout') if not url: url = get_global_option('circuit_breaker_url') timeout = get_global_option('circuit_breaker_timeout') match = pattern.match(url) if not match: logger.error('Malformatted URL: {0}'.format(url)) sys.exit(1) use_basic_auth = False if match.group('username') and match.group('password'): use_basic_auth = True # Make the actual URL to call auth = () if use_basic_auth: url = '{scheme}{url}'.format(scheme=match.group('scheme'), url=match.group('url')) auth = (match.group('username'), match.group('password')) headers = {} if table_name: headers["x-table-name"] = table_name if gsi_name: headers["x-gsi-name"] = gsi_name # Make the actual request try: response = requests.get(url, auth=auth, timeout=timeout / 1000.00, headers=headers) if int(response.status_code) == 200: logger.info('Circuit breaker is closed') return False else: logger.warning( 'Circuit breaker returned with status code {0:d}'.format( response.status_code)) except requests.exceptions.SSLError as error: logger.warning('Circuit breaker: {0}'.format(error)) except requests.exceptions.Timeout as error: logger.warning('Circuit breaker: {0}'.format(error)) except requests.exceptions.ConnectionError as error: logger.warning('Circuit breaker: {0}'.format(error)) except requests.exceptions.HTTPError as error: logger.warning('Circuit breaker: {0}'.format(error)) except requests.exceptions.TooManyRedirects as error: logger.warning('Circuit breaker: {0}'.format(error)) except Exception as error: logger.error('Unhandled exception: {0}'.format(error)) logger.error('Please file a bug at ' 'https://github.com/sebdah/dynamic-dynamodb/issues') return True
def __circuit_breaker_is_open(): """ Checks wether the circuit breaker is open :returns: bool -- True if the circuit is open """ logger.debug('Checking circuit breaker status') # Parse the URL to make sure it is OK pattern = re.compile( r'^(?P<scheme>http(s)?://)' r'((?P<username>.+):(?P<password>.+)@){0,1}' r'(?P<url>.*)$' ) match = pattern.match(get_global_option('circuit_breaker_url')) if not match: logger.error('Malformatted URL: {0}'.format( get_global_option('circuit_breaker_url'))) sys.exit(1) use_basic_auth = False if match.group('username') and match.group('password'): use_basic_auth = True # Make the actual URL to call if use_basic_auth: url = '{scheme}{url}'.format( scheme=match.group('scheme'), url=match.group('url')) auth = (match.group('username'), match.group('password')) else: url = get_global_option('circuit_breaker_url') auth = () # Make the actual request try: response = requests.get( url, auth=auth, timeout=get_global_option('circuit_breaker_timeout') / 1000.00) if int(response.status_code) == 200: logger.info('Circuit breaker is closed') return False else: logger.warning( 'Circuit breaker returned with status code {0:d}'.format( response.status_code)) except requests.exceptions.SSLError as error: logger.warning('Circuit breaker: {0}'.format(error)) except requests.exceptions.Timeout as error: logger.warning('Circuit breaker: {0}'.format(error)) except requests.exceptions.ConnectionError as error: logger.warning('Circuit breaker: {0}'.format(error)) except requests.exceptions.HTTPError as error: logger.warning('Circuit breaker: {0}'.format(error)) except requests.exceptions.TooManyRedirects as error: logger.warning('Circuit breaker: {0}'.format(error)) except Exception as error: logger.error('Unhandled exception: {0}'.format(error)) logger.error( 'Please file a bug at ' 'https://github.com/sebdah/dynamic-dynamodb/issues') return True
def __update_throughput( table_name, table_key, gsi_name, gsi_key, read_units, write_units): """ Update throughput on the GSI :type table_name: str :param table_name: Name of the DynamoDB table :type table_key: str :param table_key: Table configuration option key name :type gsi_name: str :param gsi_name: Name of the GSI :type gsi_key: str :param gsi_key: Configuration option key name :type read_units: int :param read_units: New read unit provisioning :type write_units: int :param write_units: New write unit provisioning """ try: current_ru = dynamodb.get_provisioned_gsi_read_units( table_name, gsi_name) current_wu = dynamodb.get_provisioned_gsi_write_units( table_name, gsi_name) except JSONResponseError: raise # Check that we are in the right time frame if get_gsi_option(table_key, gsi_key, 'maintenance_windows'): if (not __is_maintenance_window(table_name, gsi_name, get_gsi_option( table_key, gsi_key, 'maintenance_windows'))): logger.warning( '{0} - GSI: {1} - ' 'Current time is outside maintenance window'.format( table_name, gsi_name)) return else: logger.info( '{0} - GSI: {1} - ' 'Current time is within maintenance window'.format( table_name, gsi_name)) # Check table status try: gsi_status = dynamodb.get_gsi_status(table_name, gsi_name) except JSONResponseError: raise logger.debug('{0} - GSI: {1} - GSI status is {2}'.format( table_name, gsi_name, gsi_status)) if gsi_status != 'ACTIVE': logger.warning( '{0} - GSI: {1} - Not performing throughput changes when GSI ' 'status is {2}'.format(table_name, gsi_name, gsi_status)) return # If this setting is True, we will only scale down when # BOTH reads AND writes are low if get_gsi_option(table_key, gsi_key, 'always_decrease_rw_together'): read_units, write_units = __calculate_always_decrease_rw_values( table_name, gsi_name, read_units, current_ru, write_units, current_wu) if read_units == current_ru and write_units == current_wu: logger.info('{0} - GSI: {1} - No changes to perform'.format( table_name, gsi_name)) return if not get_global_option('dry_run'): dynamodb.update_gsi_provisioning( table_name, table_key, gsi_name, gsi_key, int(read_units), int(write_units)) logger.info( '{0} - GSI: {1} - ' 'Provisioning updated to {2} reads and {3} writes'.format( table_name, gsi_name, read_units, write_units))
def __update_throughput( table_name, table_key, gsi_name, gsi_key, read_units, write_units): """ Update throughput on the GSI :type table_name: str :param table_name: Name of the DynamoDB table :type table_key: str :param table_key: Table configuration option key name :type gsi_name: str :param gsi_name: Name of the GSI :type gsi_key: str :param gsi_key: Configuration option key name :type read_units: int :param read_units: New read unit provisioning :type write_units: int :param write_units: New write unit provisioning """ current_ru = gsi_stats.get_provisioned_read_units( table_name, gsi_name) current_wu = gsi_stats.get_provisioned_write_units( table_name, gsi_name) # Check that we are in the right time frame if get_gsi_option(table_key, gsi_key, 'maintenance_windows'): if (not __is_maintenance_window(table_name, gsi_name, get_gsi_option( table_key, gsi_key, 'maintenance_windows'))): logger.warning( '{0} - GSI: {1} - ' 'Current time is outside maintenance window'.format( table_name, gsi_name)) return else: logger.info( '{0} - GSI: {1} - ' 'Current time is within maintenance window'.format( table_name, gsi_name)) # Check table status gsi_status = dynamodb.get_gsi_status(table_name, gsi_name) logger.debug('{0} - GSI: {1} - GSI status is {2}'.format( table_name, gsi_name, gsi_status)) if gsi_status != 'ACTIVE': logger.warning( '{0} - GSI: {1} - Not performing throughput changes when GSI ' 'status is {2}'.format(table_name, gsi_name, gsi_status)) return # If this setting is True, we will only scale down when # BOTH reads AND writes are low if get_gsi_option(table_key, gsi_key, 'always_decrease_rw_together'): if read_units < current_ru and write_units < current_wu: logger.debug( '{0} - GSI: {1} - ' 'Both reads and writes will be decreased'.format( table_name, gsi_name)) elif read_units < current_ru: logger.info( '{0} - GSI: {1} - ' 'Will not decrease reads nor writes, waiting for ' 'both to become low before decrease'.format( table_name, gsi_name)) return elif write_units < current_wu: logger.info( '{0} - GSI: {1} - ' 'Will not decrease reads nor writes, waiting for ' 'both to become low before decrease'.format( table_name, gsi_name)) return if not get_global_option('dry_run'): dynamodb.update_gsi_provisioning( table_name, gsi_name, int(read_units), int(write_units)) logger.info( '{0} - GSI: {1} - ' 'Provisioning updated to {2} reads and {3} writes'.format( table_name, gsi_name, read_units, write_units))
def main(): """ Main function called from dynamic-dynamodb """ while True: table_names = set() configured_tables = config['tables'].keys() not_used_tables = configured_tables # Add regexp table names for table_instance in dynamodb.list_tables(): for key_name in configured_tables: if re.match(key_name, table_instance.table_name): logger.debug("Table {0} match with config key {1}".format( table_instance.table_name, key_name)) table_names.add( ( table_instance.table_name, key_name )) not_used_tables.remove(key_name) if not_used_tables: logger.warning( 'No tables matching the following configured ' 'tables found: {0}'.format( ', '.join(not_used_tables))) table_names = sorted(table_names) if config['global']['daemon']: pid_file = '/tmp/dynamic-dynamodb.{0}.pid'.format( config['global']['instance']) daemon = DynamicDynamoDBDaemon(pid_file, tables=table_names) if config['global']['daemon'] == 'start': daemon.start( check_interval=config['global']['check_interval']) elif config['global']['daemon'] == 'stop': daemon.stop() sys.exit(0) elif config['global']['daemon'] == 'restart': daemon.restart() elif config['global']['daemon'] in ['foreground', 'fg']: daemon.run( check_interval=config['global']['check_interval']) else: print 'Valid options for --daemon are start, stop and restart' sys.exit(1) else: # Ensure provisioning for table_name, table_key in table_names: table.ensure_provisioning(table_name, table_key) gsi_names = set() # Add regexp table names if 'gsis' in config['tables'][table_key]: for gst_instance in dynamodb.table_gsis(table_name): gsi_name = gst_instance[u'IndexName'] gsi_keys = config['tables'][table_key]['gsis'].keys() for gsi_key in gsi_keys: if re.match(gsi_key, gsi_name): logger.debug( 'Table {0} GSI {1} match with ' 'GSI config key {2}'.format( table_name, gsi_name, gsi_key)) gsi_names.add( ( gsi_name, gsi_key )) gsi_names = sorted(gsi_names) for gsi_name, gsi_key in gsi_names: gsi.ensure_provisioning( table_name, table_key, gsi_name, gsi_key) # Sleep between the checks logger.debug('Sleeping {0} seconds until next check'.format( config['global']['check_interval'])) time.sleep(config['global']['check_interval'])
def update_table_provisioning( table_name, reads, writes, retry_with_only_increase=False): """ Update provisioning for a given table :type table_name: str :param table_name: Name of the table :type reads: int :param reads: New number of provisioned read units :type writes: int :param writes: New number of provisioned write units :type retry_with_only_increase: bool :param retry_with_only_increase: Set to True to ensure only increases """ table = get_table(table_name) if retry_with_only_increase: current_reads = int(get_provisioned_table_read_units(table_name)) current_writes = int(get_provisioned_table_write_units(table_name)) # Ensure that we are only doing increases if current_reads > reads: reads = current_reads if current_writes > writes: writes = current_writes logger.info( '{0} - Retrying to update provisioning, excluding any decreases. ' 'Setting new reads to {1} and new writes to {2}'.format( table_name, reads, writes)) try: table.update( throughput={ 'read': reads, 'write': writes }) logger.info( '{0} - Provisioning updated to {1} reads and {2} writes'.format( table_name, reads, writes)) except JSONResponseError as error: exception = error.body['__type'].split('#')[1] know_exceptions = [ 'LimitExceededException', 'ValidationException', 'ResourceInUseException'] if exception in know_exceptions: logger.warning('{0} - {1}: {2}'.format( table_name, exception, error.body['message'])) else: logger.error( ( '{0} - Unhandled exception: {1}: {2}. ' 'Please file a bug report at ' 'https://github.com/sebdah/dynamic-dynamodb/issues' ).format(table_name, exception, error.body['message'])) if (not retry_with_only_increase and exception == 'LimitExceededException'): logger.info( '{0} - Will retry to update provisioning ' 'with only increases'.format(table_name)) update_table_provisioning( table_name, reads, writes, retry_with_only_increase=True)
def ensure_provisioning( table_name, key_name, num_consec_read_checks, num_consec_write_checks): """ Ensure that provisioning is correct :type table_name: str :param table_name: Name of the DynamoDB table :type key_name: str :param key_name: Configuration option key name :type num_consec_read_checks: int :param num_consec_read_checks: How many consecutive checks have we had :type num_consec_write_checks: int :param num_consec_write_checks: How many consecutive checks have we had :returns: (int, int) -- num_consec_read_checks, num_consec_write_checks """ if get_global_option('circuit_breaker_url') or get_table_option( key_name, 'circuit_breaker_url'): if circuit_breaker.is_open(table_name, key_name): logger.warning('Circuit breaker is OPEN!') return (0, 0) # Handle throughput alarm checks __ensure_provisioning_alarm(table_name, key_name) ts = TimeSeriesTable(get_table_option('time_series_tables', 'time_series_tables'), get_table_option('time_series_tables_no_scale_period_in_seconds', 'time_series_tables_no_scale_period_in_seconds')) if ts.is_in_future(table_name): logger.info('Time series table ' + table_name + " is in the future, skipping provisioning") return (0, 0) try: read_update_needed, updated_read_units, num_consec_read_checks = \ __ensure_provisioning_reads( table_name, key_name, num_consec_read_checks) write_update_needed, updated_write_units, num_consec_write_checks = \ __ensure_provisioning_writes( table_name, key_name, num_consec_write_checks) if read_update_needed: num_consec_read_checks = 0 if write_update_needed: num_consec_write_checks = 0 # Handle throughput updates if read_update_needed or write_update_needed: logger.info( '{0} - Changing provisioning to {1:d} ' 'read units and {2:d} write units'.format( table_name, int(updated_read_units), int(updated_write_units))) __update_throughput( table_name, key_name, updated_read_units, updated_write_units) else: logger.info('{0} - No need to change provisioning'.format( table_name)) except JSONResponseError: raise except BotoServerError: raise return num_consec_read_checks, num_consec_write_checks
def update_table_provisioning( table_name, key_name, reads, writes, retry_with_only_increase=False): """ Update provisioning for a given table :type table_name: str :param table_name: Name of the table :type key_name: str :param key_name: Configuration option key name :type reads: int :param reads: New number of provisioned read units :type writes: int :param writes: New number of provisioned write units :type retry_with_only_increase: bool :param retry_with_only_increase: Set to True to ensure only increases """ table = get_table(table_name) current_reads = int(get_provisioned_table_read_units(table_name)) current_writes = int(get_provisioned_table_write_units(table_name)) # Make sure we aren't scaling down if we turned off downscaling if (not get_table_option(key_name, 'enable_reads_down_scaling') or not get_table_option(key_name, 'enable_writes_down_scaling')): if (not get_table_option(key_name, 'enable_reads_down_scaling') and current_reads > reads): reads = current_reads if (not get_table_option(key_name, 'enable_writes_down_scaling') and current_writes > writes): writes = current_writes # Return if we do not need to scale at all if reads == current_reads and writes == current_writes: logger.info( '{0} - No need to scale up reads nor writes'.format( table_name)) return if retry_with_only_increase: # Ensure that we are only doing increases if current_reads > reads: reads = current_reads if current_writes > writes: writes = current_writes # Return if we do not need to scale at all if reads == current_reads and writes == current_writes: logger.info( '{0} - No need to scale up reads nor writes'.format( table_name)) return logger.info( '{0} - Retrying to update provisioning, excluding any decreases. ' 'Setting new reads to {1} and new writes to {2}'.format( table_name, reads, writes)) # Check that we are in the right time frame maintenance_windows = get_table_option(key_name, 'maintenance_windows') if maintenance_windows: if not __is_table_maintenance_window(table_name, maintenance_windows): logger.warning( '{0} - We are outside a maintenace window. ' 'Will only perform up scaling activites'.format(table_name)) # Ensure that we are only doing increases if current_reads > reads: reads = current_reads if current_writes > writes: writes = current_writes # Return if we do not need to scale up if reads == current_reads and writes == current_writes: logger.info( '{0} - No need to scale up reads nor writes'.format( table_name)) return else: logger.info( '{0} - Current time is within maintenance window'.format( table_name)) logger.info( '{0} - Updating provisioning to {1} reads and {2} writes'.format( table_name, reads, writes)) # Return if dry-run if get_global_option('dry_run'): return try: table.update( throughput={ 'read': reads, 'write': writes }) # See if we should send notifications for scale-down, scale-up or both sns_message_types = [] if current_reads > reads or current_writes > writes: sns_message_types.append('scale-down') if current_reads < reads or current_writes < writes: sns_message_types.append('scale-up') message = [] if current_reads > reads: message.append('{0} - Reads: DOWN from {1} to {2}\n'.format( table_name, current_reads, reads)) elif current_reads < reads: message.append('{0} - Reads: UP from {1} to {2}\n'.format( table_name, current_reads, reads)) if current_writes > writes: message.append('{0} - Writes: DOWN from {1} to {2}\n'.format( table_name, current_writes, writes)) elif current_writes < writes: message.append('{0} - Writes: UP from {1} to {2}\n'.format( table_name, current_writes, writes)) sns.publish_table_notification( key_name, ''.join(message), sns_message_types, subject='Updated provisioning for table {0}'.format(table_name)) except JSONResponseError as error: exception = error.body['__type'].split('#')[1] know_exceptions = [ 'LimitExceededException', 'ValidationException', 'ResourceInUseException'] if exception in know_exceptions: logger.warning('{0} - {1}: {2}'.format( table_name, exception, error.body['message'])) else: if 'message' in error.body: msg = error.body['message'] else: msg = error logger.error( ( '{0} - Unhandled exception: {1}: {2}. ' 'Please file a bug report at ' 'https://github.com/sebdah/dynamic-dynamodb/issues' ).format(table_name, exception, msg)) if (not retry_with_only_increase and exception == 'LimitExceededException'): logger.info( '{0} - Will retry to update provisioning ' 'with only increases'.format(table_name)) update_table_provisioning( table_name, key_name, reads, writes, retry_with_only_increase=True)
def update_table_provisioning(table_name, key_name, reads, writes, retry_with_only_increase=False): """ Update provisioning for a given table :type table_name: str :param table_name: Name of the table :type key_name: str :param key_name: Configuration option key name :type reads: int :param reads: New number of provisioned read units :type writes: int :param writes: New number of provisioned write units :type retry_with_only_increase: bool :param retry_with_only_increase: Set to True to ensure only increases """ table = get_table(table_name) current_reads = int(get_provisioned_table_read_units(table_name)) current_writes = int(get_provisioned_table_write_units(table_name)) if retry_with_only_increase: # Ensure that we are only doing increases if current_reads > reads: reads = current_reads if current_writes > writes: writes = current_writes # Return if we do not need to scale at all if reads == current_reads and writes == current_writes: logger.info("{0} - No need to scale up reads nor writes".format(table_name)) return logger.info( "{0} - Retrying to update provisioning, excluding any decreases. " "Setting new reads to {1} and new writes to {2}".format(table_name, reads, writes) ) # Check that we are in the right time frame maintenance_windows = get_table_option(key_name, "maintenance_windows") if maintenance_windows: if not __is_table_maintenance_window(table_name, maintenance_windows): logger.warning( "{0} - We are outside a maintenace window. " "Will only perform up scaling activites".format(table_name) ) # Ensure that we are only doing increases if current_reads > reads: reads = current_reads if current_writes > writes: writes = current_writes # Return if we do not need to scale up if reads == current_reads and writes == current_writes: logger.info("{0} - No need to scale up reads nor writes".format(table_name)) return else: logger.info("{0} - Current time is within maintenance window".format(table_name)) logger.info("{0} - Updating provisioning to {1} reads and {2} writes".format(table_name, reads, writes)) # Return if dry-run if get_global_option("dry_run"): return try: table.update(throughput={"read": reads, "write": writes}) # See if we should send notifications for scale-down, scale-up or both sns_message_types = [] if current_reads > reads or current_writes > current_writes: sns_message_types.append("scale-down") if current_reads < reads or current_writes < current_writes: sns_message_types.append("scale-up") message = ("{0} - Provisioning updated to {1} reads and {2} writes").format(table_name, reads, writes) sns.publish_table_notification( key_name, message, sns_message_types, subject="Updated provisioning for table {0}".format(table_name) ) except JSONResponseError as error: exception = error.body["__type"].split("#")[1] know_exceptions = ["LimitExceededException", "ValidationException", "ResourceInUseException"] if exception in know_exceptions: logger.warning("{0} - {1}: {2}".format(table_name, exception, error.body["message"])) else: logger.error( ( "{0} - Unhandled exception: {1}: {2}. " "Please file a bug report at " "https://github.com/sebdah/dynamic-dynamodb/issues" ).format(table_name, exception, error.body["message"]) ) if not retry_with_only_increase and exception == "LimitExceededException": logger.info("{0} - Will retry to update provisioning " "with only increases".format(table_name)) update_table_provisioning(table_name, key_name, reads, writes, retry_with_only_increase=True)
def update_gsi_provisioning( table_name, table_key, gsi_name, gsi_key, reads, writes, retry_with_only_increase=False): """ Update provisioning on a global secondary index :type table_name: str :param table_name: Name of the DynamoDB table :type table_key: str :param table_key: Table configuration option key name :type gsi_name: str :param gsi_name: Name of the GSI :type gsi_key: str :param gsi_key: GSI configuration option key name :type reads: int :param reads: Number of reads to provision :type writes: int :param writes: Number of writes to provision :type retry_with_only_increase: bool :param retry_with_only_increase: Set to True to ensure only increases """ current_reads = int(get_provisioned_gsi_read_units(table_name, gsi_name)) current_writes = int(get_provisioned_gsi_write_units(table_name, gsi_name)) # Make sure we aren't scaling down if we turned off downscaling if (not get_gsi_option(table_key, gsi_key, 'enable_reads_down_scaling') or not get_gsi_option( table_key, gsi_key, 'enable_writes_down_scaling')): if (not get_gsi_option( table_key, gsi_key, 'enable_reads_down_scaling') and current_reads > reads): reads = current_reads if (not get_gsi_option( table_key, gsi_key, 'enable_writes_down_scaling') and current_writes > writes): writes = current_writes # Return if we do not need to scale at all if reads == current_reads and writes == current_writes: logger.info( '{0} - No need to scale up reads nor writes'.format( table_name)) return if retry_with_only_increase: # Ensure that we are only doing increases if current_reads > reads: reads = current_reads if current_writes > writes: writes = current_writes # Return if we do not need to scale at all if reads == current_reads and writes == current_writes: logger.info( '{0} - GSI: {1} - No need to scale up reads nor writes'.format( table_name, gsi_name)) return logger.info( '{0} - GSI: {1} - Retrying to update provisioning, ' 'excluding any decreases. ' 'Setting new reads to {2} and new writes to {3}'.format( table_name, gsi_name, reads, writes)) # Check that we are in the right time frame m_windows = get_gsi_option(table_key, gsi_key, 'maintenance_windows') if m_windows: if not __is_gsi_maintenance_window(table_name, gsi_name, m_windows): logger.warning( '{0} - GSI: {1} - We are outside a maintenace window. ' 'Will only perform up scaling activites'.format( table_name, gsi_name)) # Ensure that we are only doing increases if current_reads > reads: reads = current_reads if current_writes > writes: writes = current_writes # Return if we do not need to scale up if reads == current_reads and writes == current_writes: logger.info( '{0} - GSI: {1} - ' 'No need to scale up reads nor writes'.format( table_name, gsi_name)) return else: logger.info( '{0} - GSI: {1} - ' 'Current time is within maintenance window'.format( table_name, gsi_name)) logger.info( '{0} - GSI: {1} - ' 'Updating provisioning to {2} reads and {3} writes'.format( table_name, gsi_name, reads, writes)) # Return if dry-run if get_global_option('dry_run'): return try: DYNAMODB_CONNECTION.update_table( table_name=table_name, global_secondary_index_updates=[ { "Update": { "IndexName": gsi_name, "ProvisionedThroughput": { "ReadCapacityUnits": reads, "WriteCapacityUnits": writes } } } ]) message = [] if current_reads > reads: message.append( '{0} - GSI: {1} - Reads: DOWN from {2} to {3}\n'.format( table_name, gsi_name, current_reads, reads)) elif current_reads < reads: message.append( '{0} - GSI: {1} - Reads: UP from {2} to {3}\n'.format( table_name, gsi_name, current_reads, reads)) if current_writes > writes: message.append( '{0} - GSI: {1} - Writes: DOWN from {2} to {3}\n'.format( table_name, gsi_name, current_writes, writes)) elif current_writes < writes: message.append( '{0} - GSI: {1} - Writes: UP from {2} to {3}\n'.format( table_name, gsi_name, current_writes, writes)) # See if we should send notifications for scale-down, scale-up or both sns_message_types = [] if current_reads > reads or current_writes > writes: sns_message_types.append('scale-down') if current_reads < reads or current_writes < writes: sns_message_types.append('scale-up') sns.publish_gsi_notification( table_key, gsi_key, ''.join(message), sns_message_types, subject='Updated provisioning for GSI {0}'.format(gsi_name)) except JSONResponseError as error: exception = error.body['__type'].split('#')[1] know_exceptions = ['LimitExceededException'] if exception in know_exceptions: logger.warning('{0} - GSI: {1} - {2}: {3}'.format( table_name, gsi_name, exception, error.body['message'])) else: logger.error( ( '{0} - GSI: {1} - Unhandled exception: {2}: {3}. ' 'Please file a bug report at ' 'https://github.com/sebdah/dynamic-dynamodb/issues' ).format( table_name, gsi_name, exception, error.body['message'])) if (not retry_with_only_increase and exception == 'LimitExceededException'): logger.info( '{0} - GSI: {1} - Will retry to update provisioning ' 'with only increases'.format(table_name, gsi_name)) update_gsi_provisioning( table_name, table_key, gsi_name, gsi_key, reads, writes, retry_with_only_increase=True)
def update_throughput(table_name, read_units, write_units, key_name): """ Update throughput on the DynamoDB table :type table_name: str :param table_name: Name of the DynamoDB table :type read_units: int :param read_units: New read unit provisioning :type write_units: int :param write_units: New write unit provisioning :type key_name: str :param key_name: Configuration option key name """ table = dynamodb.get_table(table_name) # Check that we are in the right time frame if get_table_option(key_name, 'maintenance_windows'): if not __is_maintenance_window(table_name, get_table_option(key_name, 'maintenance_windows')): logger.warning( '{0} - Current time is outside maintenance window'.format( table_name)) return else: logger.info( '{0} - Current time is within maintenance window'.format( table_name)) # Check table status if table.status != 'ACTIVE': logger.warning( '{0} - Not performing throughput changes when table ' 'is in {1} state'.format(table_name, table.status)) # If this setting is True, we will only scale down when # BOTH reads AND writes are low if get_table_option(key_name, 'always_decrease_rw_together'): if (read_units < table.read_units) or (table.read_units == get_table_option(key_name, 'min_provisioned_reads')): if (write_units < table.write_units) or (table.write_units == get_table_option(key_name, 'min_provisioned_writes')): logger.info( '{0} - Both reads and writes will be decreased'.format( table_name)) elif read_units < table.read_units: logger.info( '{0} - Will not decrease reads nor writes, waiting for ' 'both to become low before decrease'.format(table_name)) read_units = table.read_units elif write_units < table.write_units: logger.info( '{0} - Will not decrease reads nor writes, waiting for ' 'both to become low before decrease'.format(table_name)) write_units = table.write_units if read_units == table.read_units and write_units == table.write_units: logger.debug('{0} - No need to update provisioning') return if not get_global_option('dry_run'): try: table.update_throughput(int(read_units), int(write_units)) logger.info('Provisioning updated') except DynamoDBResponseError as error: dynamodb_error = error.body['__type'].rsplit('#', 1)[1] if dynamodb_error == 'LimitExceededException': logger.warning( '{0} - {1}'.format(table_name, error.body['message'])) if int(read_units) > table.read_units: logger.info('{0} - Scaling up reads to {1:d}'.format( table_name, int(read_units))) update_throughput( table_name, int(read_units), int(table.write_units), key_name) elif int(write_units) > table.write_units: logger.info('{0} - Scaling up writes to {1:d}'.format( table_name, int(write_units))) update_throughput( table_name, int(table.read_units), int(write_units), key_name) elif dynamodb_error == 'ValidationException': logger.warning('{0} - ValidationException: {1}'.format( table_name, error.body['message'])) elif dynamodb_error == 'ResourceInUseException': logger.warning('{0} - ResourceInUseException: {1}'.format( table_name, error.body['message'])) elif dynamodb_error == 'AccessDeniedException': logger.warning('{0} - AccessDeniedException: {1}'.format( table_name, error.body['message'])) else: logger.error( ( '{0} - Unhandled exception: {1}: {2}. ' 'Please file a bug report at ' 'https://github.com/sebdah/dynamic-dynamodb/issues' ).format( table_name, dynamodb_error, error.body['message']))
def update_gsi_provisioning( table_name, table_key, gsi_name, gsi_key, reads, writes, retry_with_only_increase=False): """ Update provisioning on a global secondary index :type table_name: str :param table_name: Name of the DynamoDB table :type table_key: str :param table_key: Table configuration option key name :type gsi_name: str :param gsi_name: Name of the GSI :type gsi_key: str :param gsi_key: GSI configuration option key name :type reads: int :param reads: Number of reads to provision :type writes: int :param writes: Number of writes to provision :type retry_with_only_increase: bool :param retry_with_only_increase: Set to True to ensure only increases """ current_reads = int(get_provisioned_table_read_units(table_name)) current_writes = int(get_provisioned_table_write_units(table_name)) if retry_with_only_increase: # Ensure that we are only doing increases if current_reads > reads: reads = current_reads if current_writes > writes: writes = current_writes logger.info( '{0} - Retrying to update provisioning, excluding any decreases. ' 'Setting new reads to {1} and new writes to {2}'.format( table_name, reads, writes)) try: DYNAMODB_CONNECTION.update_table( table_name=table_name, global_secondary_index_updates=[ { "Update": { "IndexName": gsi_name, "ProvisionedThroughput": { "ReadCapacityUnits": reads, "WriteCapacityUnits": writes } } } ]) message = ( '{0} - GSI: {1} - Provisioning updated to ' '{2} reads and {3} writes').format( table_name, gsi_name, reads, writes) logger.info(message) # See if we should send notifications for scale-down, scale-up or both sns_message_types = [] if current_reads > reads or current_writes > current_writes: sns_message_types.append('scale-down') if current_reads < reads or current_writes < current_writes: sns_message_types.append('scale-up') sns.publish_gsi_notification( table_key, gsi_key, message, sns_message_types, subject='Updated provisioning for GSI {0}'.format(gsi_name)) except JSONResponseError as error: exception = error.body['__type'].split('#')[1] know_exceptions = ['LimitExceededException'] if exception in know_exceptions: logger.warning('{0} - GSI: {1} - {2}: {3}'.format( table_name, gsi_name, exception, error.body['message'])) else: logger.error( ( '{0} - GSI: {1} - Unhandled exception: {2}: {3}. ' 'Please file a bug report at ' 'https://github.com/sebdah/dynamic-dynamodb/issues' ).format( table_name, gsi_name, exception, error.body['message'])) if (not retry_with_only_increase and exception == 'LimitExceededException'): logger.info( '{0} - GSI: {1} - Will retry to update provisioning ' 'with only increases'.format(table_name, gsi_name)) update_gsi_provisioning( table_name, table_key, gsi_name, gsi_key, reads, writes, retry_with_only_increase=True)
def update_gsi_provisioning( table_name, gsi_name, reads, writes, retry_with_only_increase=False): """ Update provisioning on a global secondary index :type table_name: str :param table_name: Name of the table :type gsi_name: str :param gsi_name: Name of the GSI :type reads: int :param reads: Number of reads to provision :type writes: int :param writes: Number of writes to provision :type retry_with_only_increase: bool :param retry_with_only_increase: Set to True to ensure only increases """ if retry_with_only_increase: current_reads = int(get_provisioned_table_read_units(table_name)) current_writes = int(get_provisioned_table_write_units(table_name)) # Ensure that we are only doing increases if current_reads > reads: reads = current_reads if current_writes > writes: writes = current_writes logger.info( '{0} - Retrying to update provisioning, excluding any decreases. ' 'Setting new reads to {1} and new writes to {2}'.format( table_name, reads, writes)) try: DYNAMODB_CONNECTION.update_table( table_name=table_name, global_secondary_index_updates=[ { "Update": { "IndexName": gsi_name, "ProvisionedThroughput": { "ReadCapacityUnits": reads, "WriteCapacityUnits": writes } } } ]) except JSONResponseError as error: exception = error.body['__type'].split('#')[1] know_exceptions = ['LimitExceededException'] if exception in know_exceptions: logger.warning('{0} - GSI: {1} - {2}: {3}'.format( table_name, gsi_name, exception, error.body['message'])) else: logger.error( ( '{0} - GSI: {1} - Unhandled exception: {2}: {3}. ' 'Please file a bug report at ' 'https://github.com/sebdah/dynamic-dynamodb/issues' ).format( table_name, gsi_name, exception, error.body['message'])) if (not retry_with_only_increase and exception == 'LimitExceededException'): logger.info( '{0} - GSI: {1} - Will retry to update provisioning ' 'with only increases'.format(table_name, gsi_name)) update_gsi_provisioning( table_name, gsi_name, reads, writes, retry_with_only_increase=True)
def ensure_provisioning(table_name, table_key, gsi_name, gsi_key, num_consec_read_checks, num_consec_write_checks): """ Ensure that provisioning is correct for Global Secondary Indexes :type table_name: str :param table_name: Name of the DynamoDB table :type table_key: str :param table_key: Table configuration option key name :type gsi_name: str :param gsi_name: Name of the GSI :type gsi_key: str :param gsi_key: GSI configuration option key name :type num_consec_read_checks: int :param num_consec_read_checks: How many consecutive checks have we had :type num_consec_write_checks: int :param num_consec_write_checks: How many consecutive checks have we had :returns: (int, int) -- num_consec_read_checks, num_consec_write_checks """ if get_global_option('circuit_breaker_url'): if circuit_breaker.is_open(): logger.warning('Circuit breaker is OPEN!') return (0, 0) logger.info( '{0} - Will ensure provisioning for global secondary index {1}'.format( table_name, gsi_name)) try: read_update_needed, updated_read_units, num_consec_read_checks = \ __ensure_provisioning_reads( table_name, table_key, gsi_name, gsi_key, num_consec_read_checks) write_update_needed, updated_write_units, num_consec_write_checks = \ __ensure_provisioning_writes( table_name, table_key, gsi_name, gsi_key, num_consec_write_checks) # Handle throughput updates if read_update_needed or write_update_needed: logger.info('{0} - GSI: {1} - Changing provisioning to {2:d} ' 'read units and {3:d} write units'.format( table_name, gsi_name, int(updated_read_units), int(updated_write_units))) __update_throughput(table_name, table_key, gsi_name, gsi_key, updated_read_units, updated_write_units) else: logger.info( '{0} - GSI: {1} - No need to change provisioning'.format( table_name, gsi_name)) except JSONResponseError: raise except BotoServerError: raise return num_consec_read_checks, num_consec_write_checks