def get_min_provisioned_reads(current_provisioning, table_name, key_name): """ Returns the minimum provisioned reads If the min_provisioned_reads value is less than current_provisioning * 2, then we return current_provisioning * 2, as DynamoDB cannot be scaled up with more than 100%. :type current_provisioning: int :param current_provisioning: The current provisioning :type table_name: str :param table_name: Name of the table :type key_name: str :param key_name: Name of the key :returns: int -- Minimum provisioned reads """ min_provisioned_reads = 1 if get_table_option(key_name, 'min_provisioned_reads'): min_provisioned_reads = int( get_table_option(key_name, 'min_provisioned_reads')) if min_provisioned_reads > int(current_provisioning * 2): min_provisioned_reads = int(current_provisioning * 2) logger.debug( '{0} - ' 'Cannot reach min_provisioned_reads as max scale up ' 'is 100% of current provisioning'.format( table_name)) logger.debug( '{0} - ' 'Setting min provisioned reads to {1}'.format( table_name, min_provisioned_reads)) return min_provisioned_reads
def decrease_reads_in_units(current_provisioning, units, key_name, table_name): """ Decrease the current_provisioning with units units :type current_provisioning: int :param current_provisioning: The current provisioning :type units: int :param units: How many units should we decrease with :returns: int -- New provisioning value :type key_name: str :param key_name: Name of the key :type table_name: str :param table_name: Name of the DynamoDB table """ updated_provisioning = int(current_provisioning) - int(units) min_provisioned_reads = get_min_provisioned_reads( current_provisioning, table_name, key_name) if updated_provisioning < min_provisioned_reads: logger.info( '{0} - Reached provisioned reads min limit: {1:d}'.format( table_name, min_provisioned_reads)) return min_provisioned_reads logger.debug( '{0} - Read provisioning will be decreased to {1:d} units'.format( table_name, updated_provisioning)) return updated_provisioning
def list_tables(): """ Return list of DynamoDB tables available from AWS :returns: list -- List of DynamoDB tables """ tables = [] try: tables = DYNAMODB_CONNECTION.list_tables() except DynamoDBResponseError as error: dynamodb_error = error.body['__type'].rsplit('#', 1)[1] if dynamodb_error == 'ResourceNotFoundException': logger.error('No tables found') elif dynamodb_error == 'AccessDeniedException': logger.debug( 'Your AWS API keys lack access to listing tables. ' 'That is an issue if you are trying to use regular expressions ' 'in your table configuration.') else: logger.error( ( 'Unhandled exception: {0}: {1}. ' 'Please file a bug report at ' 'https://github.com/sebdah/dynamic-dynamodb/issues' ).format( dynamodb_error, error.body['message'])) return tables
def increase_writes_in_units(table_name, current_provisioning, units, key_name): """ Increase the current_provisioning with units units :type table_name: str :param table_name: Name of the DynamoDB table :type current_provisioning: int :param current_provisioning: The current provisioning :type units: int :param units: How many units should we increase with :returns: int -- New provisioning value :type key_name: str :param key_name: Name of the key """ updated_provisioning = int(current_provisioning) + int(units) logger.debug( 'Write provisioning will be increased to {0:d} units'.format( updated_provisioning)) if get_table_option(key_name, 'max_provisioned_writes') > 0: if (updated_provisioning > get_table_option(key_name, 'max_provisioned_writes')): logger.info('Reached provisioned writes max limit: {0:d}'.format( int(get_table_option(key_name, 'max_provisioned_writes')))) return get_table_option(key_name, 'max_provisioned_writes') return updated_provisioning
def run(self, check_interval=1): """ Run the daemon :type check_interval: int :param check_interval: Delay in seconds between checks """ while True: used_keys = set() table_names = set() configured_tables = configuration['tables'].keys() # Add regexp table names for table_name in core.dynamodb.list_tables(): for key_name in configured_tables: if re.match(key_name, table_name): logger.debug( "Table {0} match with config key {1}".format( table_name, key_name)) table_names.add((table_name, key_name)) used_keys.add(key_name) # Remove used tables for table_name in used_keys: configured_tables.remove(table_name) # Ensure provisioning for table_name, key_name in sorted(table_names): core.ensure_provisioning(table_name, key_name) # Sleep between the checks time.sleep(check_interval)
def __get_min_writes(current_provisioning, min_provisioned_writes, log_tag): """ Get the minimum number of writes to current_provisioning :type current_provisioning: int :param current_provisioning: Current provisioned writes :type min_provisioned_writes: int :param min_provisioned_writes: Configured min provisioned writes :type log_tag: str :param log_tag: Prefix for the log :returns: int -- Minimum number of writes """ # Fallback value to ensure that we always have at least 1 read writes = 1 if min_provisioned_writes: writes = int(min_provisioned_writes) if writes > int(current_provisioning * 2): writes = int(current_provisioning * 2) logger.debug('{0} - ' 'Cannot reach min-provisioned-writes as max scale up ' 'is 100% of current provisioning'.format(log_tag)) logger.debug('{0} - Setting min provisioned writes to {1}'.format( log_tag, min_provisioned_writes)) return writes
def decrease_reads_in_units(current_provisioning, units, min_provisioned_reads, log_tag): """ Decrease the current_provisioning with units units :type current_provisioning: int :param current_provisioning: The current provisioning :type units: int :param units: How many units should we decrease with :returns: int -- New provisioning value :type min_provisioned_reads: int :param min_provisioned_reads: Configured min provisioned reads :type log_tag: str :param log_tag: Prefix for the log """ updated_provisioning = int(current_provisioning) - int(units) min_provisioned_reads = __get_min_reads(current_provisioning, min_provisioned_reads, log_tag) if updated_provisioning < min_provisioned_reads: logger.info('{0} - Reached provisioned reads min limit: {1:d}'.format( log_tag, min_provisioned_reads)) return min_provisioned_reads logger.debug( '{0} - Read provisioning will be decreased to {1:d} units'.format( log_tag, updated_provisioning)) return updated_provisioning
def get_tables_and_gsis(): """ Get a set of tables and gsis and their configuration keys :returns: set -- A set of tuples (table_name, table_conf_key) """ table_names = set() configured_tables = get_configured_tables() not_used_tables = set(configured_tables) # Add regexp table names for table_instance in list_tables(): for key_name in configured_tables: try: if re.match(key_name, table_instance.table_name): logger.debug("Table {0} match with config key {1}".format(table_instance.table_name, key_name)) table_names.add((table_instance.table_name, key_name)) not_used_tables.discard(key_name) else: logger.debug( "Table {0} did not match with config key {1}".format(table_instance.table_name, key_name) ) except re.error: logger.error('Invalid regular expression: "{0}"'.format(key_name)) sys.exit(1) if not_used_tables: logger.warning( "No tables matching the following configured " "tables found: {0}".format(", ".join(not_used_tables)) ) return sorted(table_names)
def increase_reads_in_percent(table_name, current_provisioning, percent, key_name): """ Increase the current_provisioning with percent % :type table_name: str :param table_name: Name of the DynamoDB table :type current_provisioning: int :param current_provisioning: The current provisioning :type percent: int :param percent: How many percent should we increase with :returns: int -- New provisioning value :type key_name: str :param key_name: Name of the key """ increase = int(float(current_provisioning) * (float(percent) / 100)) updated_provisioning = current_provisioning + increase logger.debug('Read provisioning will be increased to {0:d} units'.format( updated_provisioning)) if get_table_option(key_name, 'max_provisioned_reads') > 0: if (updated_provisioning > get_table_option(key_name, 'max_provisioned_reads')): logger.info('Reached provisioned reads max limit: {0:d}'.format( int(get_table_option(key_name, 'max_provisioned_reads')))) return get_table_option(key_name, 'max_provisioned_reads') return updated_provisioning
def decrease_writes_in_percent(current_provisioning, percent, min_provisioned_writes, log_tag): """ Decrease the current_provisioning with percent % :type current_provisioning: int :param current_provisioning: The current provisioning :type percent: int :param percent: How many percent should we decrease with :returns: int -- New provisioning value :type min_provisioned_writes: int :param min_provisioned_writes: Configured min provisioned writes :type log_tag: str :param log_tag: Prefix for the log """ decrease = int(float(current_provisioning) * (float(percent) / 100)) updated_provisioning = current_provisioning - decrease min_provisioned_writes = __get_min_writes(current_provisioning, min_provisioned_writes, log_tag) if updated_provisioning < min_provisioned_writes: logger.info('{0} - Reached provisioned writes min limit: {1:d}'.format( log_tag, min_provisioned_writes)) return min_provisioned_writes logger.debug( '{0} - Write provisioning will be decreased to {1:d} units'.format( log_tag, updated_provisioning)) return updated_provisioning
def decrease_reads_in_percent(table_name, current_provisioning, percent, key_name): """ Decrease the current_provisioning with percent % :type table_name: str :param table_name: Name of the DynamoDB table :type current_provisioning: int :param current_provisioning: The current provisioning :type percent: int :param percent: How many percent should we decrease with :returns: int -- New provisioning value :type key_name: str :param key_name: Name of the key """ decrease = int(float(current_provisioning) * (float(percent) / 100)) updated_provisioning = current_provisioning - decrease logger.debug('Read provisioning will be decreased to {0:d} units'.format( updated_provisioning)) min_provisioned_reads = get_min_provisioned_reads(table_name, current_provisioning, key_name) if min_provisioned_reads > 0: if updated_provisioning < min_provisioned_reads: logger.info('Reached provisioned reads min limit: {0:d}'.format( min_provisioned_reads)) return min_provisioned_reads return updated_provisioning
def increase_writes_in_units(table_name, current_provisioning, units, key_name): """ Increase the current_provisioning with units units :type table_name: str :param table_name: Name of the DynamoDB table :type current_provisioning: int :param current_provisioning: The current provisioning :type units: int :param units: How many units should we increase with :returns: int -- New provisioning value :type key_name: str :param key_name: Name of the key """ updated_provisioning = int(current_provisioning) + int(units) logger.debug('Write provisioning will be increased to {0:d} units'.format( updated_provisioning)) if get_table_option(key_name, 'max_provisioned_writes') > 0: if (updated_provisioning > get_table_option(key_name, 'max_provisioned_writes')): logger.info('Reached provisioned writes max limit: {0:d}'.format( int(get_table_option(key_name, 'max_provisioned_writes')))) return get_table_option(key_name, 'max_provisioned_writes') return updated_provisioning
def decrease_writes_in_units(table_name, current_provisioning, units, key_name): """ Decrease the current_provisioning with units units :type table_name: str :param table_name: Name of the DynamoDB table :type current_provisioning: int :param current_provisioning: The current provisioning :type units: int :param units: How many units should we decrease with :returns: int -- New provisioning value :type key_name: str :param key_name: Name of the key """ updated_provisioning = int(current_provisioning) - int(units) logger.debug('Write provisioning will be decreased to {0:d} units'.format( updated_provisioning)) min_provisioned_writes = get_min_provisioned_writes( table_name, current_provisioning, key_name) if min_provisioned_writes > 0: if updated_provisioning < min_provisioned_writes: logger.info('Reached provisioned writes min limit: {0:d}'.format( min_provisioned_writes)) return min_provisioned_writes return updated_provisioning
def increase_writes_in_units(current_provisioning, units, max_provisioned_writes, log_tag): """ Increase the current_provisioning with units units :type current_provisioning: int :param current_provisioning: The current provisioning :type units: int :param units: How many units should we increase with :returns: int -- New provisioning value :type max_provisioned_writes: int :param max_provisioned_writes: Configured max provisioned writes :type log_tag: str :param log_tag: Prefix for the log """ if int(units) > int(current_provisioning): updated_provisioning = 2 * int(current_provisioning) else: updated_provisioning = int(current_provisioning) + int(units) if max_provisioned_writes > 0: if updated_provisioning > max_provisioned_writes: logger.info( '{0} - Reached provisioned writes max limit: {1}'.format( log_tag, max_provisioned_writes)) return max_provisioned_writes logger.debug( '{0} - Write provisioning will be increased to {1:d} units'.format( log_tag, updated_provisioning)) return updated_provisioning
def __get_connection_dynamodb(retries=3): """ Ensure connection to DynamoDB :type retries: int :param retries: Number of times to retry to connect to DynamoDB """ connected = False while not connected: try: if (configuration['global']['aws_access_key_id'] and configuration['global']['aws_secret_access_key']): connection = dynamodb.connect_to_region( configuration['global']['region'], aws_access_key_id=configuration['global']['aws_access_key_id'], aws_secret_access_key=\ configuration['global']['aws_secret_access_key']) else: connection = dynamodb.connect_to_region( configuration['global']['region']) connected = True except Exception as err: logger.error('Failed to connect to DynamoDB: {0}'.format(err)) if retries == 0: logger.error( 'Please report an issue at: ' 'https://github.com/sebdah/dynamic-dynamodb/issues') raise else: logger.error('Retrying in 5 seconds') retries -= 1 time.sleep(5) logger.debug('Connected to DynamoDB') return connection
def main(): """ Main function called from dynamic-dynamodb """ try: if get_global_option('daemon'): daemon = DynamicDynamoDBDaemon( '{0}/dynamic-dynamodb.{1}.pid'.format( get_global_option('pid_file_dir'), get_global_option('instance'))) if get_global_option('daemon') == 'start': daemon.start() elif get_global_option('daemon') == 'stop': logger.debug('Stopping daemon') daemon.stop() logger.info('Daemon stopped') sys.exit(0) elif get_global_option('daemon') == 'restart': daemon.restart() elif get_global_option('daemon') in ['foreground', 'fg']: daemon.run() else: print('Valid options for --daemon are start, stop and restart') sys.exit(1) else: while True: execute() except Exception as error: logger.exception(error)
def main(): """ Main function called from dynamic-dynamodb """ try: if get_global_option("daemon"): daemon = DynamicDynamoDBDaemon( "{0}/dynamic-dynamodb.{1}.pid".format(get_global_option("pid_file_dir"), get_global_option("instance")) ) if get_global_option("daemon") == "start": daemon.start() elif get_global_option("daemon") == "stop": logger.debug("Stopping daemon") daemon.stop() logger.info("Daemon stopped") sys.exit(0) elif get_global_option("daemon") == "restart": daemon.restart() elif get_global_option("daemon") in ["foreground", "fg"]: daemon.run() else: print("Valid options for --daemon are start, stop and restart") sys.exit(1) else: while True: execute() except Exception as error: logger.exception(error)
def increase_writes_in_percent( current_provisioning, percent, max_provisioned_writes, log_tag): """ Increase the current_provisioning with percent % :type current_provisioning: int :param current_provisioning: The current provisioning :type percent: int :param percent: How many percent should we increase with :returns: int -- New provisioning value :type max_provisioned_writes: int :param max_provisioned_writes: Configured max provisioned writes :type log_tag: str :param log_tag: Prefix for the log """ increase = int(math.ceil(float(current_provisioning)*(float(percent)/100))) updated_provisioning = current_provisioning + increase if max_provisioned_writes > 0: if updated_provisioning > max_provisioned_writes: logger.info( '{0} - Reached provisioned writes max limit: {1}'.format( log_tag, max_provisioned_writes)) return max_provisioned_writes logger.debug( '{0} - Write provisioning will be increased to {1:d} units'.format( log_tag, updated_provisioning)) return updated_provisioning
def __get_min_writes(current_provisioning, min_provisioned_writes, log_tag): """ Get the minimum number of writes to current_provisioning :type current_provisioning: int :param current_provisioning: Current provisioned writes :type min_provisioned_writes: int :param min_provisioned_writes: Configured min provisioned writes :type log_tag: str :param log_tag: Prefix for the log :returns: int -- Minimum number of writes """ # Fallback value to ensure that we always have at least 1 read writes = 1 if min_provisioned_writes: writes = int(min_provisioned_writes) if writes > int(current_provisioning * 2): writes = int(current_provisioning * 2) logger.debug( '{0} - ' 'Cannot reach min-provisioned-writes as max scale up ' 'is 100% of current provisioning'.format(log_tag)) logger.debug( '{0} - Setting min provisioned writes to {1}'.format( log_tag, min_provisioned_writes)) return writes
def increase_writes_in_units( current_provisioning, units, max_provisioned_writes, log_tag): """ Increase the current_provisioning with units units :type current_provisioning: int :param current_provisioning: The current provisioning :type units: int :param units: How many units should we increase with :returns: int -- New provisioning value :type max_provisioned_writes: int :param max_provisioned_writes: Configured max provisioned writes :type log_tag: str :param log_tag: Prefix for the log """ if int(units) > int(current_provisioning): updated_provisioning = 2 * int(current_provisioning) else: updated_provisioning = int(current_provisioning) + int(units) if max_provisioned_writes > 0: if updated_provisioning > max_provisioned_writes: logger.info( '{0} - Reached provisioned writes max limit: {1}'.format( log_tag, max_provisioned_writes)) return max_provisioned_writes logger.debug( '{0} - Write provisioning will be increased to {1:d} units'.format( log_tag, updated_provisioning)) return updated_provisioning
def increase_reads_in_percent(table_name, current_provisioning, percent, key_name): """ Increase the current_provisioning with percent % :type table_name: str :param table_name: Name of the DynamoDB table :type current_provisioning: int :param current_provisioning: The current provisioning :type percent: int :param percent: How many percent should we increase with :returns: int -- New provisioning value :type key_name: str :param key_name: Name of the key """ increase = int(float(current_provisioning)*(float(percent)/100)) updated_provisioning = current_provisioning + increase logger.debug( 'Read provisioning will be increased to {0:d} units'.format( updated_provisioning)) if get_table_option(key_name, 'max_provisioned_reads') > 0: if (updated_provisioning > get_table_option(key_name, 'max_provisioned_reads')): logger.info('Reached provisioned reads max limit: {0:d}'.format( int(get_table_option(key_name, 'max_provisioned_reads')))) return get_table_option(key_name, 'max_provisioned_reads') return updated_provisioning
def decrease_writes_in_percent( current_provisioning, percent, key_name, table_name): """ Decrease the current_provisioning with percent % :type current_provisioning: int :param current_provisioning: The current provisioning :type percent: int :param percent: How many percent should we decrease with :returns: int -- New provisioning value :type key_name: str :param key_name: Name of the key :type table_name: str :param table_name: Name of the DynamoDB table """ decrease = int(float(current_provisioning)*(float(percent)/100)) updated_provisioning = current_provisioning - decrease min_provisioned_writes = get_min_provisioned_writes( current_provisioning, table_name, key_name) if updated_provisioning < min_provisioned_writes: logger.info( '{0} - Reached provisioned writes min limit: {1:d}'.format( table_name, min_provisioned_writes)) return min_provisioned_writes logger.debug( '{0} - Write provisioning will be decreased to {1:d} units'.format( table_name, updated_provisioning)) return updated_provisioning
def decrease_reads_in_percent(table_name, current_provisioning, percent, key_name): """ Decrease the current_provisioning with percent % :type table_name: str :param table_name: Name of the DynamoDB table :type current_provisioning: int :param current_provisioning: The current provisioning :type percent: int :param percent: How many percent should we decrease with :returns: int -- New provisioning value :type key_name: str :param key_name: Name of the key """ decrease = int(float(current_provisioning)*(float(percent)/100)) updated_provisioning = current_provisioning - decrease logger.debug( 'Read provisioning will be decreased to {0:d} units'.format( updated_provisioning)) min_provisioned_reads = get_min_provisioned_reads( table_name, current_provisioning, key_name) if min_provisioned_reads > 0: if updated_provisioning < min_provisioned_reads: logger.info('Reached provisioned reads min limit: {0:d}'.format( min_provisioned_reads)) return min_provisioned_reads return updated_provisioning
def get_tables_and_gsis(): """ Get a set of tables and gsis and their configuration keys :returns: set -- A set of tuples (table_name, table_conf_key) """ table_names = set() configured_tables = get_configured_tables() not_used_tables = set(configured_tables) # Add regexp table names for table_instance in list_tables(): for key_name in configured_tables: try: if re.match(key_name, table_instance.table_name): logger.debug("Table {0} match with config key {1}".format( table_instance.table_name, key_name)) table_names.add((table_instance.table_name, key_name)) not_used_tables.discard(key_name) else: logger.debug( "Table {0} did not match with config key {1}".format( table_instance.table_name, key_name)) except re.error: logger.error( 'Invalid regular expression: "{0}"'.format(key_name)) sys.exit(1) if not_used_tables: logger.warning('No tables matching the following configured ' 'tables found: {0}'.format(', '.join(not_used_tables))) return sorted(table_names)
def decrease_writes_in_units(table_name, current_provisioning, units, key_name): """ Decrease the current_provisioning with units units :type table_name: str :param table_name: Name of the DynamoDB table :type current_provisioning: int :param current_provisioning: The current provisioning :type units: int :param units: How many units should we decrease with :returns: int -- New provisioning value :type key_name: str :param key_name: Name of the key """ updated_provisioning = int(current_provisioning) - int(units) logger.debug( 'Write provisioning will be decreased to {0:d} units'.format( updated_provisioning)) min_provisioned_writes = get_min_provisioned_writes( table_name, current_provisioning, key_name) if min_provisioned_writes > 0: if updated_provisioning < min_provisioned_writes: logger.info('Reached provisioned writes min limit: {0:d}'.format( min_provisioned_writes)) return min_provisioned_writes return updated_provisioning
def increase_writes_in_percent(current_provisioning, percent, max_provisioned_writes, log_tag): """ Increase the current_provisioning with percent % :type current_provisioning: int :param current_provisioning: The current provisioning :type percent: int :param percent: How many percent should we increase with :returns: int -- New provisioning value :type max_provisioned_writes: int :param max_provisioned_writes: Configured max provisioned writes :type log_tag: str :param log_tag: Prefix for the log """ increase = int( math.ceil(float(current_provisioning) * (float(percent) / 100))) updated_provisioning = current_provisioning + increase if max_provisioned_writes > 0: if updated_provisioning > max_provisioned_writes: logger.info( '{0} - Reached provisioned writes max limit: {1}'.format( log_tag, max_provisioned_writes)) return max_provisioned_writes logger.debug( '{0} - Write provisioning will be increased to {1:d} units'.format( log_tag, updated_provisioning)) return updated_provisioning
def decrease_reads_in_units( current_provisioning, units, min_provisioned_reads, log_tag): """ Decrease the current_provisioning with units units :type current_provisioning: int :param current_provisioning: The current provisioning :type units: int :param units: How many units should we decrease with :returns: int -- New provisioning value :type min_provisioned_reads: int :param min_provisioned_reads: Configured min provisioned reads :type log_tag: str :param log_tag: Prefix for the log """ updated_provisioning = int(current_provisioning) - int(units) min_provisioned_reads = __get_min_reads( current_provisioning, min_provisioned_reads, log_tag) if updated_provisioning < min_provisioned_reads: logger.info( '{0} - Reached provisioned reads min limit: {1:d}'.format( log_tag, int(min_provisioned_reads))) return min_provisioned_reads logger.debug( '{0} - Read provisioning will be decreased to {1:d} units'.format( log_tag, int(updated_provisioning))) return updated_provisioning
def list_tables(): """ Return list of DynamoDB tables available from AWS :returns: list -- List of DynamoDB tables """ tables = [] try: tables = DYNAMODB_CONNECTION.list_tables() except DynamoDBResponseError as error: dynamodb_error = error.body['__type'].rsplit('#', 1)[1] if dynamodb_error == 'ResourceNotFoundException': logger.error('No tables found') elif dynamodb_error == 'AccessDeniedException': logger.debug( 'Your AWS API keys lack access to listing tables. ' 'That is an issue if you are trying to use regular expressions ' 'in your table configuration.') else: logger.error( ('Unhandled exception: {0}: {1}. ' 'Please file a bug report at ' 'https://github.com/sebdah/dynamic-dynamodb/issues').format( dynamodb_error, error.body['message'])) return tables
def decrease_reads_in_percent( current_provisioning, percent, min_provisioned_reads, log_tag): """ Decrease the current_provisioning with percent % :type current_provisioning: int :param current_provisioning: The current provisioning :type percent: int :param percent: How many percent should we decrease with :type min_provisioned_reads: int :param min_provisioned_reads: Configured min provisioned reads :type log_tag: str :param log_tag: Prefix for the log :returns: int -- New provisioning value """ percent = float(percent) decrease = int(float(current_provisioning)*(float(percent)/100)) updated_provisioning = current_provisioning - decrease min_provisioned_reads = __get_min_reads( current_provisioning, min_provisioned_reads, log_tag) if updated_provisioning < min_provisioned_reads: logger.info( '{0} - Reached provisioned reads min limit: {1:d}'.format( log_tag, int(min_provisioned_reads))) return min_provisioned_reads logger.debug( '{0} - Read provisioning will be decreased to {1:d} units'.format( log_tag, int(updated_provisioning))) return updated_provisioning
def execute_table_in_loop(table_name, table_key, table_logger): if not get_global_option('daemon') and get_global_option('run_once'): execute_table(table_name, table_key) else: t = threading.currentThread() while getattr(t, "do_run", True): execute_table(table_name, table_key, table_logger) logger.debug('Sleeping {0} seconds until next check'.format( get_global_option('check_interval'))) time.sleep(get_global_option('check_interval'))
def get_provisioned_write_units(table_name): """ Returns the number of provisioned write units for the table :type table_name: str :param table_name: Name of the DynamoDB table :returns: int -- Number of write units """ table = dynamodb.get_table(table_name) logger.debug('{0} - Provisioned write units: {1:d}'.format( table_name, table.write_units)) return int(table.write_units)
def __update_throughput(table_name, table_key, gsi_name, gsi_key, read_units, write_units): """ Update throughput on the GSI :type table_name: str :param table_name: Name of the DynamoDB table :type table_key: str :param table_key: Table configuration option key name :type gsi_name: str :param gsi_name: Name of the GSI :type gsi_key: str :param gsi_key: Configuration option key name :type read_units: int :param read_units: New read unit provisioning :type write_units: int :param write_units: New write unit provisioning """ try: current_ru = dynamodb.get_provisioned_gsi_read_units( table_name, gsi_name) current_wu = dynamodb.get_provisioned_gsi_write_units( table_name, gsi_name) except JSONResponseError: raise # Check table status try: gsi_status = dynamodb.get_gsi_status(table_name, gsi_name) except JSONResponseError: raise logger.debug('{0} - GSI: {1} - GSI status is {2}'.format( table_name, gsi_name, gsi_status)) if gsi_status != 'ACTIVE': logger.warning( '{0} - GSI: {1} - Not performing throughput changes when GSI ' 'status is {2}'.format(table_name, gsi_name, gsi_status)) return # If this setting is True, we will only scale down when # BOTH reads AND writes are low if get_gsi_option(table_key, gsi_key, 'always_decrease_rw_together'): read_units, write_units = __calculate_always_decrease_rw_values( table_name, gsi_name, read_units, current_ru, write_units, current_wu) if read_units == current_ru and write_units == current_wu: logger.info('{0} - GSI: {1} - No changes to perform'.format( table_name, gsi_name)) return dynamodb.update_gsi_provisioning(table_name, table_key, gsi_name, gsi_key, int(read_units), int(write_units))
def execute(): """ Ensure provisioning """ boto_server_error_retries = 3 # Ensure provisioning for table_name, table_key in sorted(dynamodb.get_tables_and_gsis()): logger.debug("Table: " + table_name) execute_in_thread(table_name, table_key) # Sleep between the checks if not get_global_option('run_once'): logger.debug('Sleeping {0} seconds until next thread check'.format(60)) time.sleep(60)
def __get_connection_dynamodb(retries=3): """ Ensure connection to DynamoDB :type retries: int :param retries: Number of times to retry to connect to DynamoDB """ connected = False while not connected: logger.debug('Connecting to DynamoDB in {0}'.format( get_global_option('region'))) if (get_global_option('aws_access_key_id') and get_global_option('aws_secret_access_key')): logger.debug( 'Authenticating to DynamoDB using ' 'credentials in configuration file') connection = dynamodb2.connect_to_region( get_global_option('region'), aws_access_key_id=get_global_option('aws_access_key_id'), aws_secret_access_key=get_global_option( 'aws_secret_access_key')) else: try: logger.debug( 'Authenticating to DynamoDB using EC2 instance profile') metadata = get_instance_metadata(timeout=1, num_retries=1) connection = dynamodb2.connect_to_region( metadata['placement']['availability-zone'][:-1], profile_name=metadata['iam']['info'][u'InstanceProfileArn']) except KeyError: logger.debug( 'Authenticating to DynamoDB using ' 'env vars / boto configuration') connection = dynamodb2.connect_to_region( get_global_option('region')) if not connection: if retries == 0: logger.error('Failed to connect to DynamoDB. Giving up.') raise else: logger.error( 'Failed to connect to DynamoDB. Retrying in 5 seconds') retries -= 1 time.sleep(5) else: connected = True logger.debug('Connected to DynamoDB in {0}'.format( get_global_option('region'))) return connection
def __get_connection_SNS(): """ Ensure connection to SNS """ try: if get_global_option("aws_access_key_id") and get_global_option("aws_secret_access_key"): logger.debug("Authenticating to SNS using " "credentials in configuration file") connection = sns.connect_to_region( get_global_option("region"), aws_access_key_id=get_global_option("aws_access_key_id"), aws_secret_access_key=get_global_option("aws_secret_access_key"), ) else: try: logger.debug("Authenticating to SNS using EC2 instance profile") metadata = get_instance_metadata(timeout=1, num_retries=1) connection = sns.connect_to_region( metadata["placement"]["availability-zone"][:-1], profile_name=metadata["iam"]["info"][u"InstanceProfileArn"], ) except KeyError: logger.debug("Authenticating to SNS using " "env vars / boto configuration") connection = sns.connect_to_region(get_global_option("region")) except Exception as err: logger.error("Failed connecting to SNS: {0}".format(err)) logger.error("Please report an issue at: " "https://github.com/sebdah/dynamic-dynamodb/issues") raise logger.debug("Connected to SNS in {0}".format(get_global_option("region"))) return connection
def increase_reads_in_units( current_provisioning, units, table_name, table_key, gsi_name, gsi_key): """ Increase the current_provisioning with units units :type current_provisioning: int :param current_provisioning: The current provisioning :type units: int :param units: How many units should we increase with :type table_name: str :param table_name: Name of the DynamoDB table :type table_key: str :param table_key: Table configuration option key name :type gsi_name: str :param gsi_name: Name of the GSI :type gsi_key: str :param gsi_key: Name of the key :returns: int -- New provisioning value """ updated_provisioning = 0 if int(units) > int(current_provisioning): updated_provisioning = 2 * int(current_provisioning) else: updated_provisioning = int(current_provisioning) + int(units) if get_gsi_option(table_key, gsi_key, 'max_provisioned_reads'): max_provisioned_reads = int(get_gsi_option( table_key, gsi_key, 'max_provisioned_reads')) else: max_provisioned_reads = 0 if (max_provisioned_reads > 0 and updated_provisioning > max_provisioned_reads): logger.info( '{0} - GSI: {1} - ' 'Reached provisioned reads max limit: {2:d}'.format( table_name, gsi_name, max_provisioned_reads)) return max_provisioned_reads logger.debug( '{0} - GSI: {1} - ' 'Read provisioning will be increased to {2:d} units'.format( table_name, gsi_name, updated_provisioning)) return updated_provisioning
def get_provisioned_table_write_units(table_name): """ Returns the number of provisioned write units for the table :type table_name: str :param table_name: Name of the DynamoDB table :returns: int -- Number of write units """ desc = DYNAMODB_CONNECTION.describe_table(table_name) write_units = int( desc[u'Table'][u'ProvisionedThroughput'][u'WriteCapacityUnits']) logger.debug('{0} - Currently provisioned write units: {1:d}'.format( table_name, write_units)) return write_units
def __get_connection_dynamodb(retries=3): """ Ensure connection to DynamoDB :type retries: int :param retries: Number of times to retry to connect to DynamoDB """ connected = False while not connected: logger.debug('Connecting to DynamoDB in {0}'.format( get_global_option('region'))) if (get_global_option('aws_access_key_id') and get_global_option('aws_secret_access_key')): logger.debug('Authenticating to DynamoDB using ' 'credentials in configuration file') connection = dynamodb2.connect_to_region( get_global_option('region'), aws_access_key_id=get_global_option('aws_access_key_id'), aws_secret_access_key=get_global_option( 'aws_secret_access_key')) else: try: logger.debug( 'Authenticating to DynamoDB using EC2 instance profile') metadata = get_instance_metadata(timeout=1, num_retries=1) connection = dynamodb2.connect_to_region( metadata['placement']['availability-zone'][:-1], profile_name=metadata['iam']['info'] [u'InstanceProfileArn']) except KeyError: logger.debug('Authenticating to DynamoDB using ' 'env vars / boto configuration') connection = dynamodb2.connect_to_region( get_global_option('region')) if not connection: if retries == 0: logger.error('Failed to connect to DynamoDB. Giving up.') raise else: logger.error( 'Failed to connect to DynamoDB. Retrying in 5 seconds') retries -= 1 time.sleep(5) else: connected = True logger.debug('Connected to DynamoDB in {0}'.format( get_global_option('region'))) return connection
def get_provisioned_read_units(table_name): """ Returns the number of provisioned read units for the table :type table_name: str :param table_name: Name of the DynamoDB table :returns: int -- Number of read units """ desc = dynamodb.describe_table(table_name) read_units = int( desc[u'Table'][u'ProvisionedThroughput'][u'ReadCapacityUnits']) logger.debug('{0} - Currently provisioned read units: {1:d}'.format( table_name, read_units)) return read_units
def __update_throughput(table_name, key_name, read_units, write_units): """ Update throughput on the DynamoDB table :type table_name: str :param table_name: Name of the DynamoDB table :type key_name: str :param key_name: Configuration option key name :type read_units: int :param read_units: New read unit provisioning :type write_units: int :param write_units: New write unit provisioning """ try: current_ru = dynamodb.get_provisioned_table_read_units(table_name) current_wu = dynamodb.get_provisioned_table_write_units(table_name) except JSONResponseError: raise # Check table status try: table_status = dynamodb.get_table_status(table_name) except JSONResponseError: raise logger.debug('{0} - Table status is {1}'.format(table_name, table_status)) if table_status != 'ACTIVE': logger.warning( '{0} - Not performing throughput changes when table ' 'is {1}'.format(table_name, table_status)) return # If this setting is True, we will only scale down when # BOTH reads AND writes are low if get_table_option(key_name, 'always_decrease_rw_together'): read_units, write_units = __calculate_always_decrease_rw_values( table_name, read_units, current_ru, write_units, current_wu) if read_units == current_ru and write_units == current_wu: logger.info('{0} - No changes to perform'.format(table_name)) return dynamodb.update_table_provisioning( table_name, key_name, int(read_units), int(write_units))
def execute_in_thread(table_name, table_key): if table_name in table_threads: thread = table_threads.get(table_name) if thread.isAlive(): logger.debug("Thread {} still running".format(table_name)) else: logger.error("Thread {} was stopped!".format(table_name)) sys.exit(1) else: table_logger = log_handler.getTableLogger(table_name) logger.info("Start new thread: " + table_name) thread = threading.Thread(target=execute_table_in_loop, args=[table_name, table_key, table_logger]) table_threads[table_name] = thread thread.start()
def increase_writes_in_percent( current_provisioning, percent, table_name, table_key, gsi_name, gsi_key): """ Increase the current_provisioning with percent % :type current_provisioning: int :param current_provisioning: The current provisioning :type percent: int :param percent: How many percent should we increase with :type table_name: str :param table_name: Name of the DynamoDB table :type table_key: str :param table_key: Table configuration option key name :type gsi_name: str :param gsi_name: Name of the GSI :type gsi_key: str :param gsi_key: Name of the key :returns: int -- New provisioning value """ increase = int(math.ceil(float(current_provisioning)*(float(percent)/100))) updated_provisioning = current_provisioning + increase if get_gsi_option(table_key, gsi_key, 'max_provisioned_writes'): max_provisioned_writes = int(get_gsi_option( table_key, gsi_key, 'max_provisioned_writes')) else: max_provisioned_writes = 0 if (max_provisioned_writes > 0 and updated_provisioning > max_provisioned_writes): logger.info( '{0} - GSI: {1} - ' 'Reached provisioned writes max limit: {2:d}'.format( table_name, gsi_name, max_provisioned_writes)) return max_provisioned_writes logger.debug( '{0} - GSI: {1}' 'Write provisioning will be increased to {2:d} units'.format( table_name, gsi_name, updated_provisioning)) return updated_provisioning
def __update_throughput(table_name, table_key, gsi_name, gsi_key, read_units, write_units): """ Update throughput on the GSI :type table_name: str :param table_name: Name of the DynamoDB table :type table_key: str :param table_key: Table configuration option key name :type gsi_name: str :param gsi_name: Name of the GSI :type gsi_key: str :param gsi_key: Configuration option key name :type read_units: int :param read_units: New read unit provisioning :type write_units: int :param write_units: New write unit provisioning """ try: current_ru = dynamodb.get_provisioned_gsi_read_units(table_name, gsi_name) current_wu = dynamodb.get_provisioned_gsi_write_units(table_name, gsi_name) except JSONResponseError: raise # Check table status try: gsi_status = dynamodb.get_gsi_status(table_name, gsi_name) except JSONResponseError: raise logger.debug("{0} - GSI: {1} - GSI status is {2}".format(table_name, gsi_name, gsi_status)) if gsi_status != "ACTIVE": logger.warning( "{0} - GSI: {1} - Not performing throughput changes when GSI " "status is {2}".format(table_name, gsi_name, gsi_status) ) return # If this setting is True, we will only scale down when # BOTH reads AND writes are low if get_gsi_option(table_key, gsi_key, "always_decrease_rw_together"): read_units, write_units = __calculate_always_decrease_rw_values( table_name, gsi_name, read_units, current_ru, write_units, current_wu ) if read_units == current_ru and write_units == current_wu: logger.info("{0} - GSI: {1} - No changes to perform".format(table_name, gsi_name)) return dynamodb.update_gsi_provisioning(table_name, table_key, gsi_name, gsi_key, int(read_units), int(write_units))
def main(): """ Main function called from dynamic-dynamodb """ try: if get_global_option('show_config'): print json.dumps(config.get_configuration(), indent=2) elif get_global_option('daemon'): daemon = DynamicDynamoDBDaemon( '{0}/dynamic-dynamodb.{1}.pid'.format( get_global_option('pid_file_dir'), get_global_option('instance'))) if get_global_option('daemon') == 'start': logger.debug('Starting daemon') try: daemon.start() logger.info('Daemon started') except IOError as error: logger.error( 'Could not create pid file: {0}'.format(error)) logger.error('Daemon not started') elif get_global_option('daemon') == 'stop': logger.debug('Stopping daemon') daemon.stop() logger.info('Daemon stopped') sys.exit(0) elif get_global_option('daemon') == 'restart': logger.debug('Restarting daemon') daemon.restart() logger.info('Daemon restarted') elif get_global_option('daemon') in ['foreground', 'fg']: logger.debug('Starting daemon in foreground') daemon.run() logger.info('Daemon started in foreground') else: print( 'Valid options for --daemon are start, ' 'stop, restart, and foreground') sys.exit(1) else: if get_global_option('run_once'): execute() else: while True: execute() except Exception as error: logger.exception(error) except KeyboardInterrupt: while threading.active_count() > 1: for thread in table_threads.values(): thread.do_run = False logger.info('Waiting for all threads... {}s'.format( get_global_option('check_interval'))) time.sleep(get_global_option('check_interval')) raise
def get_provisioned_table_write_units(table_name): """ Returns the number of provisioned write units for the table :type table_name: str :param table_name: Name of the DynamoDB table :returns: int -- Number of write units """ try: desc = DYNAMODB_CONNECTION.describe_table(table_name) except JSONResponseError: raise write_units = int(desc[u"Table"][u"ProvisionedThroughput"][u"WriteCapacityUnits"]) logger.debug("{0} - Currently provisioned write units: {1:d}".format(table_name, write_units)) return write_units
def get_provisioned_write_units(table_name): """ Returns the number of provisioned write units for the table :type table_name: str :param table_name: Name of the DynamoDB table :returns: int -- Number of write units """ try: table = dynamodb.get_table(table_name) except DynamoDBResponseError: # Return if the table does not exist return None logger.debug('{0} - Provisioned write units: {1:d}'.format( table_name, table.write_units)) return int(table.write_units)
def list_tables(): """ Return list of DynamoDB tables available from AWS :returns: list -- List of DynamoDB tables """ tables = [] try: table_list = DYNAMODB_CONNECTION.list_tables() while True: for table_name in table_list[u'TableNames']: tables.append(get_table(table_name)) if u'LastEvaluatedTableName' in table_list: table_list = DYNAMODB_CONNECTION.list_tables( table_list[u'LastEvaluatedTableName']) else: break except DynamoDBResponseError as error: dynamodb_error = error.body['__type'].rsplit('#', 1)[1] if dynamodb_error == 'ResourceNotFoundException': logger.error('No tables found') elif dynamodb_error == 'AccessDeniedException': logger.debug( 'Your AWS API keys lack access to listing tables. ' 'That is an issue if you are trying to use regular ' 'expressions in your table configuration.') elif dynamodb_error == 'UnrecognizedClientException': logger.error( 'Invalid security token. Are your AWS API keys correct?') else: logger.error( ( 'Unhandled exception: {0}: {1}. ' 'Please file a bug report at ' 'https://github.com/sebdah/dynamic-dynamodb/issues' ).format( dynamodb_error, error.body['message'])) except JSONResponseError as error: logger.error('Communication error: {0}'.format(error)) sys.exit(1) return tables
def increase_reads_in_percent(current_provisioning, percent, max_provisioned_reads, consumed_read_units_percent, log_tag): """ Increase the current_provisioning with percent % :type current_provisioning: int :param current_provisioning: The current provisioning :type percent: int :param percent: How many percent should we increase with :type max_provisioned_reads: int :param max_provisioned_reads: Configured max provisioned reads :returns: int -- New provisioning value :type consumed_read_units_percent: float :param consumed_read_units_percent: Number of consumed read units :type log_tag: str :param log_tag: Prefix for the log """ current_provisioning = float(current_provisioning) consumed_read_units_percent = float(consumed_read_units_percent) percent = float(percent) consumption_based_current_provisioning = \ float(math.ceil(current_provisioning*(consumed_read_units_percent/100))) if consumption_based_current_provisioning > current_provisioning: increase = int( math.ceil(consumption_based_current_provisioning * (percent / 100))) updated_provisioning = consumption_based_current_provisioning + increase else: increase = int(math.ceil(current_provisioning * (percent / 100))) updated_provisioning = current_provisioning + increase if max_provisioned_reads > 0: if updated_provisioning > max_provisioned_reads: logger.info( '{0} - Reached provisioned reads max limit: {1}'.format( log_tag, max_provisioned_reads)) return max_provisioned_reads logger.debug( '{0} - Read provisioning will be increased to {1} units'.format( log_tag, updated_provisioning)) return updated_provisioning
def get_provisioned_table_write_units(table_name): """ Returns the number of provisioned write units for the table :type table_name: str :param table_name: Name of the DynamoDB table :returns: int -- Number of write units """ try: desc = DYNAMODB_CONNECTION.describe_table(table_name) except JSONResponseError: raise write_units = int( desc[u'Table'][u'ProvisionedThroughput'][u'WriteCapacityUnits']) logger.debug('{0} - Currently provisioned write units: {1:d}'.format( table_name, write_units)) return write_units
def increase_writes_in_units(current_provisioning, units, max_provisioned_writes, consumed_write_units_percent, log_tag): """ Increase the current_provisioning with units units :type current_provisioning: int :param current_provisioning: The current provisioning :type units: int :param units: How many units should we increase with :returns: int -- New provisioning value :type max_provisioned_writes: int :param max_provisioned_writes: Configured max provisioned writes :type consumed_write_units_percent: float :param consumed_write_units_percent: Number of consumed write units :type log_tag: str :param log_tag: Prefix for the log """ units = int(units) current_provisioning = float(current_provisioning) consumed_write_units_percent = float(consumed_write_units_percent) consumption_based_current_provisioning = \ int(math.ceil(current_provisioning*(consumed_write_units_percent/100))) if consumption_based_current_provisioning > current_provisioning: updated_provisioning = consumption_based_current_provisioning + units else: updated_provisioning = int(current_provisioning) + units if max_provisioned_writes > 0: if updated_provisioning > max_provisioned_writes: logger.info( '{0} - Reached provisioned writes max limit: {1}'.format( log_tag, max_provisioned_writes)) return max_provisioned_writes logger.debug( '{0} - Write provisioning will be increased to {1:d} units'.format( log_tag, int(updated_provisioning))) return updated_provisioning
def __get_connection_cloudwatch(): """ Ensure connection to CloudWatch """ region = get_global_option('region') try: if (get_global_option('aws_access_key_id') and get_global_option('aws_secret_access_key')): logger.debug('Authenticating to CloudWatch using ' 'credentials in configuration file') connection = cloudwatch.connect_to_region( region, aws_access_key_id=get_global_option('aws_access_key_id'), aws_secret_access_key=get_global_option( 'aws_secret_access_key')) else: logger.debug('Authenticating using boto\'s authentication handler') connection = cloudwatch.connect_to_region(region) except Exception as err: logger.error('Failed connecting to CloudWatch: {0}'.format(err)) logger.error('Please report an issue at: ' 'https://github.com/sebdah/dynamic-dynamodb/issues') raise logger.debug('Connected to CloudWatch in {0}'.format(region)) return connection
def main(): """ Main function called from dynamic-dynamodb """ try: if get_global_option('show_config'): print json.dumps(config.get_configuration(), indent=2) elif get_global_option('daemon'): daemon = DynamicDynamoDBDaemon( '{0}/dynamic-dynamodb.{1}.pid'.format( get_global_option('pid_file_dir'), get_global_option('instance'))) if get_global_option('daemon') == 'start': logger.debug('Starting daemon') try: daemon.start() logger.info('Daemon started') except IOError as error: logger.error( 'Could not create pid file: {0}'.format(error)) logger.error('Daemon not started') elif get_global_option('daemon') == 'stop': logger.debug('Stopping daemon') daemon.stop() logger.info('Daemon stopped') sys.exit(0) elif get_global_option('daemon') == 'restart': logger.debug('Restarting daemon') daemon.restart() logger.info('Daemon restarted') elif get_global_option('daemon') in ['foreground', 'fg']: logger.debug('Starting daemon in foreground') daemon.run() logger.info('Daemon started in foreground') else: print( 'Valid options for --daemon are start, ' 'stop, restart, and foreground') sys.exit(1) else: if get_global_option('run_once'): execute() else: while True: execute() except Exception as error: logger.exception(error)
def __get_connection_cloudwatch(): """ Ensure connection to CloudWatch """ try: if (configuration['global']['aws_access_key_id'] and configuration['global']['aws_secret_access_key']): connection = cloudwatch.connect_to_region( configuration['global']['region'], aws_access_key_id=configuration['global']['aws_access_key_id'], aws_secret_access_key=\ configuration['global']['aws_secret_access_key']) else: connection = cloudwatch.connect_to_region( configuration['global']['region']) except Exception as err: logger.error('Failed connecting to CloudWatch: {0}'.format(err)) logger.error('Please report an issue at: ' 'https://github.com/sebdah/dynamic-dynamodb/issues') raise logger.debug('Connected to CloudWatch') return connection
def get_provisioned_gsi_read_units(table_name, gsi_name): """ Returns the number of provisioned read units for the table :type table_name: str :param table_name: Name of the DynamoDB table :type gsi_name: str :param gsi_name: Name of the GSI :returns: int -- Number of read units """ try: desc = DYNAMODB_CONNECTION.describe_table(table_name) except JSONResponseError: raise for gsi in desc['Table']['GlobalSecondaryIndexes']: if gsi['IndexName'] == gsi_name: read_units = int(gsi['ProvisionedThroughput']['ReadCapacityUnits']) break logger.debug( '{0} - GSI: {1} - Currently provisioned read units: {2:d}'.format( table_name, gsi_name, read_units)) return read_units
def __get_connection_SNS(): """ Ensure connection to SNS """ try: if (get_global_option('aws_access_key_id') and get_global_option('aws_secret_access_key')): logger.debug( 'Authenticating to SNS using ' 'credentials in configuration file') connection = sns.connect_to_region( get_global_option('region'), aws_access_key_id=get_global_option( 'aws_access_key_id'), aws_secret_access_key=get_global_option( 'aws_secret_access_key')) else: try: logger.debug( 'Authenticating to SNS using EC2 instance profile') metadata = get_instance_metadata(timeout=1, num_retries=1) connection = sns.connect_to_region( metadata['placement']['availability-zone'][:-1], profile_name=metadata['iam']['info'][u'InstanceProfileArn']) except KeyError: logger.debug( 'Authenticating to SNS using ' 'env vars / boto configuration') connection = sns.connect_to_region(get_global_option('region')) except Exception as err: logger.error('Failed connecting to SNS: {0}'.format(err)) logger.error( 'Please report an issue at: ' 'https://github.com/sebdah/dynamic-dynamodb/issues') raise logger.debug('Connected to SNS in {0}'.format(get_global_option('region'))) return connection
def __get_connection_dynamodb(retries=3): """ Ensure connection to DynamoDB :type retries: int :param retries: Number of times to retry to connect to DynamoDB """ connected = False region = get_global_option('region') while not connected: if (get_global_option('aws_access_key_id') and get_global_option('aws_secret_access_key')): logger.debug( 'Authenticating to DynamoDB using ' 'credentials in configuration file') connection = dynamodb2.connect_to_region( region, aws_access_key_id=get_global_option('aws_access_key_id'), aws_secret_access_key=get_global_option( 'aws_secret_access_key')) else: logger.debug( 'Authenticating using boto\'s authentication handler') connection = dynamodb2.connect_to_region(region) if not connection: if retries == 0: logger.error('Failed to connect to DynamoDB. Giving up.') raise else: logger.error( 'Failed to connect to DynamoDB. Retrying in 5 seconds') retries -= 1 time.sleep(5) else: connected = True logger.debug('Connected to DynamoDB in {0}'.format(region)) return connection