def run(self, check_interval=1): """ Run the daemon :type check_interval: int :param check_interval: Delay in seconds between checks """ while True: # Ensure provisioning for table_name, table_key in sorted(self.tables): table.ensure_provisioning(table_name, table_key) gsi_names = set() # Add regexp table names for gst_instance in dynamodb.table_gsis(table_name): gsi_name = gst_instance[u'IndexName'] for gsi_key in config['tables'][table_key]['gsis'].keys(): if re.match(gsi_key, gsi_name): logger.debug( 'Table {0} GSI {1} match with ' 'GSI config key {2}'.format( table_name, gsi_name, gsi_key)) gsi_names.add( ( gsi_name, gsi_key )) gsi_names = sorted(gsi_names) for gsi_name, gsi_key in gsi_names: gsi.ensure_provisioning( table_name, table_key, gsi_name, gsi_key) # Sleep between the checks logger.debug('Sleeping {0} seconds until next check'.format( check_interval)) time.sleep(check_interval)
def execute(): """ Ensure provisioning """ boto_server_error_retries = 3 # Ensure provisioning for table_name, table_key in sorted(dynamodb.get_tables_and_gsis()): try: table_num_consec_read_checks = \ CHECK_STATUS['tables'][table_name]['reads'] except KeyError: table_num_consec_read_checks = 0 try: table_num_consec_write_checks = \ CHECK_STATUS['tables'][table_name]['writes'] except KeyError: table_num_consec_write_checks = 0 try: # The return var shows how many times the scale-down criteria # has been met. This is coupled with a var in config, # "num_intervals_scale_down", to delay the scale-down table_num_consec_read_checks, table_num_consec_write_checks = \ table.ensure_provisioning( table_name, table_key, table_num_consec_read_checks, table_num_consec_write_checks) CHECK_STATUS['tables'][table_name] = { 'reads': table_num_consec_read_checks, 'writes': table_num_consec_write_checks } gsi_names = set() # Add regexp table names for gst_instance in dynamodb.table_gsis(table_name): gsi_name = gst_instance[u'IndexName'] try: gsi_keys = get_table_option(table_key, 'gsis').keys() except AttributeError: # Continue if there are not GSIs configured continue for gsi_key in gsi_keys: try: if re.match(gsi_key, gsi_name): logger.debug( 'Table {0} GSI {1} matches ' 'GSI config key {2}'.format( table_name, gsi_name, gsi_key)) gsi_names.add((gsi_name, gsi_key)) except re.error: logger.error('Invalid regular expression: "{0}"'.format( gsi_key)) sys.exit(1) for gsi_name, gsi_key in sorted(gsi_names): try: gsi_num_consec_read_checks = \ CHECK_STATUS['tables'][table_name]['reads'] except KeyError: gsi_num_consec_read_checks = 0 try: gsi_num_consec_write_checks = \ CHECK_STATUS['tables'][table_name]['writes'] except KeyError: gsi_num_consec_write_checks = 0 gsi_num_consec_read_checks, gsi_num_consec_write_checks = \ gsi.ensure_provisioning( table_name, table_key, gsi_name, gsi_key, gsi_num_consec_read_checks, gsi_num_consec_write_checks) CHECK_STATUS['gsis'][gsi_name] = { 'reads': gsi_num_consec_read_checks, 'writes': gsi_num_consec_write_checks } except JSONResponseError as error: exception = error.body['__type'].split('#')[1] if exception == 'ResourceNotFoundException': logger.error('{0} - Table {1} does not exist anymore'.format( table_name, table_name)) continue except BotoServerError as error: if boto_server_error_retries > 0: logger.error( 'Unknown boto error. Status: "{0}". Reason: "{1}"'.format( error.status, error.reason)) logger.error( 'Please bug report if this error persists') boto_server_error_retries -= 1 continue else: raise # Sleep between the checks if not get_global_option('run_once'): logger.debug('Sleeping {0} seconds until next check'.format( get_global_option('check_interval'))) time.sleep(get_global_option('check_interval'))
def execute(): """ Ensure provisioning """ boto_server_error_retries = 3 # Ensure provisioning for table_name, table_key in sorted(dynamodb.get_tables_and_gsis()): try: table_num_consec_read_checks = \ CHECK_STATUS['tables'][table_name]['reads'] except KeyError: table_num_consec_read_checks = 0 try: table_num_consec_write_checks = \ CHECK_STATUS['tables'][table_name]['writes'] except KeyError: table_num_consec_write_checks = 0 try: # The return var shows how many times the scale-down criteria # has been met. This is coupled with a var in config, # "num_intervals_scale_down", to delay the scale-down table_num_consec_read_checks, table_num_consec_write_checks = \ table.ensure_provisioning( table_name, table_key, table_num_consec_read_checks, table_num_consec_write_checks) CHECK_STATUS['tables'][table_name] = { 'reads': table_num_consec_read_checks, 'writes': table_num_consec_write_checks } gsi_names = set() # Add regexp table names for gst_instance in dynamodb.table_gsis(table_name): gsi_name = gst_instance[u'IndexName'] try: gsi_keys = get_table_option(table_key, 'gsis').keys() except AttributeError: # Continue if there are not GSIs configured continue for gsi_key in gsi_keys: try: if re.match(gsi_key, gsi_name): logger.debug('Table {0} GSI {1} matches ' 'GSI config key {2}'.format( table_name, gsi_name, gsi_key)) gsi_names.add((gsi_name, gsi_key)) except re.error: logger.error( 'Invalid regular expression: "{0}"'.format( gsi_key)) sys.exit(1) for gsi_name, gsi_key in sorted(gsi_names): try: gsi_num_consec_read_checks = \ CHECK_STATUS['tables'][table_name]['reads'] except KeyError: gsi_num_consec_read_checks = 0 try: gsi_num_consec_write_checks = \ CHECK_STATUS['tables'][table_name]['writes'] except KeyError: gsi_num_consec_write_checks = 0 gsi_num_consec_read_checks, gsi_num_consec_write_checks = \ gsi.ensure_provisioning( table_name, table_key, gsi_name, gsi_key, gsi_num_consec_read_checks, gsi_num_consec_write_checks) CHECK_STATUS['gsis'][gsi_name] = { 'reads': gsi_num_consec_read_checks, 'writes': gsi_num_consec_write_checks } except JSONResponseError as error: exception = error.body['__type'].split('#')[1] if exception == 'ResourceNotFoundException': logger.error('{0} - Table {1} does not exist anymore'.format( table_name, table_name)) continue except BotoServerError as error: if boto_server_error_retries > 0: logger.error( 'Unknown boto error. Status: "{0}". Reason: "{1}"'.format( error.status, error.reason)) logger.error('Please bug report if this error persists') boto_server_error_retries -= 1 continue else: raise # Sleep between the checks logger.debug('Sleeping {0} seconds until next check'.format( get_global_option('check_interval'))) time.sleep(get_global_option('check_interval'))
def execute(): """ Ensure provisioning """ boto_server_error_retries = 3 # Ensure provisioning for table_name, table_key in sorted(dynamodb.get_tables_and_gsis()): try: table.ensure_provisioning(table_name, table_key) gsi_names = set() # Add regexp table names for gst_instance in dynamodb.table_gsis(table_name): gsi_name = gst_instance[u'IndexName'] try: gsi_keys = get_table_option(table_key, 'gsis').keys() except AttributeError: # Continue if there are not GSIs configured continue for gsi_key in gsi_keys: try: if re.match(gsi_key, gsi_name): logger.debug( 'Table {0} GSI {1} matches ' 'GSI config key {2}'.format( table_name, gsi_name, gsi_key)) gsi_names.add((gsi_name, gsi_key)) except re.error: logger.error('Invalid regular expression: "{0}"'.format( gsi_key)) sys.exit(1) for gsi_name, gsi_key in sorted(gsi_names): gsi.ensure_provisioning( table_name, table_key, gsi_name, gsi_key) except JSONResponseError as error: exception = error.body['__type'].split('#')[1] if exception == 'ResourceNotFoundException': logger.error('{0} - Table {1} does not exist anymore'.format( table_name, table_name)) continue except BotoServerError as error: if boto_server_error_retries > 0: logger.error( 'Unknown boto error. Status: "{0}". Reason: "{1}"'.format( error.status, error.reason)) logger.error( 'Please bug report if this error persists') boto_server_error_retries -= 1 continue else: raise # Sleep between the checks logger.debug('Sleeping {0} seconds until next check'.format( get_global_option('check_interval'))) time.sleep(get_global_option('check_interval'))
def main(): """ Main function called from dynamic-dynamodb """ try: boto_server_error_retries = 3 while True: if get_global_option('daemon'): pid_file = '/tmp/dynamic-dynamodb.{0}.pid'.format( get_global_option('instance')) daemon = DynamicDynamoDBDaemon(pid_file) if get_global_option('daemon') == 'start': daemon.start( check_interval=get_global_option('check_interval')) elif get_global_option('daemon') == 'stop': daemon.stop() sys.exit(0) elif get_global_option('daemon') == 'restart': daemon.restart( check_interval=get_global_option('check_interval')) elif get_global_option('daemon') in ['foreground', 'fg']: daemon.run( check_interval=get_global_option('check_interval')) else: print( 'Valid options for --daemon are ' 'start, stop and restart') sys.exit(1) else: # Ensure provisioning for table_name, table_key in dynamodb.get_tables_and_gsis(): try: table.ensure_provisioning(table_name, table_key) gsi_names = set() # Add regexp table names if get_table_option(table_key, 'gsis'): for gst_instance in dynamodb.table_gsis(table_name): gsi_name = gst_instance[u'IndexName'] try: gsi_keys = get_table_option( table_key, 'gsis').keys() except AttributeError: continue for gsi_key in gsi_keys: try: if re.match(gsi_key, gsi_name): logger.debug( 'Table {0} GSI {1} match with ' 'GSI config key {2}'.format( table_name, gsi_name, gsi_key)) gsi_names.add( ( gsi_name, gsi_key )) except re.error: logger.error( 'Invalid regular expression: ' '"{0}"'.format(gsi_key)) sys.exit(1) gsi_names = sorted(gsi_names) for gsi_name, gsi_key in gsi_names: gsi.ensure_provisioning( table_name, table_key, gsi_name, gsi_key) except JSONResponseError as error: exception = error.body['__type'].split('#')[1] if exception == 'ResourceNotFoundException': logger.error( '{0} - Table {1} does not exist anymore'.format( table_name, table_name)) continue except BotoServerError as error: if boto_server_error_retries > 0: logger.error( 'Unknown boto error. Status: "{0}". ' 'Reason: "{1}"'.format( error.status, error.reason)) logger.error( 'Please bug report if this error persists') boto_server_error_retries -= 1 continue else: raise # Sleep between the checks logger.debug('Sleeping {0} seconds until next check'.format( get_global_option('check_interval'))) time.sleep(get_global_option('check_interval')) except Exception as error: logger.exception(error)
def main(): """ Main function called from dynamic-dynamodb """ while True: table_names = set() configured_tables = config['tables'].keys() not_used_tables = configured_tables # Add regexp table names for table_instance in dynamodb.list_tables(): for key_name in configured_tables: if re.match(key_name, table_instance.table_name): logger.debug("Table {0} match with config key {1}".format( table_instance.table_name, key_name)) table_names.add( ( table_instance.table_name, key_name )) not_used_tables.remove(key_name) if not_used_tables: logger.warning( 'No tables matching the following configured ' 'tables found: {0}'.format( ', '.join(not_used_tables))) table_names = sorted(table_names) if config['global']['daemon']: pid_file = '/tmp/dynamic-dynamodb.{0}.pid'.format( config['global']['instance']) daemon = DynamicDynamoDBDaemon(pid_file, tables=table_names) if config['global']['daemon'] == 'start': daemon.start( check_interval=config['global']['check_interval']) elif config['global']['daemon'] == 'stop': daemon.stop() sys.exit(0) elif config['global']['daemon'] == 'restart': daemon.restart() elif config['global']['daemon'] in ['foreground', 'fg']: daemon.run( check_interval=config['global']['check_interval']) else: print 'Valid options for --daemon are start, stop and restart' sys.exit(1) else: # Ensure provisioning for table_name, table_key in table_names: table.ensure_provisioning(table_name, table_key) gsi_names = set() # Add regexp table names if 'gsis' in config['tables'][table_key]: for gst_instance in dynamodb.table_gsis(table_name): gsi_name = gst_instance[u'IndexName'] gsi_keys = config['tables'][table_key]['gsis'].keys() for gsi_key in gsi_keys: if re.match(gsi_key, gsi_name): logger.debug( 'Table {0} GSI {1} match with ' 'GSI config key {2}'.format( table_name, gsi_name, gsi_key)) gsi_names.add( ( gsi_name, gsi_key )) gsi_names = sorted(gsi_names) for gsi_name, gsi_key in gsi_names: gsi.ensure_provisioning( table_name, table_key, gsi_name, gsi_key) # Sleep between the checks logger.debug('Sleeping {0} seconds until next check'.format( config['global']['check_interval'])) time.sleep(config['global']['check_interval'])
def run(self, check_interval=1): """ Run the daemon :type check_interval: int :param check_interval: Delay in seconds between checks """ try: while True: # Ensure provisioning for table_name, table_key in \ sorted(dynamodb.get_tables_and_gsis()): try: table.ensure_provisioning(table_name, table_key) gsi_names = set() # Add regexp table names for gst_instance in dynamodb.table_gsis(table_name): gsi_name = gst_instance[u'IndexName'] try: gsi_keys = get_table_option( table_key, 'gsis').keys() except AttributeError: # Continue if there are not GSIs configured continue for gsi_key in gsi_keys: try: if re.match(gsi_key, gsi_name): logger.debug( 'Table {0} GSI {1} match with ' 'GSI config key {2}'.format( table_name, gsi_name, gsi_key)) gsi_names.add( ( gsi_name, gsi_key )) except re.error: logger.error( 'Invalid regular expression: ' '"{0}"'.format(gsi_key)) sys.exit(1) gsi_names = sorted(gsi_names) for gsi_name, gsi_key in gsi_names: gsi.ensure_provisioning( table_name, table_key, gsi_name, gsi_key) except JSONResponseError as error: exception = error.body['__type'].split('#')[1] if exception == 'ResourceNotFoundException': logger.error( '{0} - Table {1} does not exist anymore'.format( table_name, table_name)) continue # Sleep between the checks logger.debug('Sleeping {0} seconds until next check'.format( check_interval)) time.sleep(check_interval) except Exception as error: logger.exception(error)