def run(self, check_interval=1):
""" Run the daemon
:type check_interval: int
:param check_interval: Delay in seconds between checks
"""
while True:
# Ensure provisioning
for table_name, table_key in sorted(self.tables):
table.ensure_provisioning(table_name, table_key)
gsi_names = set()
# Add regexp table names
for gst_instance in dynamodb.table_gsis(table_name):
gsi_name = gst_instance[u'IndexName']
for gsi_key in config['tables'][table_key]['gsis'].keys():
if re.match(gsi_key, gsi_name):
logger.debug(
'Table {0} GSI {1} match with '
'GSI config key {2}'.format(
table_name, gsi_name, gsi_key))
gsi_names.add(
(
gsi_name,
gsi_key
))
gsi_names = sorted(gsi_names)
for gsi_name, gsi_key in gsi_names:
gsi.ensure_provisioning(
table_name,
table_key,
gsi_name,
gsi_key)
# Sleep between the checks
logger.debug('Sleeping {0} seconds until next check'.format(
check_interval))
time.sleep(check_interval)
def execute():
""" Ensure provisioning """
boto_server_error_retries = 3
# Ensure provisioning
for table_name, table_key in sorted(dynamodb.get_tables_and_gsis()):
try:
table.ensure_provisioning(table_name, table_key)
gsi_names = set()
# Add regexp table names
for gst_instance in dynamodb.table_gsis(table_name):
gsi_name = gst_instance[u'IndexName']
try:
gsi_keys = get_table_option(table_key, 'gsis').keys()
except AttributeError:
# Continue if there are not GSIs configured
continue
for gsi_key in gsi_keys:
try:
if re.match(gsi_key, gsi_name):
logger.debug(
'Table {0} GSI {1} matches '
'GSI config key {2}'.format(
table_name, gsi_name, gsi_key))
gsi_names.add((gsi_name, gsi_key))
except re.error:
logger.error('Invalid regular expression: "{0}"'.format(
gsi_key))
sys.exit(1)
for gsi_name, gsi_key in sorted(gsi_names):
gsi.ensure_provisioning(
table_name,
table_key,
gsi_name,
gsi_key)
except JSONResponseError as error:
exception = error.body['__type'].split('#')[1]
if exception == 'ResourceNotFoundException':
logger.error('{0} - Table {1} does not exist anymore'.format(
table_name,
table_name))
continue
except BotoServerError as error:
if boto_server_error_retries > 0:
logger.error(
'Unknown boto error. Status: "{0}". Reason: "{1}"'.format(
error.status,
error.reason))
logger.error(
'Please bug report if this error persists')
boto_server_error_retries -= 1
continue
else:
raise
# Sleep between the checks
logger.debug('Sleeping {0} seconds until next check'.format(
get_global_option('check_interval')))
time.sleep(get_global_option('check_interval'))
def main():
""" Main function called from dynamic-dynamodb """
try:
boto_server_error_retries = 3
while True:
if get_global_option('daemon'):
pid_file = '/tmp/dynamic-dynamodb.{0}.pid'.format(
get_global_option('instance'))
daemon = DynamicDynamoDBDaemon(pid_file)
if get_global_option('daemon') == 'start':
daemon.start(
check_interval=get_global_option('check_interval'))
elif get_global_option('daemon') == 'stop':
daemon.stop()
sys.exit(0)
elif get_global_option('daemon') == 'restart':
daemon.restart(
check_interval=get_global_option('check_interval'))
elif get_global_option('daemon') in ['foreground', 'fg']:
daemon.run(
check_interval=get_global_option('check_interval'))
else:
print(
'Valid options for --daemon are '
'start, stop and restart')
sys.exit(1)
else:
# Ensure provisioning
for table_name, table_key in dynamodb.get_tables_and_gsis():
try:
table.ensure_provisioning(table_name, table_key)
gsi_names = set()
# Add regexp table names
if get_table_option(table_key, 'gsis'):
for gst_instance in dynamodb.table_gsis(table_name):
gsi_name = gst_instance[u'IndexName']
try:
gsi_keys = get_table_option(
table_key, 'gsis').keys()
except AttributeError:
continue
for gsi_key in gsi_keys:
try:
if re.match(gsi_key, gsi_name):
logger.debug(
'Table {0} GSI {1} match with '
'GSI config key {2}'.format(
table_name,
gsi_name,
gsi_key))
gsi_names.add(
(
gsi_name,
gsi_key
))
except re.error:
logger.error(
'Invalid regular expression: '
'"{0}"'.format(gsi_key))
sys.exit(1)
gsi_names = sorted(gsi_names)
for gsi_name, gsi_key in gsi_names:
gsi.ensure_provisioning(
table_name,
table_key,
gsi_name,
gsi_key)
except JSONResponseError as error:
exception = error.body['__type'].split('#')[1]
if exception == 'ResourceNotFoundException':
logger.error(
'{0} - Table {1} does not exist anymore'.format(
table_name, table_name))
continue
except BotoServerError as error:
if boto_server_error_retries > 0:
logger.error(
'Unknown boto error. Status: "{0}". '
'Reason: "{1}"'.format(
error.status,
error.reason))
logger.error(
'Please bug report if this error persists')
boto_server_error_retries -= 1
continue
else:
raise
# Sleep between the checks
logger.debug('Sleeping {0} seconds until next check'.format(
get_global_option('check_interval')))
time.sleep(get_global_option('check_interval'))
except Exception as error:
logger.exception(error)
def main():
""" Main function called from dynamic-dynamodb """
while True:
table_names = set()
configured_tables = config['tables'].keys()
not_used_tables = configured_tables
# Add regexp table names
for table_instance in dynamodb.list_tables():
for key_name in configured_tables:
if re.match(key_name, table_instance.table_name):
logger.debug("Table {0} match with config key {1}".format(
table_instance.table_name, key_name))
table_names.add(
(
table_instance.table_name,
key_name
))
not_used_tables.remove(key_name)
if not_used_tables:
logger.warning(
'No tables matching the following configured '
'tables found: {0}'.format(
', '.join(not_used_tables)))
table_names = sorted(table_names)
if config['global']['daemon']:
pid_file = '/tmp/dynamic-dynamodb.{0}.pid'.format(
config['global']['instance'])
daemon = DynamicDynamoDBDaemon(pid_file, tables=table_names)
if config['global']['daemon'] == 'start':
daemon.start(
check_interval=config['global']['check_interval'])
elif config['global']['daemon'] == 'stop':
daemon.stop()
sys.exit(0)
elif config['global']['daemon'] == 'restart':
daemon.restart()
elif config['global']['daemon'] in ['foreground', 'fg']:
daemon.run(
check_interval=config['global']['check_interval'])
else:
print 'Valid options for --daemon are start, stop and restart'
sys.exit(1)
else:
# Ensure provisioning
for table_name, table_key in table_names:
table.ensure_provisioning(table_name, table_key)
gsi_names = set()
# Add regexp table names
if 'gsis' in config['tables'][table_key]:
for gst_instance in dynamodb.table_gsis(table_name):
gsi_name = gst_instance[u'IndexName']
gsi_keys = config['tables'][table_key]['gsis'].keys()
for gsi_key in gsi_keys:
if re.match(gsi_key, gsi_name):
logger.debug(
'Table {0} GSI {1} match with '
'GSI config key {2}'.format(
table_name, gsi_name, gsi_key))
gsi_names.add(
(
gsi_name,
gsi_key
))
gsi_names = sorted(gsi_names)
for gsi_name, gsi_key in gsi_names:
gsi.ensure_provisioning(
table_name,
table_key,
gsi_name,
gsi_key)
# Sleep between the checks
logger.debug('Sleeping {0} seconds until next check'.format(
config['global']['check_interval']))
time.sleep(config['global']['check_interval'])
def run(self, check_interval=1):
""" Run the daemon
:type check_interval: int
:param check_interval: Delay in seconds between checks
"""
try:
while True:
# Ensure provisioning
for table_name, table_key in \
sorted(dynamodb.get_tables_and_gsis()):
try:
table.ensure_provisioning(table_name, table_key)
gsi_names = set()
# Add regexp table names
for gst_instance in dynamodb.table_gsis(table_name):
gsi_name = gst_instance[u'IndexName']
try:
gsi_keys = get_table_option(
table_key, 'gsis').keys()
except AttributeError:
# Continue if there are not GSIs configured
continue
for gsi_key in gsi_keys:
try:
if re.match(gsi_key, gsi_name):
logger.debug(
'Table {0} GSI {1} match with '
'GSI config key {2}'.format(
table_name, gsi_name, gsi_key))
gsi_names.add(
(
gsi_name,
gsi_key
))
except re.error:
logger.error(
'Invalid regular expression: '
'"{0}"'.format(gsi_key))
sys.exit(1)
gsi_names = sorted(gsi_names)
for gsi_name, gsi_key in gsi_names:
gsi.ensure_provisioning(
table_name,
table_key,
gsi_name,
gsi_key)
except JSONResponseError as error:
exception = error.body['__type'].split('#')[1]
if exception == 'ResourceNotFoundException':
logger.error(
'{0} - Table {1} does not exist anymore'.format(
table_name, table_name))
continue
# Sleep between the checks
logger.debug('Sleeping {0} seconds until next check'.format(
check_interval))
time.sleep(check_interval)
except Exception as error:
logger.exception(error)
