def update_run():
'''
Each update is using Instance.active to mark the model is still availalbe on CSP.
Instance.state is used to reflect the "local" state, e.g. if someone triggered a delete, the
state will moved to DELETING. If the instance is gone from CSP, the state will set to DELETED.
'''
global __running, __last_update
__running = True
cfg = ConfigFile()
max_retries = 3
error_occured = False
for vault_namespace in cfg.getList(['vault', 'namespaces'], ['']):
for provider in cfg.getList(['vault.namespace.{}'.format(vault_namespace), 'providers'],
['ec2', 'azure', 'gce']):
logger.info("Check provider %s in vault_namespace %s", provider, vault_namespace)
email_text = set()
for n in range(max_retries):
try:
_update_provider(provider, vault_namespace)
except Exception:
logger.exception("Update failed for {} in namespace {}".format(provider, vault_namespace))
email_text.add(traceback.format_exc())
time.sleep(5)
else:
break
else:
error_occured = True
send_mail('Error on update {} in namespace {}'.format(provider, vault_namespace),
"\n{}\n".format('#'*79).join(email_text))
auto_delete_instances()
send_leftover_notification()
__running = False
if not error_occured:
__last_update = datetime.now(timezone.utc)
if not getScheduler().get_job('update_db'):
init_cron()
def init_cron():
getScheduler().add_job(update_run, trigger='interval', minutes=5, id='update_db')
def start_update():
global __running
if not __running:
getScheduler().get_job('update_db').reschedule(trigger='date', run_date=datetime.now(timezone.utc))
def init_cron():
getScheduler().add_job(cleanup_run, trigger='interval', minutes=60, id='cleanup_all')
getScheduler().add_job(list_clusters, trigger='interval', hours=18, id='list_clusters')