def cleanup_run():
cfg = ConfigFile()
if cfg.has('cleanup'):
for vault_namespace in cfg.getList(['cleanup', 'namespaces'], cfg.getList(['vault', 'namespaces'], [''])):
try:
providers = cfg.getList(['vault.namespace.{}'.format(vault_namespace), 'providers'],
['ec2', 'azure', 'gce'])
logger.debug("[{}] Run cleanup for {}".format(vault_namespace, ','.join(providers)))
if 'azure' in providers:
Azure(vault_namespace).cleanup_all()
if 'ec2' in providers:
EC2(vault_namespace).cleanup_all()
if 'gce' in providers:
GCE(vault_namespace).cleanup_all()
except Exception as e:
logger.exception("[{}] Cleanup failed!".format(vault_namespace))
send_mail('{} on Cleanup in [{}]'.format(type(e).__name__, vault_namespace), traceback.format_exc())
def list_clusters():
cfg = ConfigFile()
if cfg.has('clusters'):
for vault_namespace in cfg.getList(['clusters', 'namespaces'], ['']):
try:
clusters = EC2(vault_namespace).all_clusters()
logger.info("%d clusters found", len(clusters))
send_cluster_notification(vault_namespace, clusters)
except Exception as e:
logger.exception("[{}] List clusters failed!".format(vault_namespace))
send_mail('{} on List clusters in [{}]'.format(
type(e).__name__, vault_namespace), traceback.format_exc())
def update_run():
'''
Each update is using Instance.active to mark the model is still availalbe on CSP.
Instance.state is used to reflect the "local" state, e.g. if someone triggered a delete, the
state will moved to DELETING. If the instance is gone from CSP, the state will set to DELETED.
'''
global __running, __last_update
__running = True
cfg = ConfigFile()
max_retries = 3
error_occured = False
for vault_namespace in cfg.getList(['vault', 'namespaces'], ['']):
for provider in cfg.getList(['vault.namespace.{}'.format(vault_namespace), 'providers'],
['ec2', 'azure', 'gce']):
logger.info("Check provider %s in vault_namespace %s", provider, vault_namespace)
email_text = set()
for n in range(max_retries):
try:
_update_provider(provider, vault_namespace)
except Exception:
logger.exception("Update failed for {} in namespace {}".format(provider, vault_namespace))
email_text.add(traceback.format_exc())
time.sleep(5)
else:
break
else:
error_occured = True
send_mail('Error on update {} in namespace {}'.format(provider, vault_namespace),
"\n{}\n".format('#'*79).join(email_text))
auto_delete_instances()
send_leftover_notification()
__running = False
if not error_occured:
__last_update = datetime.now(timezone.utc)
if not getScheduler().get_job('update_db'):
init_cron()
def auto_delete_instances():
cfg = ConfigFile()
for vault_namespace in cfg.getList(['vault', 'namespaces'], ['']):
o = Instance.objects
o = o.filter(state=StateChoice.ACTIVE, vault_namespace=vault_namespace,
ttl__gt=timedelta(0), age__gte=F('ttl'), csp_info__icontains='openqa_created_by')
email_text = set()
for i in o:
logger.info("[{}][{}] TTL expire for instance {}".format(i.provider, i.vault_namespace, i.instance_id))
try:
delete_instance(i)
except Exception:
msg = "[{}][{}] Deleting instance ({}) failed".format(i.provider, i.vault_namespace, i.instance_id)
logger.exception(msg)
email_text.add("{}\n\n{}".format(msg, traceback.format_exc()))
if len(email_text) > 0:
send_mail('[{}] Error on auto deleting instance(s)'.format(vault_namespace),
"\n{}\n".format('#'*79).join(email_text))
def _update_provider(name, vault_namespace):
cfg = ConfigFile()
if 'azure' in name:
instances = Azure(vault_namespace).list_resource_groups()
instances = [azure_to_local_instance(i, vault_namespace) for i in instances]
logger.info("Got %d resources groups from Azure", len(instances))
sync_csp_to_local_db(instances, ProviderChoice.AZURE, vault_namespace)
if 'ec2' in name:
instances = []
for region in cfg.getList(['ec2', 'regions'], EC2(vault_namespace).all_regions()):
instances_csp = EC2(vault_namespace).list_instances(region=region)
instances += [ec2_to_local_instance(i, vault_namespace, region) for i in instances_csp]
logger.info("Got %d instances from EC2 in region %s", len(instances), region)
sync_csp_to_local_db(instances, ProviderChoice.EC2, vault_namespace)
if 'gce' in name:
instances = GCE(vault_namespace).list_all_instances()
instances = [gce_to_local_instance(i, vault_namespace) for i in instances]
logger.info("Got %d instances from GCE", len(instances))
sync_csp_to_local_db(instances, ProviderChoice.GCE, vault_namespace)