def cfgGet(self, section, field):
mapping = {
'cleanup/max-images-per-flavor': {
'default': 1
},
'cleanup/max-image-age-hours': {
'default': 24 * 31
},
'cleanup/min-image-age-hours': {
'default': 24
},
'cleanup/azure-storage-resourcegroup': {
'default': 'openqa-upload'
},
'cleanup/azure-storage-account-name': {
'default': 'openqa'
},
'cleanup/ec2-max-snapshot-age-days': {
'default': -1
},
}
key = '/'.join([section, field])
if key not in mapping:
raise LookupError("Missing {} in mapping list".format(key))
e = mapping[key]
namespace_section = '{}.namespace.{}'.format(section, self.__namespace)
cfg = ConfigFile()
return type(e['default'])(cfg.get([namespace_section, field],
cfg.get([section, field],
e['default'])))
def send_cluster_notification(namespace, clusters):
cfg = ConfigFile()
cfg_path = ['notify.cluster.namespace.{}'.format(namespace), 'to']
if not cfg.has('notify') or not cfg.has(cfg_path):
return
if len(clusters):
clusters_str = ' '.join([str(cluster) for cluster in clusters])
logger.debug("Full clusters list - %s", clusters_str)
send_mail("EC2 clusters found",
clusters_str,
receiver_email=cfg.get(cfg_path))
def list_clusters():
cfg = ConfigFile()
if cfg.has('clusters'):
for vault_namespace in cfg.getList(['clusters', 'namespaces'], ['']):
try:
clusters = EC2(vault_namespace).all_clusters()
logger.info("%d clusters found", len(clusters))
send_cluster_notification(vault_namespace, clusters)
except Exception as e:
logger.exception("[{}] List clusters failed!".format(vault_namespace))
send_mail('{} on List clusters in [{}]'.format(
type(e).__name__, vault_namespace), traceback.format_exc())
def send_mail(subject, message, receiver_email=None):
cfg = ConfigFile()
if not cfg.has('notify'):
return
smtp_server = cfg.get(['notify', 'smtp'])
port = cfg.get(['notify', 'smtp-port'], 25)
sender_email = cfg.get(['notify', 'from'])
if receiver_email is None:
receiver_email = cfg.get(['notify', 'to'])
email = '''\
Subject: [Openqa-Cloud-Watch] {subject}
From: {_from}
To: {_to}
{message}
'''.format(subject=subject,
_from=sender_email,
_to=receiver_email,
message=message)
logger.info("Send Email To:'%s' Subject:'[Openqa-Cloud-Watch] %s'",
receiver_email, subject)
server = smtplib.SMTP(smtp_server, port)
server.ehlo()
server.sendmail(sender_email, receiver_email.split(','), email)
def send_leftover_notification():
cfg = ConfigFile()
if not cfg.has('notify'):
return
o = Instance.objects
o = o.filter(
active=True,
csp_info__icontains='openqa_created_by',
age__gt=timedelta(hours=int(cfg.get(['notify', 'age-hours'], 12))))
if o.filter(notified=False).count() == 0:
return
subject = cfg.get(['notify', 'subject'], 'CSP left overs')
body_prefix = "Message from {url}\n\n".format(url=build_absolute_uri())
send_mail(subject, body_prefix + draw_instance_table(o))
# Handle namespaces
namespaces = list(dict.fromkeys([i.vault_namespace for i in o]))
for namespace in namespaces:
cfg_path = ['notify.namespace.{}'.format(namespace), 'to']
if not cfg.has(cfg_path):
continue
receiver_email = cfg.get(cfg_path)
namespace_objects = o.filter(vault_namespace=namespace)
if namespace_objects.filter(notified=False).count() == 0:
continue
send_mail(subject,
body_prefix + draw_instance_table(namespace_objects),
receiver_email=receiver_email)
o.update(notified=True)
def auto_delete_instances():
cfg = ConfigFile()
for vault_namespace in cfg.getList(['vault', 'namespaces'], ['']):
o = Instance.objects
o = o.filter(state=StateChoice.ACTIVE, vault_namespace=vault_namespace,
ttl__gt=timedelta(0), age__gte=F('ttl'), csp_info__icontains='openqa_created_by')
email_text = set()
for i in o:
logger.info("[{}][{}] TTL expire for instance {}".format(i.provider, i.vault_namespace, i.instance_id))
try:
delete_instance(i)
except Exception:
msg = "[{}][{}] Deleting instance ({}) failed".format(i.provider, i.vault_namespace, i.instance_id)
logger.exception(msg)
email_text.add("{}\n\n{}".format(msg, traceback.format_exc()))
if len(email_text) > 0:
send_mail('[{}] Error on auto deleting instance(s)'.format(vault_namespace),
"\n{}\n".format('#'*79).join(email_text))
def update_run():
'''
Each update is using Instance.active to mark the model is still availalbe on CSP.
Instance.state is used to reflect the "local" state, e.g. if someone triggered a delete, the
state will moved to DELETING. If the instance is gone from CSP, the state will set to DELETED.
'''
global __running, __last_update
__running = True
cfg = ConfigFile()
max_retries = 3
error_occured = False
for vault_namespace in cfg.getList(['vault', 'namespaces'], ['']):
for provider in cfg.getList(['vault.namespace.{}'.format(vault_namespace), 'providers'],
['ec2', 'azure', 'gce']):
logger.info("Check provider %s in vault_namespace %s", provider, vault_namespace)
email_text = set()
for n in range(max_retries):
try:
_update_provider(provider, vault_namespace)
except Exception:
logger.exception("Update failed for {} in namespace {}".format(provider, vault_namespace))
email_text.add(traceback.format_exc())
time.sleep(5)
else:
break
else:
error_occured = True
send_mail('Error on update {} in namespace {}'.format(provider, vault_namespace),
"\n{}\n".format('#'*79).join(email_text))
auto_delete_instances()
send_leftover_notification()
__running = False
if not error_occured:
__last_update = datetime.now(timezone.utc)
if not getScheduler().get_job('update_db'):
init_cron()
def _update_provider(name, vault_namespace):
cfg = ConfigFile()
if 'azure' in name:
instances = Azure(vault_namespace).list_resource_groups()
instances = [azure_to_local_instance(i, vault_namespace) for i in instances]
logger.info("Got %d resources groups from Azure", len(instances))
sync_csp_to_local_db(instances, ProviderChoice.AZURE, vault_namespace)
if 'ec2' in name:
instances = []
for region in cfg.getList(['ec2', 'regions'], EC2(vault_namespace).all_regions()):
instances_csp = EC2(vault_namespace).list_instances(region=region)
instances += [ec2_to_local_instance(i, vault_namespace, region) for i in instances_csp]
logger.info("Got %d instances from EC2 in region %s", len(instances), region)
sync_csp_to_local_db(instances, ProviderChoice.EC2, vault_namespace)
if 'gce' in name:
instances = GCE(vault_namespace).list_all_instances()
instances = [gce_to_local_instance(i, vault_namespace) for i in instances]
logger.info("Got %d instances from GCE", len(instances))
sync_csp_to_local_db(instances, ProviderChoice.GCE, vault_namespace)
def getData(self, name=None):
use_file_cache = ConfigFile().getBoolean(['vault', 'use-file-cache'])
if self.auth_json is None and use_file_cache:
self.auth_json = self.loadAuthCache()
if self.isExpired():
self.auth_json = self.getCredentials()
expire = datetime.today() + timedelta(seconds=self.auth_json['lease_duration'])
self.auth_json['auth_expire'] = expire.isoformat()
if expire > self.client_token_expire:
self.renewClientToken(self.auth_json['lease_duration'])
if use_file_cache:
self.saveAuthCache()
if name is None:
return self.auth_json['data']
return self.auth_json['data'][name]
def __init__(self, vault_namespace):
cfg = ConfigFile()
self.url = cfg.get(['vault', 'url'])
self.user = cfg.get(['vault', 'user'])
self.namespace = vault_namespace
self.password = cfg.get(['vault', 'password'])
self.certificate_dir = cfg.get(['vault', 'cert_dir'], '/etc/ssl/certs')
self.auth_json = None
self.client_token = None
self.client_token_expire = None
def __init__(self, vault_namespace):
cfg = ConfigFile()
self.url = cfg.get(['vault', 'url'])
self.user = cfg.get(['vault', 'user'])
self.namespace = vault_namespace
self.password = cfg.get(['vault', 'password'])
self.certificate_dir = cfg.get(['vault', 'cert_dir'], '/etc/ssl/certs')
if cfg.getBoolean(['vault', 'use-file-cache'
]) and self._getAuthCacheFile().exists():
logger.info('Loading cached credentials')
self.auth_json = self.loadAuthCache()
else:
self.auth_json = None
self.client_token = None
self.client_token_expire = None
def cleanup_run():
cfg = ConfigFile()
if cfg.has('cleanup'):
for vault_namespace in cfg.getList(['cleanup', 'namespaces'], cfg.getList(['vault', 'namespaces'], [''])):
try:
providers = cfg.getList(['vault.namespace.{}'.format(vault_namespace), 'providers'],
['ec2', 'azure', 'gce'])
logger.debug("[{}] Run cleanup for {}".format(vault_namespace, ','.join(providers)))
if 'azure' in providers:
Azure(vault_namespace).cleanup_all()
if 'ec2' in providers:
EC2(vault_namespace).cleanup_all()
if 'gce' in providers:
GCE(vault_namespace).cleanup_all()
except Exception as e:
logger.exception("[{}] Cleanup failed!".format(vault_namespace))
send_mail('{} on Cleanup in [{}]'.format(type(e).__name__, vault_namespace), traceback.format_exc())
def __init__(self, namespace):
self.__namespace = namespace
self.dry_run = ConfigFile().getBoolean(['default', 'dry_run'], False)
self.logger = logging.getLogger(self.__module__)
def __init__(self, namespace):
self.__namespace = namespace
self.dry_run = ConfigFile().getBoolean(['default', 'dry_run'], False)
def renew(self):
if ConfigFile().getBoolean(['vault', 'use-file-cache'
]) and self._getAuthCacheFile().exists():
self._getAuthCacheFile().unlink()
self.revoke()
self.getData()