def send_leftover_notification():
cfg = ConfigFile()
if not cfg.has('notify'):
return
o = Instance.objects
o = o.filter(
active=True,
csp_info__icontains='openqa_created_by',
age__gt=timedelta(hours=int(cfg.get(['notify', 'age-hours'], 12))))
if o.filter(notified=False).count() == 0:
return
subject = cfg.get(['notify', 'subject'], 'CSP left overs')
body_prefix = "Message from {url}\n\n".format(url=build_absolute_uri())
send_mail(subject, body_prefix + draw_instance_table(o))
# Handle namespaces
namespaces = list(dict.fromkeys([i.vault_namespace for i in o]))
for namespace in namespaces:
cfg_path = ['notify.namespace.{}'.format(namespace), 'to']
if not cfg.has(cfg_path):
continue
receiver_email = cfg.get(cfg_path)
namespace_objects = o.filter(vault_namespace=namespace)
if namespace_objects.filter(notified=False).count() == 0:
continue
send_mail(subject,
body_prefix + draw_instance_table(namespace_objects),
receiver_email=receiver_email)
o.update(notified=True)
def send_cluster_notification(namespace, clusters):
cfg = ConfigFile()
cfg_path = ['notify.cluster.namespace.{}'.format(namespace), 'to']
if not cfg.has('notify') or not cfg.has(cfg_path):
return
if len(clusters):
clusters_str = ' '.join([str(cluster) for cluster in clusters])
logger.debug("Full clusters list - %s", clusters_str)
send_mail("EC2 clusters found",
clusters_str,
receiver_email=cfg.get(cfg_path))
def send_mail(subject, message, receiver_email=None):
cfg = ConfigFile()
if not cfg.has('notify'):
return
smtp_server = cfg.get(['notify', 'smtp'])
port = cfg.get(['notify', 'smtp-port'], 25)
sender_email = cfg.get(['notify', 'from'])
if receiver_email is None:
receiver_email = cfg.get(['notify', 'to'])
email = '''\
Subject: [Openqa-Cloud-Watch] {subject}
From: {_from}
To: {_to}
{message}
'''.format(subject=subject,
_from=sender_email,
_to=receiver_email,
message=message)
logger.info("Send Email To:'%s' Subject:'[Openqa-Cloud-Watch] %s'",
receiver_email, subject)
server = smtplib.SMTP(smtp_server, port)
server.ehlo()
server.sendmail(sender_email, receiver_email.split(','), email)
def list_clusters():
cfg = ConfigFile()
if cfg.has('clusters'):
for vault_namespace in cfg.getList(['clusters', 'namespaces'], ['']):
try:
clusters = EC2(vault_namespace).all_clusters()
logger.info("%d clusters found", len(clusters))
send_cluster_notification(vault_namespace, clusters)
except Exception as e:
logger.exception("[{}] List clusters failed!".format(vault_namespace))
send_mail('{} on List clusters in [{}]'.format(
type(e).__name__, vault_namespace), traceback.format_exc())
def cleanup_run():
cfg = ConfigFile()
if cfg.has('cleanup'):
for vault_namespace in cfg.getList(['cleanup', 'namespaces'], cfg.getList(['vault', 'namespaces'], [''])):
try:
providers = cfg.getList(['vault.namespace.{}'.format(vault_namespace), 'providers'],
['ec2', 'azure', 'gce'])
logger.debug("[{}] Run cleanup for {}".format(vault_namespace, ','.join(providers)))
if 'azure' in providers:
Azure(vault_namespace).cleanup_all()
if 'ec2' in providers:
EC2(vault_namespace).cleanup_all()
if 'gce' in providers:
GCE(vault_namespace).cleanup_all()
except Exception as e:
logger.exception("[{}] Cleanup failed!".format(vault_namespace))
send_mail('{} on Cleanup in [{}]'.format(type(e).__name__, vault_namespace), traceback.format_exc())