def list_clusters():
for namespace in PCWConfig.get_namespaces_for('clusters'):
try:
clusters = EC2(namespace).all_clusters()
logger.info("%d clusters found", len(clusters))
send_cluster_notification(namespace, clusters)
except Exception as e:
logger.exception("[{}] List clusters failed!".format(namespace))
send_mail('{} on List clusters in [{}]'.format(type(e).__name__, namespace), traceback.format_exc())
def test_get_namespaces_for_feature_default_only(pcw_file):
set_pcw_ini(pcw_file, """
[default]
namespaces = test1, test2
""")
namespaces = PCWConfig.get_namespaces_for(
'test_get_namespaces_for_feature_default_only')
assert type(namespaces) is list
assert len(namespaces) == 0
def test_get_namespaces_for_feature_default_feature_exists_no_namespace_in_feature(
pcw_file):
set_pcw_ini(
pcw_file, """
[default]
namespaces = test1, test2
[no_namespace_in_feature]
some_other_property = value
""")
namespaces = PCWConfig.get_namespaces_for('no_namespace_in_feature')
assert type(namespaces) is list
assert len(namespaces) == 2
assert not {'test1', 'test2'} ^ set(namespaces)
def cleanup_run():
for namespace in PCWConfig.get_namespaces_for('cleanup'):
try:
providers = PCWConfig.get_providers_for('cleanup', namespace)
logger.debug("[{}] Run cleanup for {}".format(namespace, ','.join(providers)))
if 'azure' in providers:
Azure(namespace).cleanup_all()
if 'ec2' in providers:
EC2(namespace).cleanup_all()
if 'gce' in providers:
GCE(namespace).cleanup_all()
except Exception as e:
logger.exception("[{}] Cleanup failed!".format(namespace))
send_mail('{} on Cleanup in [{}]'.format(type(e).__name__, namespace), traceback.format_exc())
def send_leftover_notification():
if PCWConfig.has('notify'):
o = Instance.objects
o = o.filter(
active=True,
csp_info__icontains='openqa_created_by',
age__gt=timedelta(
hours=PCWConfig.get_feature_property('notify', 'age-hours')))
body_prefix = "Message from {url}\n\n".format(url=build_absolute_uri())
# Handle namespaces
for namespace in PCWConfig.get_namespaces_for('notify'):
receiver_email = PCWConfig.get_feature_property(
'notify', 'to', namespace)
namespace_objects = o.filter(vault_namespace=namespace)
if namespace_objects.filter(
notified=False).count() > 0 and receiver_email:
send_mail('CSP left overs - {}'.format(namespace),
body_prefix + draw_instance_table(namespace_objects),
receiver_email=receiver_email)
o.update(notified=True)
def update_run():
'''
Each update is using Instance.active to mark the model is still availalbe on CSP.
Instance.state is used to reflect the "local" state, e.g. if someone triggered a delete, the
state will moved to DELETING. If the instance is gone from CSP, the state will set to DELETED.
'''
global __running, __last_update
__running = True
max_retries = 3
error_occured = False
for namespace in PCWConfig.get_namespaces_for('vault'):
for provider in PCWConfig.get_providers_for('vault', namespace):
logger.info("[%s] Check provider %s", namespace, provider)
email_text = set()
for n in range(max_retries):
try:
_update_provider(provider, namespace)
except Exception:
logger.exception("[{}] Update failed for {}".format(
namespace, provider))
email_text.add(traceback.format_exc())
time.sleep(5)
else:
break
else:
error_occured = True
send_mail(
'Error on update {} in namespace {}'.format(
provider, namespace),
"\n{}\n".format('#' * 79).join(email_text))
auto_delete_instances()
send_leftover_notification()
__running = False
if not error_occured:
__last_update = datetime.now(timezone.utc)
if not getScheduler().get_job('update_db'):
init_cron()
def auto_delete_instances():
for namespace in PCWConfig.get_namespaces_for('vault'):
o = Instance.objects
o = o.filter(state=StateChoice.ACTIVE,
vault_namespace=namespace,
ttl__gt=timedelta(0),
age__gte=F('ttl'),
csp_info__icontains='openqa_created_by')
email_text = set()
for i in o:
logger.info("[{}][{}] TTL expire for instance {}".format(
i.provider, i.vault_namespace, i.instance_id))
try:
delete_instance(i)
except Exception:
msg = "[{}][{}] Deleting instance ({}) failed".format(
i.provider, i.vault_namespace, i.instance_id)
logger.exception(msg)
email_text.add("{}\n\n{}".format(msg, traceback.format_exc()))
if len(email_text) > 0:
send_mail(
'[{}] Error on auto deleting instance(s)'.format(namespace),
"\n{}\n".format('#' * 79).join(email_text))
def test_get_namespaces_for_feature_not_defined(pcw_file):
namespaces = PCWConfig.get_namespaces_for(
'test_get_namespaces_for_feature_not_defined')
assert type(namespaces) is list
assert len(namespaces) == 0