def send_cluster_notification(namespace, clusters):
if len(clusters) and PCWConfig.has('notify'):
clusters_str = ' '.join([str(cluster) for cluster in clusters])
logger.debug("Full clusters list - %s", clusters_str)
send_mail("EC2 clusters found",
clusters_str,
receiver_email=PCWConfig.get_feature_property(
'cluster.notify', 'to', namespace))
def needs_to_delete_image(self, order_number, image_date):
if self.older_than_min_age(image_date):
max_images_per_flavor = PCWConfig.get_feature_property(
'cleanup', 'max-images-per-flavor', self._namespace)
max_image_age = image_date + timedelta(
hours=PCWConfig.get_feature_property(
'cleanup', 'max-image-age-hours', self._namespace))
return order_number >= max_images_per_flavor or max_image_age < datetime.now(
timezone.utc)
else:
return False
def cleanup_all(self):
cleanup_ec2_max_snapshot_age_days = PCWConfig.get_feature_property(
'cleanup', 'ec2-max-snapshot-age-days', self._namespace)
cleanup_ec2_max_volumes_age_days = PCWConfig.get_feature_property(
'cleanup', 'ec2-max-volumes-age-days', self._namespace)
self.cleanup_images()
if cleanup_ec2_max_snapshot_age_days >= 0:
self.cleanup_snapshots(cleanup_ec2_max_snapshot_age_days)
if cleanup_ec2_max_volumes_age_days >= 0:
self.cleanup_volumes(cleanup_ec2_max_volumes_age_days)
if PCWConfig.getBoolean('cleanup/vpc_cleanup', self._namespace):
self.cleanup_uploader_vpcs()
def __init__(self, vault_namespace):
self.url = PCWConfig.get_feature_property('vault', 'url')
self.user = PCWConfig.get_feature_property('vault', 'user')
self.namespace = vault_namespace
self.password = PCWConfig.get_feature_property('vault', 'password')
self.certificate_dir = PCWConfig.get_feature_property('vault', 'cert_dir')
if PCWConfig.getBoolean('vault/use-file-cache') and self._getAuthCacheFile().exists():
logger.info('Loading cached credentials')
self.auth_json = self.loadAuthCache()
else:
self.auth_json = None
self.client_token = None
self.client_token_expire = None
def send_mail(subject, message, receiver_email=None):
if PCWConfig.has('notify'):
smtp_server = PCWConfig.get_feature_property('notify', 'smtp')
port = PCWConfig.get_feature_property('notify', 'smtp-port')
sender_email = PCWConfig.get_feature_property('notify', 'from')
if receiver_email is None:
receiver_email = PCWConfig.get_feature_property('notify', 'to')
mimetext = MIMEText(message)
mimetext['Subject'] = '[Openqa-Cloud-Watch] {}'.format(subject)
mimetext['From'] = sender_email
mimetext['To'] = receiver_email
logger.info("Send Email To:'%s' Subject:'[Openqa-Cloud-Watch] %s'",
receiver_email, subject)
server = smtplib.SMTP(smtp_server, port)
server.ehlo()
server.sendmail(sender_email, receiver_email.split(','),
mimetext.as_string())
def cleanup_run():
for namespace in PCWConfig.get_namespaces_for('cleanup'):
try:
providers = PCWConfig.get_providers_for('cleanup', namespace)
logger.debug("[{}] Run cleanup for {}".format(namespace, ','.join(providers)))
if 'azure' in providers:
Azure(namespace).cleanup_all()
if 'ec2' in providers:
EC2(namespace).cleanup_all()
if 'gce' in providers:
GCE(namespace).cleanup_all()
except Exception as e:
logger.exception("[{}] Cleanup failed!".format(namespace))
send_mail('{} on Cleanup in [{}]'.format(type(e).__name__, namespace), traceback.format_exc())
def test_get_providers_for_existed_feature(pcw_file):
set_pcw_ini(
pcw_file, """
[providerfeature.namespace.fake]
providers = azure
""")
providers = PCWConfig.get_providers_for('providerfeature', 'fake')
assert not {'azure'} ^ set(providers)
def test_get_namespaces_for_feature_default_only(pcw_file):
set_pcw_ini(pcw_file, """
[default]
namespaces = test1, test2
""")
namespaces = PCWConfig.get_namespaces_for(
'test_get_namespaces_for_feature_default_only')
assert type(namespaces) is list
assert len(namespaces) == 0
def test_get_feature_property_from_pcw_ini_feature(pcw_file):
set_pcw_ini(
pcw_file, """
[cleanup]
max-images-per-flavor = 666
azure-storage-resourcegroup = bla-blub
""")
assert PCWConfig.get_feature_property('cleanup', 'max-images-per-flavor',
'fake') == 666
assert type(
PCWConfig.get_feature_property('cleanup', 'max-images-per-flavor',
'fake')) is int
assert PCWConfig.get_feature_property('cleanup',
'azure-storage-resourcegroup',
'fake') == 'bla-blub'
assert type(
PCWConfig.get_feature_property(
'cleanup', 'azure-storage-resourcegroup', 'fake')) is str
def test_getBoolean_namespace_but_not_defined(pcw_file):
set_pcw_ini(
pcw_file, """
[feature]
bool_property = True
[feature.namespace.random_namespace]
providers = azure
""")
assert PCWConfig.getBoolean('feature/bool_property', 'random_namespace')
def test_getBoolean_defined_namespace(pcw_file):
set_pcw_ini(
pcw_file, """
[feature]
bool_property = False
[feature.namespace.random_namespace]
bool_property = True
""")
assert PCWConfig.getBoolean('feature/bool_property', 'random_namespace')
def list_clusters():
for namespace in PCWConfig.get_namespaces_for('clusters'):
try:
clusters = EC2(namespace).all_clusters()
logger.info("%d clusters found", len(clusters))
send_cluster_notification(namespace, clusters)
except Exception as e:
logger.exception("[{}] List clusters failed!".format(namespace))
send_mail('{} on List clusters in [{}]'.format(type(e).__name__, namespace), traceback.format_exc())
def bs_client(self):
if (self.__blob_service_client is None):
storage_account = PCWConfig.get_feature_property(
'cleanup', 'azure-storage-account-name', self._namespace)
storage_key = self.get_storage_key(storage_account)
connection_string = "{};AccountName={};AccountKey={};EndpointSuffix=core.windows.net".format(
"DefaultEndpointsProtocol=https", storage_account, storage_key)
self.__blob_service_client = BlobServiceClient.from_connection_string(
connection_string)
return self.__blob_service_client
def test_get_feature_property_from_pcw_ini_with_namespace(pcw_file):
set_pcw_ini(
pcw_file, """
[cleanup]
max-images-per-flavor = 666
azure-storage-resourcegroup = bla-blub
[cleanup.namespace.testns]
max-images-per-flavor = 42
azure-storage-resourcegroup = bla-blub-ns
""")
cleanup_max_images_per_flavor = PCWConfig.get_feature_property(
'cleanup', 'max-images-per-flavor', 'testns')
cleanup_azure_storage_resourcegroup = PCWConfig.get_feature_property(
'cleanup', 'azure-storage-resourcegroup', 'testns')
assert cleanup_max_images_per_flavor == 42
assert type(cleanup_max_images_per_flavor) is int
assert cleanup_azure_storage_resourcegroup == 'bla-blub-ns'
assert type(cleanup_azure_storage_resourcegroup) is str
def get_openqa_job_link(self):
tags = self.tags()
if tags.get('openqa_created_by',
'') == 'openqa-suse-de' and 'openqa_var_JOB_ID' in tags:
url = '{}/t{}'.format(
PCWConfig.get_feature_property('webui', 'openqa_url'),
tags['openqa_var_JOB_ID'])
title = tags.get('openqa_var_NAME', '')
return {'url': url, 'title': title}
return None
def send_leftover_notification():
if PCWConfig.has('notify'):
o = Instance.objects
o = o.filter(
active=True,
csp_info__icontains='openqa_created_by',
age__gt=timedelta(
hours=PCWConfig.get_feature_property('notify', 'age-hours')))
body_prefix = "Message from {url}\n\n".format(url=build_absolute_uri())
# Handle namespaces
for namespace in PCWConfig.get_namespaces_for('notify'):
receiver_email = PCWConfig.get_feature_property(
'notify', 'to', namespace)
namespace_objects = o.filter(vault_namespace=namespace)
if namespace_objects.filter(
notified=False).count() > 0 and receiver_email:
send_mail('CSP left overs - {}'.format(namespace),
body_prefix + draw_instance_table(namespace_objects),
receiver_email=receiver_email)
o.update(notified=True)
def test_get_namespaces_for_feature_default_feature_exists_no_namespace_in_feature(
pcw_file):
set_pcw_ini(
pcw_file, """
[default]
namespaces = test1, test2
[no_namespace_in_feature]
some_other_property = value
""")
namespaces = PCWConfig.get_namespaces_for('no_namespace_in_feature')
assert type(namespaces) is list
assert len(namespaces) == 2
assert not {'test1', 'test2'} ^ set(namespaces)
def update_run():
'''
Each update is using Instance.active to mark the model is still availalbe on CSP.
Instance.state is used to reflect the "local" state, e.g. if someone triggered a delete, the
state will moved to DELETING. If the instance is gone from CSP, the state will set to DELETED.
'''
global __running, __last_update
__running = True
max_retries = 3
error_occured = False
for namespace in PCWConfig.get_namespaces_for('vault'):
for provider in PCWConfig.get_providers_for('vault', namespace):
logger.info("[%s] Check provider %s", namespace, provider)
email_text = set()
for n in range(max_retries):
try:
_update_provider(provider, namespace)
except Exception:
logger.exception("[{}] Update failed for {}".format(
namespace, provider))
email_text.add(traceback.format_exc())
time.sleep(5)
else:
break
else:
error_occured = True
send_mail(
'Error on update {} in namespace {}'.format(
provider, namespace),
"\n{}\n".format('#' * 79).join(email_text))
auto_delete_instances()
send_leftover_notification()
__running = False
if not error_occured:
__last_update = datetime.now(timezone.utc)
if not getScheduler().get_job('update_db'):
init_cron()
def getData(self, name=None):
use_file_cache = PCWConfig.getBoolean('vault/use-file-cache')
if self.auth_json is None and use_file_cache:
self.auth_json = self.loadAuthCache()
if self.isExpired():
self.auth_json = self.getCredentials()
expire = datetime.today() + timedelta(seconds=self.auth_json['lease_duration'])
self.auth_json['auth_expire'] = expire.isoformat()
if expire > self.client_token_expire:
self.renewClientToken(self.auth_json['lease_duration'])
if use_file_cache:
self.saveAuthCache()
if name is None:
return self.auth_json['data']
return self.auth_json['data'][name]
def cleanup_uploader_vpcs(self):
for region in self.all_regions:
response = self.ec2_client(region).describe_vpcs(
Filters=[{
'Name': 'isDefault',
'Values': ['false']
}, {
'Name': 'tag:Name',
'Values': ['uploader-*']
}])
for response_vpc in response['Vpcs']:
self.log_info(
'{} in {} looks like uploader leftover. (OwnerId={}).',
response_vpc['VpcId'], region, response_vpc['OwnerId'])
if PCWConfig.getBoolean('cleanup/vpc-notify-only',
self._namespace):
send_mail(
'VPC {} should be deleted, skipping due vpc-notify-only=True'
.format(response_vpc['VpcId']), '')
else:
resource_vpc = self.ec2_resource(region).Vpc(
response_vpc['VpcId'])
can_be_deleted = True
for subnet in resource_vpc.subnets.all():
if len(list(subnet.instances.all())):
self.log_warn(
'{} has associated instance(s) so can not be deleted',
response_vpc['VpcId'])
can_be_deleted = False
break
if can_be_deleted:
self.delete_vpc(region, resource_vpc,
response_vpc['VpcId'])
elif not self.dry_run:
body = 'Uploader leftover {} (OwnerId={}) in {} is locked'.format(
response_vpc['VpcId'], response_vpc['OwnerId'],
region)
send_mail('VPC deletion locked by running VMs', body)
def test_get_feature_property_with_defaults(pcw_file):
assert PCWConfig.get_feature_property('cleanup', 'max-images-per-flavor',
'fake') == 1
assert type(
PCWConfig.get_feature_property('cleanup', 'max-images-per-flavor',
'fake')) is int
assert type(
PCWConfig.get_feature_property('cleanup', 'min-image-age-hours',
'fake')) is int
assert type(
PCWConfig.get_feature_property('cleanup', 'max-image-age-hours',
'fake')) is int
assert PCWConfig.get_feature_property('cleanup',
'azure-storage-resourcegroup',
'fake') == 'openqa-upload'
assert type(
PCWConfig.get_feature_property(
'cleanup', 'azure-storage-resourcegroup', 'fake')) is str
def auto_delete_instances():
for namespace in PCWConfig.get_namespaces_for('vault'):
o = Instance.objects
o = o.filter(state=StateChoice.ACTIVE,
vault_namespace=namespace,
ttl__gt=timedelta(0),
age__gte=F('ttl'),
csp_info__icontains='openqa_created_by')
email_text = set()
for i in o:
logger.info("[{}][{}] TTL expire for instance {}".format(
i.provider, i.vault_namespace, i.instance_id))
try:
delete_instance(i)
except Exception:
msg = "[{}][{}] Deleting instance ({}) failed".format(
i.provider, i.vault_namespace, i.instance_id)
logger.exception(msg)
email_text.add("{}\n\n{}".format(msg, traceback.format_exc()))
if len(email_text) > 0:
send_mail(
'[{}] Error on auto deleting instance(s)'.format(namespace),
"\n{}\n".format('#' * 79).join(email_text))
def renew(self):
if PCWConfig.getBoolean(
'vault/use-file-cache') and self._getAuthCacheFile().exists():
self._getAuthCacheFile().unlink()
self.revoke()
self.getData()
def test_getBoolean_notdefined_namespace(pcw_file):
assert not PCWConfig.getBoolean('feature/bool_property',
'random_namespace')
def test_getBoolean_defined(pcw_file):
set_pcw_ini(pcw_file, """
[feature]
bool_property = True
""")
assert PCWConfig.getBoolean('feature/bool_property')
def test_get_feature_property_lookup_error(pcw_file):
with pytest.raises(LookupError):
PCWConfig.get_feature_property('notexisting', 'notexisting', 'fake')
def test_get_namespaces_for_feature_not_defined(pcw_file):
namespaces = PCWConfig.get_namespaces_for(
'test_get_namespaces_for_feature_not_defined')
assert type(namespaces) is list
assert len(namespaces) == 0
def __init__(self, namespace: str):
super().__init__(namespace)
self.__resource_group = PCWConfig.get_feature_property(
'cleanup', 'azure-storage-resourcegroup', namespace)
self.check_credentials()
def __init__(self, namespace: str):
self._namespace = namespace
self.dry_run = PCWConfig.getBoolean('default/dry_run')
self.logger = logging.getLogger(self.__module__)
def older_than_min_age(self, age):
return datetime.now(timezone.utc) > age + timedelta(
hours=PCWConfig.get_feature_property(
'cleanup', 'min-image-age-hours', self._namespace))