def pytest_configure(config):
if store.parallelizer_role == 'master' or trackerbot.conf.get('url') is None:
return
# A further optimization here is to make the calls to trackerbot per provider
# and perhaps only pull the providers that are needed, however that will need
# to ensure that the tests that just randomly use providers adhere to the filters
# which may be too tricky right now.
count = 0
if not config.getoption('use_template_cache'):
store.terminalreporter.line("Loading templates from trackerbot...", green=True)
provider_templates = trackerbot.provider_templates(trackerbot.api())
for provider in list_provider_keys():
TEMPLATES[provider] = provider_templates.get(provider, [])
config.cache.set('miq-trackerbot/{}'.format(provider), TEMPLATES[provider])
count += len(TEMPLATES[provider])
else:
store.terminalreporter.line("Using templates from cache...", green=True)
provider_templates = None
for provider in list_provider_keys():
templates = config.cache.get('miq-trackerbot/{}'.format(provider), None)
if templates is None:
store.terminalreporter.line(
"Loading templates for {} from source as not in cache".format(
provider), green=True)
if not provider_templates:
provider_templates = trackerbot.provider_templates(trackerbot.api())
templates = provider_templates.get(provider, [])
config.cache.set('miq-trackerbot/{}'.format(provider), templates)
count += len(templates)
TEMPLATES[provider] = templates
store.terminalreporter.line(" Loaded {} templates successfully!".format(count), green=True)
def templates_uploaded_on_providers(api, stream, template):
if get_untested_templates(api, stream, template):
print(
'report will not be generated, proceed with the next untested provider'
)
sys.exit()
for temp in api.template.get(limit=1, tested=False,
group__name=stream).get('objects', []):
if 'template_rhevm' in images_uploaded(stream):
if not provider_in_the_list(list_provider_keys('rhevm'),
temp['providers']):
return False
if 'template_rhos' in images_uploaded(stream):
if not provider_in_the_list(list_provider_keys('openstack'),
temp['providers']):
return False
if 'template_vsphere' in images_uploaded(stream):
if not provider_in_the_list(list_provider_keys('virtualcenter'),
temp['providers']):
return False
if 'template_scvmm' in images_uploaded(stream):
if not provider_in_the_list(list_provider_keys('scvmm'),
temp['providers']):
return False
return True
def ec2cleanup(exclude_volumes, exclude_eips, exclude_elbs, exclude_enis, exclude_stacks,
stack_template, output):
with open(output, 'w') as report:
report.write('ec2cleanup.py, Address, Volume, LoadBalancer and Network Interface Cleanup')
report.write("\nDate: {}\n".format(datetime.now()))
for provider_key in list_provider_keys('ec2'):
provider_mgmt = get_mgmt(provider_key)
logger.info("----- Provider: %r -----", provider_key)
logger.info("Deleting volumes...")
delete_unattached_volumes(provider_mgmt=provider_mgmt,
excluded_volumes=exclude_volumes,
output=output)
logger.info("Deleting Elastic LoadBalancers...")
delete_unused_loadbalancers(provider_mgmt=provider_mgmt,
excluded_elbs=exclude_elbs,
output=output)
logger.info("Deleting Elastic Network Interfaces...")
delete_unused_network_interfaces(provider_mgmt=provider_mgmt,
excluded_enis=exclude_enis,
output=output)
logger.info("Deleting old stacks...")
delete_stacks(provider_mgmt=provider_mgmt,
excluded_stacks=exclude_stacks,
stack_template=stack_template,
output=output)
logger.info("Releasing addresses...")
delete_disassociated_addresses(provider_mgmt=provider_mgmt,
excluded_eips=exclude_eips,
output=output)
def run(**kwargs):
# Setup defaults for the cli tool machine
host = kwargs.get('ssh_host') or \
cfme_data['template_upload']['template_upload_ec2']['aws_cli_tool_client']
user = kwargs.get('ssh_user') or credentials['host_default']['username']
passwd = kwargs.get('ssh_pass') or credentials['host_default']['password']
# Download file once and thread uploading to different gce regions
with make_ssh_client(host, user, passwd) as ssh_client:
file_name, file_path = download_image_file(kwargs.get('image_url'),
ssh_client)
thread_queue = []
for provider in list_provider_keys("gce"):
# skip provider if block_upload is set
provider_yaml = cfme_data.management_systems.get(provider)
if (provider_yaml.get('template_upload')
and provider_yaml.template_upload.get('block_upload')):
logger.info(
'Skipping upload on {} due to block_upload'.format(provider))
continue
template_name = kwargs.get('template_name')
bucket_name = kwargs.get('bucket_name')
stream = kwargs.get('stream')
with make_ssh_client(host, user, passwd) as ssh_client:
thread = Thread(target=upload_template,
args=(provider, template_name, stream, file_name,
file_path, ssh_client, bucket_name))
thread.daemon = True
thread_queue.append(thread)
thread.start()
for thread in thread_queue:
thread.join()
def ec2cleanup(exclude_volumes, exclude_eips, exclude_elbs, exclude_enis,
exclude_stacks, stack_template, output):
with open(output, 'w') as report:
report.write(
'ec2cleanup.py, Address, Volume, LoadBalancer and Network Interface Cleanup'
)
report.write("\nDate: {}\n".format(datetime.now()))
for provider_key in list_provider_keys('ec2'):
provider_mgmt = get_mgmt(provider_key)
logger.info("----- Provider: %r -----", provider_key)
logger.info("Deleting volumes...")
delete_unattached_volumes(provider_mgmt=provider_mgmt,
excluded_volumes=exclude_volumes,
output=output)
logger.info("Deleting Elastic LoadBalancers...")
delete_unused_loadbalancers(provider_mgmt=provider_mgmt,
excluded_elbs=exclude_elbs,
output=output)
logger.info("Deleting Elastic Network Interfaces...")
delete_unused_network_interfaces(provider_mgmt=provider_mgmt,
excluded_enis=exclude_enis,
output=output)
logger.info("Deleting old stacks...")
delete_stacks(provider_mgmt=provider_mgmt,
excluded_stacks=exclude_stacks,
stack_template=stack_template,
output=output)
logger.info("Releasing addresses...")
delete_disassociated_addresses(provider_mgmt=provider_mgmt,
excluded_eips=exclude_eips,
output=output)
def run(**kwargs):
# Setup defaults for the cli tool machine
host = kwargs.get('ssh_host') or \
cfme_data['template_upload']['template_upload_ec2']['aws_cli_tool_client']
user = kwargs.get('ssh_user') or credentials['host_default']['username']
passwd = kwargs.get('ssh_pass') or credentials['host_default']['password']
# Download file once and thread uploading to different gce regions
with make_ssh_client(host, user, passwd) as ssh_client:
file_name, file_path = download_image_file(kwargs.get('image_url'),
ssh_client)
thread_queue = []
for provider in list_provider_keys("gce"):
template_name = kwargs.get('template_name')
bucket_name = kwargs.get('bucket_name')
stream = kwargs.get('stream')
with make_ssh_client(host, user, passwd) as ssh_client:
thread = Thread(target=upload_template,
args=(provider, template_name, stream, file_name,
file_path, ssh_client, bucket_name))
thread.daemon = True
thread_queue.append(thread)
thread.start()
for thread in thread_queue:
thread.join()
def azure_cleanup(nic_template, pip_template, days_old):
logger.info('azure_cleanup.py, NICs, PIPs, Disks and Stack Cleanup')
logger.info("Date: {}".format(datetime.now()))
errors = []
for prov_key in list_provider_keys('azure'):
logger.info("----- Provider: '%s' -----", prov_key)
mgmt = get_mgmt(prov_key)
mgmt.logger = logger
for name, scr_id in mgmt.list_subscriptions():
logger.info("Subscription '%s' is chosen", name)
setattr(mgmt, 'subscription_id', scr_id)
for resource_group in mgmt.list_resource_groups():
mgmt.logger.info('Checking "%s" resource group:',
resource_group)
# removing stale nics
try:
mgmt.remove_nics_by_search(nic_template, resource_group)
except Exception as e:
logger.warning("NIC cleanup failed")
errors.append(e)
# removing public ips
try:
mgmt.remove_pips_by_search(pip_template, resource_group)
except Exception as e:
logger.warning("Public IP cleanup failed")
errors.append(e)
# removing stale stacks
try:
stack_list = mgmt.list_stack(resource_group=resource_group,
days_old=days_old)
if stack_list:
removed_stacks = []
for stack in stack_list:
if mgmt.is_stack_empty(
stack, resource_group=resource_group):
removed_stacks.append(stack)
mgmt.delete_stack(stack, resource_group)
if not removed_stacks:
logger.info(
"No empty stacks older '%s' days were found",
days_old)
except Exception as e:
logger.warning("Removing Stacks failed")
errors.append(e)
try:
mgmt.remove_unused_blobs(resource_group)
except Exception as e:
logger.warning("Removing unused blobs failed")
errors.append(e)
if errors:
tb.format_exc()
return 1
else:
return 0
def run(**kwargs):
thread_queue = []
providers = list_provider_keys("virtualcenter")
if kwargs['provider_data']:
mgmt_sys = providers = kwargs['provider_data']['management_systems']
else:
mgmt_sys = cfme_data.management_systems
# Store thread results, no need to use a lock
# because threads will not be adding new keys
results = {provider: None for provider in providers}
for provider in providers:
# skip provider if block_upload is set
if (mgmt_sys[provider].get('template_upload') and
mgmt_sys[provider]['template_upload'].get('block_upload')):
logger.info('Skipping upload on %s due to block_upload', provider)
continue
if kwargs['provider_data']:
if mgmt_sys[provider]['type'] != 'virtualcenter':
continue
username = mgmt_sys[provider]['username']
password = mgmt_sys[provider]['password']
else:
creds = credentials[mgmt_sys[provider]['credentials']]
username = creds['username']
password = creds['password']
host_ip = mgmt_sys[provider]['ipaddress']
hostname = mgmt_sys[provider]['hostname']
client = VMWareSystem(hostname, username, password)
if not net.is_pingable(host_ip):
continue
thread = Thread(target=upload_template,
args=(client, hostname, username, password, provider,
kwargs.get('image_url'), kwargs.get('template_name'),
kwargs['provider_data'], kwargs['stream'], results))
thread.daemon = True
thread_queue.append(thread)
thread.start()
for thread in thread_queue:
thread.join()
failed_providers = [provider for provider, result in results.items() if result is False]
skipped_providers = [provider for provider, result in results.items() if result is None]
passed_providers = [provider for provider, result in results.items() if result]
logger.info("providers skipped: %s", skipped_providers)
logger.info("providers passed: %s", passed_providers)
logger.info("providers failed: %s", failed_providers)
if not passed_providers:
raise Exception("Template upload failed for all providers")
else:
logger.info("Upload passed for at least 1 provider... success!")
def run(**kwargs):
thread_queue = []
providers = list_provider_keys("virtualcenter")
if kwargs['provider_data']:
mgmt_sys = providers = kwargs['provider_data']['management_systems']
else:
mgmt_sys = cfme_data.management_systems
# Store thread results, no need to use a lock
# because threads will not be adding new keys
results = {provider: None for provider in providers}
for provider in providers:
# skip provider if block_upload is set
if (mgmt_sys[provider].get('template_upload') and
mgmt_sys[provider]['template_upload'].get('block_upload')):
logger.info('Skipping upload on %s due to block_upload', provider)
continue
if kwargs['provider_data']:
if mgmt_sys[provider]['type'] != 'virtualcenter':
continue
username = mgmt_sys[provider]['username']
password = mgmt_sys[provider]['password']
else:
creds = credentials[mgmt_sys[provider]['credentials']]
username = creds['username']
password = creds['password']
host_ip = mgmt_sys[provider]['ipaddress']
hostname = mgmt_sys[provider]['hostname']
client = VMWareSystem(hostname, username, password)
if not net.is_pingable(host_ip):
continue
thread = Thread(target=upload_template,
args=(client, hostname, username, password, provider,
kwargs.get('image_url'), kwargs.get('template_name'),
kwargs['provider_data'], kwargs['stream'], results))
thread.daemon = True
thread_queue.append(thread)
thread.start()
for thread in thread_queue:
thread.join()
failed_providers = [provider for provider, result in results.items() if result is False]
skipped_providers = [provider for provider, result in results.items() if result is None]
passed_providers = [provider for provider, result in results.items() if result]
logger.info("providers skipped: %s", skipped_providers)
logger.info("providers passed: %s", passed_providers)
logger.info("providers failed: %s", failed_providers)
if not passed_providers:
raise Exception("Template upload failed for all providers")
else:
logger.info("Upload passed for at least 1 provider... success!")
def templates_uploaded_on_providers(api, stream, template):
if get_untested_templates(api, stream, template):
print('report will not be generated, proceed with the next untested provider')
sys.exit()
for temp in api.template.get(
limit=1, tested=False, group__name=stream).get('objects', []):
if 'template_rhevm' in images_uploaded(stream):
if not provider_in_the_list(list_provider_keys('rhevm'), temp['providers']):
return False
if 'template_rhos' in images_uploaded(stream):
if not provider_in_the_list(list_provider_keys('openstack'), temp['providers']):
return False
if 'template_vsphere' in images_uploaded(stream):
if not provider_in_the_list(list_provider_keys('virtualcenter'), temp['providers']):
return False
if 'template_scvmm' in images_uploaded(stream):
if not provider_in_the_list(list_provider_keys('scvmm'), temp['providers']):
return False
return True
def azure_cleanup(nic_template, pip_template, days_old):
logger.info('azure_cleanup.py, NICs, PIPs, Disks and Stack Cleanup')
logger.info("Date: {}".format(datetime.now()))
errors = []
for prov_key in list_provider_keys('azure'):
logger.info("----- Provider: '%s' -----", prov_key)
mgmt = get_mgmt(prov_key)
mgmt.logger = logger
for name, scr_id in mgmt.list_subscriptions():
logger.info("Subscription '%s' is chosen", name)
setattr(mgmt, 'subscription_id', scr_id)
for resource_group in mgmt.list_resource_groups():
mgmt.logger.info('Checking "%s" resource group:', resource_group)
# removing stale nics
try:
mgmt.remove_nics_by_search(nic_template, resource_group)
except Exception as e:
logger.exception("NIC cleanup failed")
errors.append(e)
# removing public ips
try:
mgmt.remove_pips_by_search(pip_template, resource_group)
except Exception as e:
logger.exception("Public IP cleanup failed")
errors.append(e)
# removing stale stacks
try:
stack_list = mgmt.list_stack(resource_group=resource_group,
days_old=days_old)
if stack_list:
removed_stacks = []
for stack in stack_list:
if mgmt.is_stack_empty(stack, resource_group=resource_group):
removed_stacks.append(stack)
mgmt.delete_stack(stack, resource_group)
if not removed_stacks:
logger.info("No empty stacks older '%s' days were found", days_old)
except Exception as e:
logger.exception("Removing Stacks failed")
errors.append(e)
try:
mgmt.remove_unused_blobs(resource_group)
except Exception as e:
logger.exception("Removing unused blobs failed")
errors.append(e)
if errors:
logger.error("Hit exceptions during cleanup! See logs.")
return 1
else:
return 0
def run(**kwargs):
try:
thread_queue = []
providers = list_provider_keys("openshift")
if kwargs['provider_data']:
mgmt_sys = providers = kwargs['provider_data'][
'management_systems']
else:
mgmt_sys = cfme_data['management_systems']
for provider in providers:
# skip provider if block_upload is set
if (mgmt_sys[provider].get('template_upload') and
mgmt_sys[provider]['template_upload'].get('block_upload')):
logger.info('Skipping upload on {} due to block_upload'.format(
provider))
continue
if 'podtesting' not in mgmt_sys[provider]['tags']:
continue
if kwargs['provider_data']:
username = mgmt_sys[provider]['username']
password = mgmt_sys[provider]['password']
else:
ssh_creds = credentials[mgmt_sys[provider]['ssh_creds']]
username = ssh_creds['username']
password = ssh_creds['password']
oc_creds = credentials[mgmt_sys[provider]['credentials']]
oc_username = oc_creds['username']
oc_password = oc_creds['password']
host_ip = mgmt_sys[provider]['ipaddress']
hostname = mgmt_sys[provider]['hostname']
upload_parameters = cfme_data['template_upload'][
'template_upload_openshift']
upload_folder = kwargs.get('upload_folder',
upload_parameters['upload_folder'])
if not net.is_pingable(host_ip):
continue
thread = Thread(target=upload_template,
args=(hostname, username, password, provider,
kwargs.get('image_url'),
kwargs.get('template_name'),
kwargs['provider_data'], kwargs['stream'],
upload_folder, oc_username, oc_password))
thread.daemon = True
thread_queue.append(thread)
thread.start()
for thread in thread_queue:
thread.join()
except Exception:
logger.exception('Exception during run method')
return False
def run(**kwargs):
thread_queue = []
providers = list_provider_keys("openstack")
if kwargs['provider_data']:
provider_data = kwargs['provider_data']
mgmt_sys = providers = provider_data['management_systems']
else:
mgmt_sys = cfme_data.management_systems
for provider in providers:
# skip provider if block_upload is set
if (mgmt_sys[provider].get('template_upload')
and mgmt_sys[provider]['template_upload'].get('block_upload')):
logger.info(
'Skipping upload on {} due to block_upload'.format(provider))
continue
if kwargs['provider_data']:
if mgmt_sys[provider]['type'] != 'openstack':
continue
username = mgmt_sys[provider]['username']
password = mgmt_sys[provider]['password']
sshname = mgmt_sys[provider]['sshname']
sshpass = mgmt_sys[provider]['sshpass']
else:
mgmt_sys = cfme_data['management_systems']
rhos_credentials = credentials[mgmt_sys[provider]['credentials']]
default_host_creds = credentials['host_default']
username = rhos_credentials['username']
password = rhos_credentials['password']
sshname = default_host_creds['username']
sshpass = default_host_creds['password']
rhosip = mgmt_sys[provider]['ipaddress']
auth_url = mgmt_sys[provider]['auth_url']
if not net.is_pingable(rhosip):
continue
if not net.net_check(ports.SSH, rhosip):
logger.error("SSH connection to %r:%r failed, port unavailable",
provider, ports.SSH)
continue
thread = Thread(target=upload_template,
args=(rhosip, sshname, sshpass, username, password,
auth_url, provider, kwargs.get('image_url'),
kwargs.get('template_name'),
kwargs['provider_data'], kwargs['stream']))
thread.daemon = True
thread_queue.append(thread)
thread.start()
for thread in thread_queue:
thread.join()
def parse_cmd_line():
parser = argparse.ArgumentParser(argument_default=None)
parser.add_argument(
'--name',
default='test_snapshot_',
help='Starting pettern of snaphsot name '
'e.g. --name test_ delete all snapshot starting with test_')
parser.add_argument(
'--providers',
default=list_provider_keys("openstack"),
nargs='+',
help='List of provider keys e.g. --providers rhos13 rhos12')
args = parser.parse_args()
return args
def run(**kwargs):
try:
thread_queue = []
providers = list_provider_keys("openshift")
if kwargs['provider_data']:
mgmt_sys = providers = kwargs['provider_data']['management_systems']
else:
mgmt_sys = cfme_data['management_systems']
for provider in providers:
# skip provider if block_upload is set
if (mgmt_sys[provider].get('template_upload') and
mgmt_sys[provider]['template_upload'].get('block_upload')):
logger.info('Skipping upload on {} due to block_upload'.format(provider))
continue
if 'podtesting' not in mgmt_sys[provider]['tags']:
continue
if kwargs['provider_data']:
username = mgmt_sys[provider]['username']
password = mgmt_sys[provider]['password']
else:
ssh_creds = credentials[mgmt_sys[provider]['ssh_creds']]
username = ssh_creds['username']
password = ssh_creds['password']
oc_creds = credentials[mgmt_sys[provider]['credentials']]
oc_username = oc_creds['username']
oc_password = oc_creds['password']
host_ip = mgmt_sys[provider]['ipaddress']
hostname = mgmt_sys[provider]['hostname']
upload_parameters = cfme_data['template_upload']['template_upload_openshift']
upload_folder = kwargs.get('upload_folder', upload_parameters['upload_folder'])
if not net.is_pingable(host_ip):
continue
thread = Thread(target=upload_template,
args=(hostname, username, password, provider,
kwargs.get('image_url'), kwargs.get('template_name'),
kwargs['provider_data'], kwargs['stream'], upload_folder,
oc_username, oc_password))
thread.daemon = True
thread_queue.append(thread)
thread.start()
for thread in thread_queue:
thread.join()
except Exception:
logger.exception('Exception during run method')
return False
def main(*providers):
for provider_key in list_provider_keys('openstack'):
print('Checking {}'.format(provider_key))
api = get_mgmt(provider_key).api
try:
fips = api.floating_ips.findall(fixed_ip=None)
except Exception:
print('Unable to get fips for {}:'.format(provider_key))
print(format_exc().splitlines()[-1])
continue
for fip in fips:
print('Deleting {} on {}'.format(fip.ip, provider_key))
fip.delete()
print('{} deleted'.format(fip.ip))
def main(*providers):
for provider_key in list_provider_keys('openstack'):
print('Checking {}'.format(provider_key))
api = get_mgmt(provider_key).api
try:
fips = api.floating_ips.findall(fixed_ip=None)
except Exception:
print('Unable to get fips for {}:'.format(provider_key))
print(format_exc().splitlines()[-1])
continue
for fip in fips:
print('Deleting {} on {}'.format(fip.ip, provider_key))
fip.delete()
print('{} deleted'.format(fip.ip))
def run(**kwargs):
providers = list_provider_keys("scvmm")
# Store result of each provider upload
results = {provider: None for provider in providers}
for provider in providers:
mgmt_sys = cfme_data['management_systems'][provider]
# skip provider if block_upload is set
if (mgmt_sys.get('template_upload')
and mgmt_sys['template_upload'].get('block_upload')):
logger.info('SCVMM:%s skipping due to block_upload', provider)
continue
attempts = 2
for i in range(0, attempts):
try:
results[
provider] = False # assume it has failed... till it has passed
logger.info('SCVMM:%s create template attempt %d/%d', provider,
i + 1, attempts)
create_template(provider, **kwargs)
logger.info('SCVMM:%s template creation done', provider)
results[provider] = True
break
except Exception:
logger.exception('Hit exception creating template on %s',
provider)
continue
failed_providers = [
provider for provider, result in results.items() if result is False
]
skipped_providers = [
provider for provider, result in results.items() if result is None
]
passed_providers = [
provider for provider, result in results.items() if result
]
logger.info("providers skipped: %s", skipped_providers)
logger.info("providers passed: %s", passed_providers)
logger.info("providers failed: %s", failed_providers)
if not passed_providers:
raise Exception("Template upload failed for all providers")
else:
logger.info("Upload passed for at least 1 provider... success!")
def run(**kwargs):
thread_queue = []
providers = list_provider_keys("openstack")
if kwargs['provider_data']:
provider_data = kwargs['provider_data']
mgmt_sys = providers = provider_data['management_systems']
else:
mgmt_sys = cfme_data.management_systems
for provider in providers:
# skip provider if block_upload is set
if (mgmt_sys[provider].get('template_upload') and
mgmt_sys[provider]['template_upload'].get('block_upload')):
logger.info('Skipping upload on {} due to block_upload'.format(provider))
continue
if kwargs['provider_data']:
if mgmt_sys[provider]['type'] != 'openstack':
continue
username = mgmt_sys[provider]['username']
password = mgmt_sys[provider]['password']
sshname = mgmt_sys[provider]['sshname']
sshpass = mgmt_sys[provider]['sshpass']
else:
mgmt_sys = cfme_data['management_systems']
rhos_credentials = credentials[mgmt_sys[provider]['credentials']]
default_host_creds = credentials['host_default']
username = rhos_credentials['username']
password = rhos_credentials['password']
sshname = default_host_creds['username']
sshpass = default_host_creds['password']
rhosip = mgmt_sys[provider]['ipaddress']
auth_url = mgmt_sys[provider]['auth_url']
if not net.is_pingable(rhosip):
continue
if not net.net_check(ports.SSH, rhosip):
logger.error("SSH connection to %r:%r failed, port unavailable", provider, ports.SSH)
continue
thread = Thread(target=upload_template,
args=(rhosip, sshname, sshpass, username, password, auth_url, provider,
kwargs.get('image_url'), kwargs.get('template_name'),
kwargs['provider_data'], kwargs['stream']))
thread.daemon = True
thread_queue.append(thread)
thread.start()
for thread in thread_queue:
thread.join()
def process_tags(provider_keys, tags=None):
"""
Process the tags provided on command line to build a list of provider keys that match
:param tags: list of tags to match against cfme_data
:param provider_keys list of provider_keys to append to
:return: list or provider keys matching tags
"""
# Check for tags first, build list of provider_keys from it
if tags:
all_provider_keys = list_provider_keys()
for key in all_provider_keys:
# need to check tags list against yaml tags list for intersection of a single tag
yaml_tags = cfme_data['management_systems'][key]['tags']
if any(tag in tags for tag in yaml_tags):
print('Matched tag from {} on provider {}:tags:{}'.format(tags, key, yaml_tags))
provider_keys.add(key)
def azure_cleanup(nic_template, pip_template, days_old, output):
with open(output, 'w') as report:
report.write('azure_cleanup.py, NICs, PIPs and Stack Cleanup')
report.write("\nDate: {}\n".format(datetime.now()))
try:
for provider_key in list_provider_keys('azure'):
provider_mgmt = get_mgmt(provider_key)
nic_list = provider_mgmt.list_free_nics(nic_template)
pip_list = provider_mgmt.list_free_pip(pip_template)
stack_list = provider_mgmt.list_stack(days_old=days_old)
report.write("----- Provider: {} -----\n".format(provider_key))
if nic_list:
report.write(
"Removing Nics with the name \'{}\':\n".format(
nic_template))
report.write("\n".join(str(k) for k in nic_list))
report.write("\n")
provider_mgmt.remove_nics_by_search(nic_template)
else:
report.write(
"No \'{}\' NICs were found\n".format(nic_template))
if pip_list:
report.write(
"Removing Public IPs with the name \'{}\':\n".format(
pip_template))
report.write("\n".join(str(k) for k in pip_list))
report.write("\n")
provider_mgmt.remove_pips_by_search(pip_template)
else:
report.write("No \'{}\' Public IPs were found\n".format(
pip_template))
if stack_list:
report.write("Removing empty Stacks:\n")
for stack in stack_list:
if provider_mgmt.is_stack_empty(stack):
provider_mgmt.delete_stack(stack)
report.write(
"Stack {} is empty - Removed\n".format(stack))
else:
report.write(
"No stacks older than \'{}\' days were found\n".format(
days_old))
return 0
except Exception:
report.write("Something bad happened during Azure cleanup\n")
report.write(tb.format_exc())
return 1
def run(**kwargs):
try:
thread_queue = []
providers = list_provider_keys("virtualcenter")
if kwargs['provider_data']:
mgmt_sys = providers = kwargs['provider_data'][
'management_systems']
else:
mgmt_sys = cfme_data.management_systems
for provider in providers:
# skip provider if block_upload is set
if (mgmt_sys[provider].get('template_upload') and
mgmt_sys[provider]['template_upload'].get('block_upload')):
logger.info('Skipping upload on {} due to block_upload'.format(
provider))
continue
if kwargs['provider_data']:
if mgmt_sys[provider]['type'] != 'virtualcenter':
continue
username = mgmt_sys[provider]['username']
password = mgmt_sys[provider]['password']
else:
creds = credentials[mgmt_sys[provider]['credentials']]
username = creds['username']
password = creds['password']
host_ip = mgmt_sys[provider]['ipaddress']
hostname = mgmt_sys[provider]['hostname']
client = VMWareSystem(hostname, username, password)
if not net.is_pingable(host_ip):
continue
thread = Thread(target=upload_template,
args=(client, hostname, username, password,
provider, kwargs.get('image_url'),
kwargs.get('template_name'),
kwargs['provider_data'], kwargs['stream']))
thread.daemon = True
thread_queue.append(thread)
thread.start()
for thread in thread_queue:
thread.join()
except Exception:
logger.exception('Exception during run method')
return False
def run(**kwargs):
try:
thread_queue = []
providers = list_provider_keys("virtualcenter")
if kwargs['provider_data']:
mgmt_sys = providers = kwargs['provider_data']['management_systems']
else:
mgmt_sys = cfme_data.management_systems
for provider in providers:
# skip provider if block_upload is set
if (mgmt_sys[provider].get('template_upload') and
mgmt_sys[provider]['template_upload'].get('block_upload')):
logger.info('Skipping upload on {} due to block_upload'.format(provider))
continue
if kwargs['provider_data']:
if mgmt_sys[provider]['type'] != 'virtualcenter':
continue
username = mgmt_sys[provider]['username']
password = mgmt_sys[provider]['password']
else:
creds = credentials[mgmt_sys[provider]['credentials']]
username = creds['username']
password = creds['password']
host_ip = mgmt_sys[provider]['ipaddress']
hostname = mgmt_sys[provider]['hostname']
client = VMWareSystem(hostname, username, password)
if not net.is_pingable(host_ip):
continue
thread = Thread(target=upload_template,
args=(client, hostname, username, password, provider,
kwargs.get('image_url'), kwargs.get('template_name'),
kwargs['provider_data'], kwargs['stream']))
thread.daemon = True
thread_queue.append(thread)
thread.start()
for thread in thread_queue:
thread.join()
except Exception:
logger.exception('Exception during run method')
return False
def azure_cleanup(nic_template, pip_template, days_old, output):
with open(output, 'w') as report:
report.write('azure_cleanup.py, NICs, PIPs and Stack Cleanup')
report.write("\nDate: {}\n".format(datetime.now()))
try:
for provider_key in list_provider_keys('azure'):
provider_mgmt = get_mgmt(provider_key)
nic_list = provider_mgmt.list_free_nics(nic_template)
report.write("----- Provider: {} -----\n".format(provider_key))
if nic_list:
report.write("Removing Nics with the name \'{}\':\n".format(nic_template))
report.write("\n".join(str(k) for k in nic_list))
report.write("\n")
provider_mgmt.remove_nics_by_search(nic_template)
else:
report.write("No \'{}\' NICs were found\n".format(nic_template))
pip_list = provider_mgmt.list_free_pip(pip_template)
if pip_list:
report.write("Removing Public IPs with the name \'{}\':\n".
format(pip_template))
report.write("\n".join(str(k) for k in pip_list))
report.write("\n")
provider_mgmt.remove_pips_by_search(pip_template)
else:
report.write("No \'{}\' Public IPs were found\n".format(pip_template))
stack_list = provider_mgmt.list_stack(days_old=days_old)
if stack_list:
report.write(
"Removing empty Stacks:\n")
for stack in stack_list:
if provider_mgmt.is_stack_empty(stack):
provider_mgmt.delete_stack(stack)
report.write("Stack {} is empty - Removed\n".format(stack))
else:
report.write("No stacks older than \'{}\' days were found\n".format(
days_old))
return 0
except Exception:
report.write("Something bad happened during Azure cleanup\n")
report.write(tb.format_exc())
return 1
def run(**kwargs):
providers = list_provider_keys("scvmm")
# Store result of each provider upload
results = {provider: None for provider in providers}
for provider in providers:
mgmt_sys = cfme_data['management_systems'][provider]
# skip provider if block_upload is set
if (mgmt_sys.get('template_upload') and
mgmt_sys['template_upload'].get('block_upload')):
logger.info('SCVMM:%s skipping due to block_upload', provider)
continue
attempts = 2
for i in range(0, attempts):
try:
results[provider] = False # assume it has failed... till it has passed
logger.info(
'SCVMM:%s create template attempt %d/%d', provider, i + 1, attempts)
create_template(provider, **kwargs)
logger.info('SCVMM:%s template creation done', provider)
results[provider] = True
break
except Exception:
logger.exception('Hit exception creating template on %s', provider)
continue
failed_providers = [provider for provider, result in results.items() if result is False]
skipped_providers = [provider for provider, result in results.items() if result is None]
passed_providers = [provider for provider, result in results.items() if result]
logger.info("providers skipped: %s", skipped_providers)
logger.info("providers passed: %s", passed_providers)
logger.info("providers failed: %s", failed_providers)
if not passed_providers:
raise Exception("Template upload failed for all providers")
else:
logger.info("Upload passed for at least 1 provider... success!")
def run(**kwargs):
# Setup defaults for the cli tool machine
host = kwargs.get('ssh_host') or \
cfme_data['template_upload']['template_upload_ec2']['aws_cli_tool_client']
user = kwargs.get('ssh_user') or credentials['host_default']['username']
passwd = kwargs.get('ssh_pass') or credentials['host_default']['password']
# Download file once and thread uploading to different gce regions
with make_ssh_client(host, user, passwd) as ssh_client:
file_name, file_path = download_image_file(kwargs.get('image_url'), ssh_client)
thread_queue = []
for provider in list_provider_keys("gce"):
# skip provider if block_upload is set
provider_yaml = cfme_data.management_systems.get(provider)
if (provider_yaml.get('template_upload') and
provider_yaml.template_upload.get('block_upload')):
logger.info('Skipping upload on {} due to block_upload'.format(provider))
continue
template_name = kwargs.get('template_name')
bucket_name = kwargs.get('bucket_name')
stream = kwargs.get('stream')
with make_ssh_client(host, user, passwd) as ssh_client:
thread = Thread(target=upload_template,
args=(provider,
template_name,
stream,
file_name,
file_path,
ssh_client,
bucket_name))
thread.daemon = True
thread_queue.append(thread)
thread.start()
for thread in thread_queue:
thread.join()
def get_orphaned_vmware_files(provider=None):
providers = [provider] if provider else list_provider_keys("virtualcenter")
for provider_key in providers:
# we can add thread here
get_datastores_per_host(provider_key)
def main(trackerbot_url, mark_usable=None):
api = trackerbot.api(trackerbot_url)
thread_q = []
thread_lock = Lock()
template_providers = defaultdict(list)
all_providers = set(list_provider_keys())
unresponsive_providers = set()
# Queue up list_template calls
for provider_key in all_providers:
ipaddress = cfme_data['management_systems'][provider_key].get('ipaddress')
if ipaddress and not net.is_pingable(ipaddress):
continue
thread = Thread(target=get_provider_templates,
args=(provider_key, template_providers, unresponsive_providers, thread_lock))
thread_q.append(thread)
thread.start()
# Join the queued calls
for thread in thread_q:
thread.join()
seen_templates = set()
if mark_usable is None:
usable = {}
else:
usable = {'usable': mark_usable}
existing_provider_templates = [
pt['id']
for pt
in trackerbot.depaginate(api, api.providertemplate.get())['objects']]
# Find some templates and update the API
for template_name, providers in template_providers.items():
template_name = str(template_name)
group_name, datestamp, stream = trackerbot.parse_template(template_name)
# Don't want sprout templates
if group_name in ('sprout', 'rhevm-internal'):
print('Ignoring {} from group {}'.format(template_name, group_name))
continue
seen_templates.add(template_name)
group = trackerbot.Group(group_name, stream=stream)
template = trackerbot.Template(template_name, group, datestamp)
for provider_key in providers:
provider = trackerbot.Provider(provider_key)
if '{}_{}'.format(template_name, provider_key) in existing_provider_templates:
print('Template {} already tracked for provider {}'.format(
template_name, provider_key))
continue
try:
trackerbot.mark_provider_template(api, provider, template, **usable)
print('Added {} template {} on provider {} (datestamp: {})'.format(
group_name, template_name, provider_key, datestamp))
except SlumberHttpBaseException as ex:
print("{}\t{}".format(ex.response.status_code, ex.content))
# Remove provider relationships where they no longer exist, skipping unresponsive providers,
# and providers not known to this environment
for pt in trackerbot.depaginate(api, api.providertemplate.get())['objects']:
provider_key, template_name = pt['provider']['key'], pt['template']['name']
if provider_key not in template_providers[template_name] \
and provider_key not in unresponsive_providers:
if provider_key in all_providers:
print("Cleaning up template {} on {}".format(template_name, provider_key))
trackerbot.delete_provider_template(api, provider_key, template_name)
else:
print("Skipping template cleanup {} on unknown provider {}".format(
template_name, provider_key))
# Remove templates that aren't on any providers anymore
for template in trackerbot.depaginate(api, api.template.get())['objects']:
if not template['providers']:
print("Deleting template {} (no providers)".format(template['name']))
api.template(template['name']).delete()
if not provider_type or cmd_args.provider:
provider_types = PROVIDER_TYPES
elif provider_type in PROVIDER_TYPES:
provider_types = [
provider_type,
]
else:
logger.error('Template upload for %r is not implemented yet.',
provider_type)
sys.exit(1)
thread_queue = []
# create uploader objects for each provider
for provider_type in provider_types:
provider_keys = list_provider_keys(provider_type)
if cmd_args.provider:
provider_keys = [
x for x in cmd_args.provider if x in provider_keys
]
for provider_key in provider_keys:
if provider_key not in list_provider_keys(provider_type):
continue
# pulling class by provider type
provider_template_upload = (
cfme_data.management_systems[provider_key].get(
'template_upload', {}))
uploader = CLASS_MAP[provider_type](
provider_key=provider_key,
output_list.append([provider_key,
vm_name,
status or NULL,
creation or NULL,
str(vm_type) or NULL])
output_queue.put(output_list)
return
if __name__ == "__main__":
args = parse_cmd_line()
# providers as a set when processing tags to ensure unique entries
providers = set(args.provider)
process_tags(providers, args.tag)
providers = providers or list_provider_keys()
queue = Queue() # for MP output
proc_list = [
Process(target=list_vms, args=(provider, queue), name='list_vms:{}'.format(provider))
for provider in providers
]
for proc in proc_list:
proc.start()
for proc in proc_list:
proc.join()
print('Done processing providers, assembling report...')
# Now pull all the results off of the queue
# Stacking the generator this way is equivalent to using list.extend instead of list.append
def run(template_name, image_url, stream, **kwargs):
"""
Download file from image_url, upload it to an S3 bucket and import into ec2
Should handle all ec2 regions and minimize uploading by copying images
:param template_name: string name of the template
:param image_url: string url to download template image
:param stream: string stream name
:param kwargs: other kwargs
:return: none
"""
mgmt_sys = cfme_data['management_systems']
ami_name = template_name
prov_to_upload = []
valid_providers = [
prov_key
for prov_key in list_provider_keys("ec2")
if ('disabled' not in mgmt_sys[prov_key]['tags'] and
not mgmt_sys[prov_key].get('template_upload', {}).get('block_upload', False))
]
if valid_providers:
logger.info("Uploading to following enabled ec2 providers/regions: %r", valid_providers)
else:
logger.info('ERROR: No providers found with ec2 type and no disabled tag')
return
# Look for template name on all ec2 regions, in case we can just copy it
# Also ec2 lets you upload duplicate names, so we'll skip upload if its already there
for prov_key in valid_providers:
ec2 = get_mgmt(provider_key=prov_key)
try:
ami_id = check_for_ami(ec2, ami_name)
except MultipleImagesError:
logger.info('ERROR: Already multiple images with name "%r"', ami_name)
return
if ami_id:
# TODO roll this into a flag that copies it to regions without it
logger.info('EC2 %r: AMI already exists with name "%r"', prov_key, ami_name)
continue
else:
# Need to upload on this region
prov_to_upload.append(prov_key)
# See if we actually need to upload
if not prov_to_upload:
logger.info('DONE: No templates to upload, all regions have the ami: "%r"', ami_name)
return
# download image
logger.info("INFO: Starting image download %r ...", kwargs.get('image_url'))
file_name, file_path = download_image_file(image_url)
logger.info("INFO: Image downloaded %r ...", file_path)
# TODO: thread + copy within amazon for when we have multiple regions enabled
# create ami's in the regions
for prov_key in prov_to_upload:
region = mgmt_sys[prov_key].get('region', prov_key)
bucket_name = mgmt_sys[prov_key].get('upload_bucket_name', 'cfme-template-upload')
logger.info('EC2:%r:%r Starting S3 upload of %r', prov_key, region, file_path)
ec2 = get_mgmt(provider_key=prov_key)
upload_to_s3(ec2=ec2, bucket_name=bucket_name, ami_name=ami_name, file_path=file_path)
create_image(ec2=ec2, ami_name=ami_name, bucket_name=bucket_name)
cleanup_s3(ec2=ec2, bucket_name=bucket_name, ami_name=ami_name)
# Track it
logger.info("EC2:%r:%r Adding template %r to trackerbot for stream %r",
prov_key, region, ami_name, stream)
trackerbot.trackerbot_add_provider_template(stream, prov_key, ami_name)
logger.info('EC2:%r:%r Template %r creation complete', prov_key, region, ami_name)
def main(trackerbot_url, mark_usable=None, selected_provider=None):
api = trackerbot.api(trackerbot_url)
thread_q = []
thread_lock = Lock()
template_providers = defaultdict(list)
all_providers = (set(list_provider_keys())
if not selected_provider
else set(selected_provider))
unresponsive_providers = set()
# Queue up list_template calls
for provider_key in all_providers:
ipaddress = cfme_data.management_systems[provider_key].get('ipaddress')
if ipaddress and not net.is_pingable(ipaddress):
continue
thread = Thread(target=get_provider_templates,
args=(provider_key, template_providers, unresponsive_providers, thread_lock))
thread_q.append(thread)
thread.start()
# Join the queued calls
for thread in thread_q:
thread.join()
seen_templates = set()
if mark_usable is None:
usable = {}
else:
usable = {'usable': mark_usable}
existing_provider_templates = [
pt['id']
for pt
in trackerbot.depaginate(api, api.providertemplate.get())['objects']]
# Find some templates and update the API
for template_name, providers in template_providers.items():
template_name = str(template_name)
template_info = TemplateName.parse_template(template_name)
# it turned out that some providers like ec2 may have templates w/o names.
# this is easy protection against such issue.
if not template_name.strip():
logger.warn('Ignoring template w/o name on provider %s', provider_key)
continue
# Don't want sprout templates
if template_info.group_name in ('sprout', 'rhevm-internal'):
logger.info('Ignoring %s from group %s', template_name, template_info.group_name)
continue
seen_templates.add(template_name)
group = trackerbot.Group(template_info.group_name, stream=template_info.stream)
try:
template = trackerbot.Template(template_name, group, template_info.datestamp)
except ValueError:
logger.exception('Failure parsing provider %s template: %s',
provider_key, template_name)
continue
for provider_key in providers:
provider = trackerbot.Provider(provider_key)
if '{}_{}'.format(template_name, provider_key) in existing_provider_templates:
logger.info('Template %s already tracked for provider %s',
template_name, provider_key)
continue
try:
trackerbot.mark_provider_template(api, provider, template, **usable)
logger.info('Added %s template %s on provider %s (datestamp: %s)',
template_info.group_name,
template_name,
provider_key,
template_info.datestamp)
except SlumberHttpBaseException:
logger.exception('%s: exception marking template %s', provider, template)
# Remove provider relationships where they no longer exist, skipping unresponsive providers,
# and providers not known to this environment
for pt in trackerbot.depaginate(api, api.providertemplate.get())['objects']:
key, template_name = pt['provider']['key'], pt['template']['name']
if key not in template_providers[template_name] and key not in unresponsive_providers:
if key in all_providers:
logger.info("Cleaning up template %s on %s", template_name, key)
trackerbot.delete_provider_template(api, key, template_name)
else:
logger.info("Skipping template cleanup %s on unknown provider %s",
template_name, key)
# Remove templates that aren't on any providers anymore
for template in trackerbot.depaginate(api, api.template.get())['objects']:
if not template['providers'] and template['name'].strip():
logger.info("Deleting template %s (no providers)", template['name'])
api.template(template['name']).delete()
def azure_cleanup(nic_template, pip_template, days_old):
logger.info('azure_cleanup.py, NICs, PIPs, Disks and Stack Cleanup')
logger.info("Date: {}".format(datetime.now()))
try:
for prov_key in list_provider_keys('azure'):
logger.info("----- Provider: {} -----".format(prov_key))
mgmt = get_mgmt(prov_key)
mgmt.logger = logger
for name, scr_id in mgmt.list_subscriptions():
logger.info("subscription {s} is chosen".format(s=name))
setattr(mgmt, 'subscription_id', scr_id)
# removing stale nics
removed_nics = mgmt.remove_nics_by_search(nic_template)
if removed_nics:
logger.info('following nics were removed:')
for nic in removed_nics:
logger.info(nic[0])
else:
logger.info("No '{}' NICs were found".format(nic_template))
# removing public ips
removed_pips = mgmt.remove_pips_by_search(pip_template)
if removed_pips:
logger.info('following pips were removed:')
for pip in removed_pips:
logger.info(pip[0])
else:
logger.info("No '{}' Public IPs were found".format(pip_template))
# removing stale stacks
stack_list = mgmt.list_stack(days_old=days_old)
if stack_list:
logger.info("Removing empty Stacks:")
removed_stacks = []
for stack in stack_list:
if mgmt.is_stack_empty(stack):
removed_stacks.append(stack)
mgmt.delete_stack(stack)
logger.info('following stacks were removed:')
for stack in removed_stacks:
logger.info([stack])
else:
logger.info("No stacks older than '{}' days were found".format(
days_old))
"""
Blob removal section
"""
# TODO: update it later to use different subscriptions and resource groups
logger.info("Removing 'bootdiagnostics-test*' containers")
bootdiag_list = []
for container in mgmt.container_client.list_containers():
if container.name.startswith('bootdiagnostics-test'):
bootdiag_list.append(container.name)
mgmt.container_client.delete_container(
container_name=container.name)
logger.info('following disks were removed:')
for disk in bootdiag_list:
logger.info([disk])
logger.info("Removing unused blobs and disks")
removed_disks = mgmt.remove_unused_blobs()
if len(removed_disks['Managed']) > 0:
logger.info('Managed disks:')
logger.info(removed_disks['Managed'])
if len(removed_disks['Unmanaged']) > 0:
logger.info('Unmanaged blobs:')
logger.info(removed_disks['Unmanaged'])
return 0
except Exception:
logger.info("Something bad happened during Azure cleanup")
logger.info(tb.format_exc())
return 1
def generate_html_report(api, stream, filename, appliance_template):
status = 'PASSED'
number_of_images_before = len(images_uploaded(stream))
if get_untested_templates(api, stream, appliance_template):
print(
'report will not be generated, proceed with the next untested provider'
)
sys.exit()
stream_data = get_latest_tested_template_on_stream(api, stream,
appliance_template)
if len(images_uploaded(stream)) > number_of_images_before:
print(
"new images are uploaded on latest directory, wait for upload on providers"
)
wait_for_templates_on_providers(api, stream, appliance_template)
if appliance_template and appliance_template != stream_data[
'template_name']:
print("the report will be generated only for the latest templates")
sys.exit()
if stream_data and not get_untested_templates(
api, stream_data['group_name'], appliance_template):
print("Found tested template for {}".format(stream))
print("Gathering tested template data for {}".format(stream))
print("Updating the template log")
stream_html = [
stream_data['template_name'], stream_data['passed_on_providers'],
stream_data['failed_on_providers'], stream_data['group_name'],
stream_data['datestamp']
]
if 'html' in filename:
data = template_env.get_template(
'template_tester_report.html').render(upstream=stream_html)
with open(filename, 'w') as report:
report.write(data)
else:
with open(filename, 'a+') as report:
if 'template_rhos' not in images_uploaded(stream):
print(
'\n\nMISSING: Image for OpenStack in latest directory')
report.write(
'\n\nMISSING: Image for OpenStack in latest directory')
elif provider_in_the_list(list_provider_keys('openstack'),
stream_data['passed_on_providers']):
report.write('\n\nPASSED: {}'.format(
images_uploaded(stream)['template_rhos']))
list(
map(
lambda x: report.write('\n{}: Passed'.format(x)),
provider_in_the_list(
list_provider_keys('openstack'),
stream_data['passed_on_providers'])))
elif provider_in_the_list(list_provider_keys('openstack'),
stream_data['failed_on_providers']):
report.write('\n\nFAILED: {}'.format(
images_uploaded(stream)['template_rhos']))
list(
map(
lambda x: report.write('\n{}: Failed'.format(x)),
provider_in_the_list(
list_provider_keys('openstack'),
stream_data['failed_on_providers'])))
else:
print(
'\n\nMISSING: OpenStack template is not available on any '
'rhos providers yet')
report.write(
'\n\nMISSING: OpenStack template is not available on any '
'rhos providers yet')
if 'template_rhevm' not in images_uploaded(stream):
print('\n\nMISSING: Image for RHEVM in latest directory')
report.write(
'\n\nMISSING: Image for RHEVM in latest directory')
elif provider_in_the_list(list_provider_keys('rhevm'),
stream_data['passed_on_providers']):
report.write('\n\nPASSED: {}'.format(
images_uploaded(stream)['template_rhevm']))
list(
map(
lambda x: report.write('\n{}: Passed'.format(x)),
provider_in_the_list(
list_provider_keys('rhevm'),
stream_data['passed_on_providers'])))
elif provider_in_the_list(list_provider_keys('rhevm'),
stream_data['failed_on_providers']):
report.write('\n\nFAILED: {}'.format(
images_uploaded(stream)['template_rhevm']))
list(
map(
lambda x: report.write('\n{}: Failed'.format(x)),
provider_in_the_list(
list_provider_keys('rhevm'),
stream_data['failed_on_providers'])))
else:
print(
'\n\nMISSING: RHEVM template is not available on any '
'rhevm providers yet')
report.write(
'\n\nMISSING: RHEVM template is not available on any '
'rhevm providers yet')
if 'template_vsphere' not in images_uploaded(stream):
print(
'\n\nMISSING: Image for VIRTUALCENTER in latest directory'
)
report.write(
'\n\nMISSING: Image for VIRTUALCENTER in latest directory'
)
elif provider_in_the_list(list_provider_keys('virtualcenter'),
stream_data['passed_on_providers']):
report.write('\n\nPASSED: {}'.format(
images_uploaded(stream)['template_vsphere']))
list(
map(
lambda x: report.write('\n{}: Passed'.format(x)),
provider_in_the_list(
list_provider_keys('virtualcenter'),
stream_data['passed_on_providers'])))
elif provider_in_the_list(list_provider_keys('virtualcenter'),
stream_data['failed_on_providers']):
report.write('\n\nFAILED: {}'.format(
images_uploaded(stream)['template_vsphere']))
list(
map(
lambda x: report.write('\n{}: Failed'.format(x)),
provider_in_the_list(
list_provider_keys('virtualcenter'),
stream_data['failed_on_providers'])))
else:
print(
'\n\nMISSING: VIRTUALCENTER template is not available on any '
'vmware providers yet')
report.write(
'\n\nMISSING: VIRTUALCENTER template is not available on any '
'vmware providers yet')
if 'template_scvmm' not in images_uploaded(stream):
print('\n\nMISSING: Image for SCVMM in latest directory')
report.write(
'\n\nMISSING: Image for SCVMM in latest directory')
elif provider_in_the_list(list_provider_keys('scvmm'),
stream_data['passed_on_providers']):
report.write('\n\nPASSED: {}'.format(
images_uploaded(stream)['template_scvmm']))
list(
map(
lambda x: report.write('\n{}: Passed'.format(x)),
provider_in_the_list(
list_provider_keys('scvmm'),
stream_data['passed_on_providers'])))
elif provider_in_the_list(list_provider_keys('scvmm'),
stream_data['failed_on_providers']):
report.write('\n\nFAILED: {}'.format(
images_uploaded(stream)['template_scvmm']))
list(
map(
lambda x: report.write('\n{}: Failed'.format(x)),
provider_in_the_list(
list_provider_keys('scvmm'),
stream_data['failed_on_providers'])))
else:
print(
'\n\nMISSING: SCVMM template is not available on any '
'scvmm providers yet')
report.write(
'\n\nMISSING: SCVMM template is not available on any '
'scvmm providers yet')
report.seek(0, 0)
lines = report.readlines()
template_missing = filter(lambda x: "MISSING" in x, lines)
template_passed = filter(lambda x: "PASSED" in x, lines)
template_failed = filter(lambda x: "FAILED" in x, lines)
if template_failed:
status = "FAILED"
if template_missing and not (template_passed
or template_failed):
report.close()
sys.exit(
"Template is MISSING....Please verify uploads....")
print("template_tester_results report generated:{}".format(status))
else:
print("No Templates tested on: {}".format(datetime.datetime.now()))
def run(**kwargs):
for provider in list_provider_keys("scvmm"):
kwargs = make_kwargs_scvmm(cfme_data, provider,
kwargs.get('image_url'),
kwargs.get('template_name'))
check_kwargs(**kwargs)
mgmt_sys = cfme_data['management_systems'][provider]
host_fqdn = mgmt_sys['hostname_fqdn']
creds = credentials[mgmt_sys['credentials']]
# For powershell to work, we need to extract the User Name from the Domain
user = creds['username'].split('\\')
if len(user) == 2:
username_powershell = user[1]
else:
username_powershell = user[0]
username_scvmm = creds['domain'] + "\\" + creds['username']
scvmm_args = {
"hostname": mgmt_sys['ipaddress'],
"username": username_powershell,
"password": creds['password'],
"domain": creds['domain'],
"provisioning": mgmt_sys['provisioning']
}
client = SCVMMSystem(**scvmm_args)
url = kwargs.get('image_url')
# Template name equals either user input of we extract the name from the url
new_template_name = kwargs.get('template_name')
if new_template_name is None:
new_template_name = os.path.basename(url)[:-4]
logger.info("SCVMM:%s Make Template out of the VHD %s", provider,
new_template_name)
# use_library is either user input or we use the cfme_data value
library = kwargs.get('library',
mgmt_sys['template_upload'].get('vhds', None))
logger.info("SCVMM:%s Template Library: %s", provider, library)
# The VHD name changed, match the template_name.
new_vhd_name = new_template_name + '.vhd'
network = mgmt_sys['template_upload'].get('network', None)
os_type = mgmt_sys['template_upload'].get('os_type', None)
cores = mgmt_sys['template_upload'].get('cores', None)
ram = mgmt_sys['template_upload'].get('ram', None)
# Uses PowerShell Get-SCVMTemplate to return a list of templates and aborts if exists.
if not client.does_template_exist(new_template_name):
if kwargs.get('upload'):
logger.info(
"SCVMM:%s Uploading VHD image to Library VHD folder.",
provider)
upload_vhd(client, url, library, new_vhd_name)
if kwargs.get('template'):
logger.info("SCVMM:%s Make Template out of the VHD %s",
provider, new_template_name)
make_template(client, host_fqdn, new_template_name, library,
network, os_type, username_scvmm, cores, ram)
try:
wait_for(lambda: client.does_template_exist(new_template_name),
fail_condition=False,
delay=5)
logger.info("SCVMM:%s template %s uploaded success", provider,
new_template_name)
logger.info("SCVMM:%s Add template %s to trackerbot", provider,
new_template_name)
trackerbot.trackerbot_add_provider_template(
kwargs.get('stream'), provider,
kwargs.get('template_name'))
except Exception:
logger.exception(
"SCVMM:%s Exception verifying the template %s", provider,
new_template_name)
else:
logger.info(
"SCVMM: A Template with that name already exists in the SCVMMLibrary"
)
def main(trackerbot_url, mark_usable=None, selected_provider=None):
api = trackerbot.api(trackerbot_url)
thread_q = []
thread_lock = Lock()
template_providers = defaultdict(list)
all_providers = (set(list_provider_keys())
if not selected_provider else set(selected_provider))
unresponsive_providers = set()
# Queue up list_template calls
for provider_key in all_providers:
ipaddress = cfme_data.management_systems[provider_key].get('ipaddress')
if ipaddress and not net.is_pingable(ipaddress):
continue
thread = Thread(target=get_provider_templates,
args=(provider_key, template_providers,
unresponsive_providers, thread_lock))
thread_q.append(thread)
thread.start()
# Join the queued calls
for thread in thread_q:
thread.join()
seen_templates = set()
if mark_usable is None:
usable = {}
else:
usable = {'usable': mark_usable}
existing_provider_templates = [
pt['id'] for pt in trackerbot.depaginate(
api, api.providertemplate.get())['objects']
]
# Find some templates and update the API
for template_name, providers in template_providers.items():
template_name = str(template_name)
template_info = TemplateName.parse_template(template_name)
# Don't want sprout templates
if template_info.group_name in ('sprout', 'rhevm-internal'):
logger.info('Ignoring %s from group %s', template_name,
template_info.group_name)
continue
seen_templates.add(template_name)
group = trackerbot.Group(template_info.group_name,
stream=template_info.stream)
try:
template = trackerbot.Template(template_name, group,
template_info.datestamp)
except ValueError:
logger.exception('Failure parsing provider %s template: %s',
provider_key, template_name)
continue
for provider_key in providers:
provider = trackerbot.Provider(provider_key)
if '{}_{}'.format(template_name,
provider_key) in existing_provider_templates:
logger.info('Template %s already tracked for provider %s',
template_name, provider_key)
continue
try:
trackerbot.mark_provider_template(api, provider, template,
**usable)
logger.info(
'Added %s template %s on provider %s (datestamp: %s)',
template_info.group_name, template_name, provider_key,
template_info.datestamp)
except SlumberHttpBaseException:
logger.exception('%s: exception marking template %s', provider,
template)
# Remove provider relationships where they no longer exist, skipping unresponsive providers,
# and providers not known to this environment
for pt in trackerbot.depaginate(api,
api.providertemplate.get())['objects']:
key, template_name = pt['provider']['key'], pt['template']['name']
if key not in template_providers[
template_name] and key not in unresponsive_providers:
if key in all_providers:
logger.info("Cleaning up template %s on %s", template_name,
key)
trackerbot.delete_provider_template(api, key, template_name)
else:
logger.info(
"Skipping template cleanup %s on unknown provider %s",
template_name, key)
# Remove templates that aren't on any providers anymore
for template in trackerbot.depaginate(api, api.template.get())['objects']:
if not template['providers']:
logger.info("Deleting template %s (no providers)",
template['name'])
api.template(template['name']).delete()
def main(*providers):
for provider_key in providers:
print("Cleaning up {}".format(provider_key))
api = get_mgmt(provider_key).capi
try:
volumes = api.volumes.findall(attachments=[])
except Exception as e:
print("Connect to provider failed:{} {} {}".format(
provider_key,
type(e).__name__, str(e)))
continue
for volume in volumes:
if iso8601.parse_date(volume.created_at) < (
datetime.now(tz=local_tz) - GRACE_TIME):
print("Deleting {}".format(volume.id))
try:
volume.delete()
except Exception as e:
print("Delete failed: {} {}".format(
type(e).__name__, str(e)))
if __name__ == "__main__":
provs = sys.argv[1:]
if provs:
main(*provs)
else:
main(*list_provider_keys("openstack"))
print(template_name)
sys.exit(0)
if not provider_type or cmd_args.provider:
provider_types = PROVIDER_TYPES
elif provider_type in PROVIDER_TYPES:
provider_types = [provider_type, ]
else:
logger.error('Template upload for %r is not implemented yet.', provider_type)
sys.exit(1)
thread_queue = []
# threaded loop
for provider_type in provider_types:
provider_keys = list_provider_keys(provider_type)
if cmd_args.provider:
provider_keys = filter(lambda x: x in provider_keys, cmd_args.provider)
for provider_key in provider_keys:
if provider_key not in list_provider_keys(provider_type):
continue
# pulling class by provider type
provider_template_upload = (cfme_data.management_systems[provider_key]
.get('template_upload', {}))
uploader = CLASS_MAP[provider_type](
provider_key=provider_key,
stream=stream,
template_name=template_name,
image_url=image_url,
def run(**kwargs):
"""Calls all the functions needed to upload new template to RHEVM.
This is called either by template_upload_all script, or by main function.
Args:
**kwargs: Kwargs generated from cfme_data['template_upload']['template_upload_rhevm'].
"""
thread_queue = []
valid_providers = []
providers = list_provider_keys("rhevm")
if kwargs['provider_data']:
mgmt_sys = providers = kwargs['provider_data']['management_systems']
for provider in providers:
if kwargs['provider_data']:
if mgmt_sys[provider]['type'] != 'rhevm':
continue
sshname = mgmt_sys[provider]['sshname']
sshpass = mgmt_sys[provider]['sshpass']
rhevip = mgmt_sys[provider]['ipaddress']
else:
mgmt_sys = cfme_data['management_systems']
ssh_rhevm_creds = mgmt_sys[provider]['ssh_creds']
sshname = credentials[ssh_rhevm_creds]['username']
sshpass = credentials[ssh_rhevm_creds]['password']
rhevip = mgmt_sys[provider]['ipaddress']
if (mgmt_sys[provider].get('template_upload') and
mgmt_sys[provider]['template_upload'].get('block_upload')):
# Providers template_upload section indicates upload should not happen on this provider
continue
logger.info("RHEVM:%r verifying provider's state before template upload", provider)
if not net.is_pingable(rhevip):
continue
elif not is_ovirt_engine_running(rhevip, sshname, sshpass):
logger.info('RHEVM:%r ovirt-engine service not running..', provider)
continue
valid_providers.append(provider)
for provider in valid_providers:
if kwargs['provider_data']:
sshname = mgmt_sys[provider]['sshname']
sshpass = mgmt_sys[provider]['sshpass']
username = mgmt_sys[provider]['username']
password = mgmt_sys[provider]['password']
else:
ssh_rhevm_creds = mgmt_sys[provider]['ssh_creds']
sshname = credentials[ssh_rhevm_creds]['username']
sshpass = credentials[ssh_rhevm_creds]['password']
rhevm_credentials = mgmt_sys[provider]['credentials']
username = credentials[rhevm_credentials]['username']
password = credentials[rhevm_credentials]['password']
rhevip = mgmt_sys[provider]['ipaddress']
thread = Thread(target=upload_template,
args=(rhevip, sshname, sshpass, username, password, provider,
kwargs.get('image_url'), kwargs.get('template_name'),
kwargs['provider_data'], kwargs['stream'],
kwargs['glance']))
thread.daemon = True
thread_queue.append(thread)
thread.start()
for thread in thread_queue:
thread.join()
"""
import sys
from traceback import format_exc
from cfme.utils.providers import list_provider_keys, get_mgmt
def main(*providers):
for provider_key in list_provider_keys('openstack'):
print('Checking {}'.format(provider_key))
api = get_mgmt(provider_key).api
try:
fips = api.floating_ips.findall(fixed_ip=None)
except Exception:
print('Unable to get fips for {}:'.format(provider_key))
print(format_exc().splitlines()[-1])
continue
for fip in fips:
print('Deleting {} on {}'.format(fip.ip, provider_key))
fip.delete()
print('{} deleted'.format(fip.ip))
if __name__ == "__main__":
provs = sys.argv[1:]
if provs:
main(*provs)
else:
main(*list_provider_keys("openstack"))
def get_orphaned_vmware_files(provider=None):
providers = [provider] if provider else list_provider_keys("virtualcenter")
for provider_key in providers:
# we can add thread here
get_datastores_per_host(provider_key)
def cleanup_vms(texts, max_hours=24, providers=None, prompt=True):
"""
Main method for the cleanup process
Generates regex match objects
Checks providers for cleanup boolean in yaml
Checks provider connectivity (using ping)
Threads process_provider_vms to build list of vms to delete
Prompts user to continue with delete
Threads deleting of the vms
Args:
texts (list): List of regex strings to match with
max_hours (int): age limit for deletion
providers (list): List of provider keys to scan and cleanup
prompt (bool): Whether or not to prompt the user before deleting vms
Returns:
int: return code, 0 on success, otherwise raises exception
"""
logger.info(
'Matching VM names against the following case-insensitive strings: %r',
texts)
# Compile regex, strip leading/trailing single quotes from cli arg
matchers = [re.compile(text.strip("'"), re.IGNORECASE) for text in texts]
providers_to_scan = []
for provider_key in providers or list_provider_keys():
# check for cleanup boolean
if not cfme_data['management_systems'][provider_key].get(
'cleanup', False):
logger.info('SKIPPING %r, cleanup set to false or missing in yaml',
provider_key)
continue
logger.info('SCANNING %r', provider_key)
providers_to_scan.append(provider_key)
# scan providers for vms with name matches
# manager = Manager()
text_match_queue = manager.Queue()
scan_fail_queue = manager.Queue()
provider_scan_args = [(provider_key, matchers, text_match_queue,
scan_fail_queue)
for provider_key in providers_to_scan]
pool_manager(scan_provider, provider_scan_args)
text_matched = []
while not text_match_queue.empty():
text_matched.append(text_match_queue.get())
# scan vms for age matches
age_match_queue = manager.Queue()
vm_scan_args = [(provider_key, vm_name, timedelta(hours=int(max_hours)),
age_match_queue, scan_fail_queue)
for provider_key, vm_name in text_matched]
pool_manager(scan_vm, vm_scan_args)
vms_to_delete = []
while not age_match_queue.empty():
vms_to_delete.append(age_match_queue.get())
scan_fail_vms = []
# add the scan failures into deleted vms for reporting sake
while not scan_fail_queue.empty():
scan_fail_vms.append(scan_fail_queue.get())
if vms_to_delete and prompt:
yesno = raw_input('Delete these VMs? [y/N]: ')
if str(yesno).lower() != 'y':
logger.info('Exiting.')
return 0
# initialize this even if we don't have anything to delete, for report consistency
deleted_vms = []
if vms_to_delete:
delete_queue = manager.Queue()
delete_vm_args = [(provider_key, vm_name, age, delete_queue)
for provider_key, vm_name, age in vms_to_delete]
pool_manager(delete_vm, delete_vm_args)
while not delete_queue.empty():
deleted_vms.append(
delete_queue.get()) # Each item is a VmReport tuple
else:
logger.info('No VMs to delete.')
with open(args.outfile, 'a') as report:
report.write('## VM/Instances deleted via:\n'
'## text matches: {}\n'
'## age matches: {}\n'.format(texts, max_hours))
message = tabulate(
sorted(scan_fail_vms + deleted_vms, key=attrgetter('result')),
headers=['Provider', 'Name', 'Age', 'Status Before', 'Delete RC'],
tablefmt='orgtbl')
report.write(message + '\n')
logger.info(message)
return 0
def run(**kwargs):
for provider in list_provider_keys("scvmm"):
# skip provider if block_upload is set
if (cfme_data[provider].get('template_upload') and
cfme_data[provider]['template_upload'].get('block_upload')):
logger.info('Skipping upload on {} due to block_upload'.format(provider))
continue
kwargs = make_kwargs_scvmm(cfme_data, provider,
kwargs.get('image_url'), kwargs.get('template_name'))
check_kwargs(**kwargs)
mgmt_sys = cfme_data['management_systems'][provider]
host_fqdn = mgmt_sys['hostname_fqdn']
creds = credentials[mgmt_sys['credentials']]
# For powershell to work, we need to extract the User Name from the Domain
user = creds['username'].split('\\')
if len(user) == 2:
username_powershell = user[1]
else:
username_powershell = user[0]
username_scvmm = creds['domain'] + "\\" + creds['username']
scvmm_args = {
"hostname": mgmt_sys['ipaddress'],
"username": username_powershell,
"password": creds['password'],
"domain": creds['domain'],
"provisioning": mgmt_sys['provisioning']
}
client = SCVMMSystem(**scvmm_args)
url = kwargs.get('image_url')
# Template name equals either user input of we extract the name from the url
new_template_name = kwargs.get('template_name')
if new_template_name is None:
new_template_name = os.path.basename(url)[:-4]
logger.info("SCVMM:%s Make Template out of the VHD %s", provider, new_template_name)
# use_library is either user input or we use the cfme_data value
library = kwargs.get('library', mgmt_sys['template_upload'].get('vhds', None))
logger.info("SCVMM:%s Template Library: %s", provider, library)
# The VHD name changed, match the template_name.
new_vhd_name = new_template_name + '.vhd'
network = mgmt_sys['template_upload'].get('network', None)
os_type = mgmt_sys['template_upload'].get('os_type', None)
cores = mgmt_sys['template_upload'].get('cores', None)
ram = mgmt_sys['template_upload'].get('ram', None)
# Uses PowerShell Get-SCVMTemplate to return a list of templates and aborts if exists.
if not client.does_template_exist(new_template_name):
if kwargs.get('upload'):
logger.info("SCVMM:%s Uploading VHD image to Library VHD folder.", provider)
upload_vhd(client, url, library, new_vhd_name)
if kwargs.get('template'):
logger.info("SCVMM:%s Make Template out of the VHD %s", provider, new_template_name)
make_template(
client,
host_fqdn,
new_template_name,
library,
network,
os_type,
username_scvmm,
cores,
ram
)
try:
wait_for(lambda: client.does_template_exist(new_template_name),
fail_condition=False, delay=5)
logger.info("SCVMM:%s template %s uploaded success", provider, new_template_name)
logger.info("SCVMM:%s Add template %s to trackerbot", provider, new_template_name)
trackerbot.trackerbot_add_provider_template(kwargs.get('stream'),
provider,
kwargs.get('template_name'))
except Exception:
logger.exception("SCVMM:%s Exception verifying the template %s",
provider, new_template_name)
else:
logger.info("SCVMM: A Template with that name already exists in the SCVMMLibrary")
def generate_html_report(api, stream, filename, appliance_template):
status = 'PASSED'
number_of_images_before = len(images_uploaded(stream))
if get_untested_templates(api, stream, appliance_template):
print('report will not be generated, proceed with the next untested provider')
sys.exit()
stream_data = get_latest_tested_template_on_stream(api, stream, appliance_template)
if len(images_uploaded(stream)) > number_of_images_before:
print("new images are uploaded on latest directory, wait for upload on providers")
wait_for_templates_on_providers(api, stream, appliance_template)
if appliance_template and appliance_template != stream_data['template_name']:
print("the report will be generated only for the latest templates")
sys.exit()
if stream_data and not get_untested_templates(api, stream_data['group_name'],
appliance_template):
print("Found tested template for {}".format(stream))
print("Gathering tested template data for {}".format(stream))
print("Updating the template log")
stream_html = [stream_data['template_name'], stream_data['passed_on_providers'],
stream_data['failed_on_providers'], stream_data['group_name'],
stream_data['datestamp']]
if 'html' in filename:
data = template_env.get_template('template_tester_report.html').render(
upstream=stream_html)
with open(filename, 'w') as report:
report.write(data)
else:
with open(filename, 'a+') as report:
if 'template_rhos' not in images_uploaded(stream):
print('\n\nMISSING: Image for OpenStack in latest directory')
report.write('\n\nMISSING: Image for OpenStack in latest directory')
elif provider_in_the_list(list_provider_keys('openstack'),
stream_data['passed_on_providers']):
report.write('\n\nPASSED: {}'.format(images_uploaded(stream)['template_rhos']))
map(lambda (x): report.write('\n{}: Passed'.format(x)), provider_in_the_list(
list_provider_keys('openstack'), stream_data['passed_on_providers']))
elif provider_in_the_list(list_provider_keys('openstack'),
stream_data['failed_on_providers']):
report.write('\n\nFAILED: {}'.format(images_uploaded(stream)['template_rhos']))
map(lambda (x): report.write('\n{}: Failed'.format(x)),
provider_in_the_list(list_provider_keys('openstack'),
stream_data['failed_on_providers']))
else:
print('\n\nMISSING: OpenStack template is not available on any '
'rhos providers yet')
report.write('\n\nMISSING: OpenStack template is not available on any '
'rhos providers yet')
if 'template_rhevm' not in images_uploaded(stream):
print('\n\nMISSING: Image for RHEVM in latest directory')
report.write('\n\nMISSING: Image for RHEVM in latest directory')
elif provider_in_the_list(list_provider_keys('rhevm'),
stream_data['passed_on_providers']):
report.write('\n\nPASSED: {}'.format(
images_uploaded(stream)['template_rhevm']))
map(lambda(x): report.write('\n{}: Passed'.format(x)), provider_in_the_list(
list_provider_keys('rhevm'), stream_data['passed_on_providers']))
elif provider_in_the_list(list_provider_keys('rhevm'),
stream_data['failed_on_providers']):
report.write('\n\nFAILED: {}'.format(
images_uploaded(stream)['template_rhevm']))
map(lambda(x): report.write('\n{}: Failed'.format(x)),
provider_in_the_list(list_provider_keys('rhevm'),
stream_data['failed_on_providers']))
else:
print('\n\nMISSING: RHEVM template is not available on any '
'rhevm providers yet')
report.write('\n\nMISSING: RHEVM template is not available on any '
'rhevm providers yet')
if 'template_vsphere' not in images_uploaded(stream):
print('\n\nMISSING: Image for VIRTUALCENTER in latest directory')
report.write('\n\nMISSING: Image for VIRTUALCENTER in latest directory')
elif provider_in_the_list(list_provider_keys('virtualcenter'),
stream_data['passed_on_providers']):
report.write('\n\nPASSED: {}'.format(
images_uploaded(stream)['template_vsphere']))
map(lambda (x): report.write('\n{}: Passed'.format(x)), provider_in_the_list(
list_provider_keys('virtualcenter'), stream_data['passed_on_providers']))
elif provider_in_the_list(list_provider_keys('virtualcenter'),
stream_data['failed_on_providers']):
report.write('\n\nFAILED: {}'.format(
images_uploaded(stream)['template_vsphere']))
map(lambda (x): report.write('\n{}: Failed'.format(x)),
provider_in_the_list(list_provider_keys('virtualcenter'),
stream_data['failed_on_providers']))
else:
print('\n\nMISSING: VIRTUALCENTER template is not available on any '
'vmware providers yet')
report.write('\n\nMISSING: VIRTUALCENTER template is not available on any '
'vmware providers yet')
if 'template_scvmm' not in images_uploaded(stream):
print('\n\nMISSING: Image for SCVMM in latest directory')
report.write('\n\nMISSING: Image for SCVMM in latest directory')
elif provider_in_the_list(list_provider_keys('scvmm'),
stream_data['passed_on_providers']):
report.write('\n\nPASSED: {}'.format(
images_uploaded(stream)['template_scvmm']))
map(lambda (x): report.write('\n{}: Passed'.format(x)), provider_in_the_list(
list_provider_keys('scvmm'), stream_data['passed_on_providers']))
elif provider_in_the_list(list_provider_keys('scvmm'),
stream_data['failed_on_providers']):
report.write('\n\nFAILED: {}'.format(
images_uploaded(stream)['template_scvmm']))
map(lambda (x): report.write('\n{}: Failed'.format(x)),
provider_in_the_list(list_provider_keys('scvmm'),
stream_data['failed_on_providers']))
else:
print('\n\nMISSING: SCVMM template is not available on any '
'scvmm providers yet')
report.write('\n\nMISSING: SCVMM template is not available on any '
'scvmm providers yet')
report.seek(0, 0)
lines = report.readlines()
template_missing = filter(lambda (x): "MISSING" in x, lines)
template_passed = filter(lambda (x): "PASSED" in x, lines)
template_failed = filter(lambda (x): "FAILED" in x, lines)
if template_failed:
status = "FAILED"
if template_missing and not (template_passed or template_failed):
report.close()
sys.exit("Template is MISSING....Please verify uploads....")
print("template_tester_results report generated:{}".format(status))
else:
print("No Templates tested on: {}".format(datetime.datetime.now()))
def run(**kwargs):
"""Calls all the functions needed to upload new template to RHEVM.
This is called either by template_upload_all script, or by main function.
Args:
**kwargs: Kwargs generated from cfme_data['template_upload']['template_upload_rhevm'].
"""
thread_queue = []
valid_providers = []
providers = list_provider_keys("rhevm")
if kwargs.get('provider_data'):
mgmt_sys = providers = kwargs['provider_data']['management_systems']
for provider in providers:
if kwargs.get('provider_data'):
if mgmt_sys[provider]['type'] != 'rhevm':
continue
sshname = mgmt_sys[provider]['sshname']
sshpass = mgmt_sys[provider]['sshpass']
rhevip = mgmt_sys[provider]['ipaddress']
else:
mgmt_sys = cfme_data['management_systems']
ssh_rhevm_creds = mgmt_sys[provider]['ssh_creds']
sshname = credentials[ssh_rhevm_creds]['username']
sshpass = credentials[ssh_rhevm_creds]['password']
rhevip = mgmt_sys[provider]['ipaddress']
if (mgmt_sys[provider].get('template_upload')
and mgmt_sys[provider]['template_upload'].get('block_upload')):
# Providers template_upload section indicates upload should not happen on this provider
continue
logger.info(
"RHEVM:%r verifying provider's state before template upload",
provider)
if not net.is_pingable(rhevip):
continue
elif not is_ovirt_engine_running(rhevip, sshname, sshpass):
logger.info('RHEVM:%r ovirt-engine service not running..',
provider)
continue
valid_providers.append(provider)
for provider in valid_providers:
if kwargs.get('provider_data'):
sshname = mgmt_sys[provider]['sshname']
sshpass = mgmt_sys[provider]['sshpass']
username = mgmt_sys[provider]['username']
password = mgmt_sys[provider]['password']
else:
ssh_rhevm_creds = mgmt_sys[provider]['ssh_creds']
sshname = credentials[ssh_rhevm_creds]['username']
sshpass = credentials[ssh_rhevm_creds]['password']
rhevm_credentials = mgmt_sys[provider]['credentials']
username = credentials[rhevm_credentials]['username']
password = credentials[rhevm_credentials]['password']
rhevip = mgmt_sys[provider]['ipaddress']
thread = Thread(target=upload_template,
args=(rhevip, sshname, sshpass, username, password,
provider, kwargs.get('image_url'),
kwargs.get('template_name'),
kwargs.get('provider_data'), kwargs['stream']))
thread.daemon = True
thread_queue.append(thread)
thread.start()
for thread in thread_queue:
thread.join()
def azure_cleanup(nic_template, pip_template, days_old, output):
with open(output, 'w') as report:
report.write('azure_cleanup.py, NICs, PIPs, Disks and Stack Cleanup')
report.write("\nDate: {}\n".format(datetime.now()))
try:
for provider_key in list_provider_keys('azure'):
provider_mgmt = get_mgmt(provider_key)
nic_list = provider_mgmt.list_free_nics(nic_template)
report.write("----- Provider: {} -----\n".format(provider_key))
if nic_list:
report.write(
"\nRemoving Nics with the name \'{}\':\n".format(
nic_template))
report.write(
tabulate(tabular_data=[[nic] for nic in nic_list],
headers=["Name"],
tablefmt='orgtbl'))
provider_mgmt.remove_nics_by_search(nic_template)
else:
report.write(
"No \'{}\' NICs were found\n".format(nic_template))
pip_list = provider_mgmt.list_free_pip(pip_template)
if pip_list:
report.write(
"\nRemoving Public IPs with the name \'{}\':\n".format(
pip_template))
report.write(
tabulate(tabular_data=[[pip] for pip in pip_list],
headers=["Name"],
tablefmt='orgtbl'))
provider_mgmt.remove_pips_by_search(pip_template)
else:
report.write("No \'{}\' Public IPs were found\n".format(
pip_template))
stack_list = provider_mgmt.list_stack(days_old=days_old)
if stack_list:
report.write("\nRemoving empty Stacks:\n")
removed_stacks = []
for stack in stack_list:
if provider_mgmt.is_stack_empty(stack):
removed_stacks.append(stack)
provider_mgmt.delete_stack(stack)
report.write(
tabulate(tabular_data=[[st] for st in removed_stacks],
headers=["Name"],
tablefmt='orgtbl')
) if len(removed_stacks) > 0 else None
else:
report.write(
"\nNo stacks older than \'{}\' days were found\n".
format(days_old))
"""
Blob removal section
"""
report.write("\nRemoving 'bootdiagnostics-test*' containers\n")
bootdiag_list = []
for container in provider_mgmt.container_client.list_containers(
):
if container.name.startswith('bootdiagnostics-test'):
bootdiag_list.append(container.name)
provider_mgmt.container_client.delete_container(
container_name=container.name)
report.write(
tabulate(tabular_data=[[disk] for disk in bootdiag_list],
headers=["Name"],
tablefmt='orgtbl'))
report.write("\nRemoving unused blobs and disks\n")
removed_disks = provider_mgmt.remove_unused_blobs()
if len(removed_disks['Managed']) > 0:
report.write('Managed disks:\n')
report.write(
tabulate(tabular_data=removed_disks['Managed'],
headers="keys",
tablefmt='orgtbl'))
if len(removed_disks['Unmanaged']) > 0:
report.write('\nUnmanaged blobs:\n')
report.write(
tabulate(tabular_data=removed_disks['Unmanaged'],
headers="keys",
tablefmt='orgtbl'))
return 0
except Exception:
report.write("Something bad happened during Azure cleanup\n")
report.write(tb.format_exc())
return 1
