def setup_one_or_skip(request, filters=None, use_global_filters=True):
""" Sets up one of matching providers or skips the test
Args:
filters: List of :py:class:`ProviderFilter` or None
request: Needed for logging a potential skip correctly in artifactor
use_global_filters: Will apply global filters as well if `True`, will not otherwise
"""
filters = filters or []
providers = list_providers(filters=filters, use_global_filters=use_global_filters)
# All providers filtered out?
if not providers:
global_providers = list_providers(filters=None, use_global_filters=use_global_filters)
if not global_providers:
# This can also mean that there simply are no providers in the yamls!
pytest.skip("No provider matching global filters found")
else:
pytest.skip("No provider matching test-specific filters found")
# Are all providers marked as problematic?
if _problematic_providers.issuperset(providers):
skip_msg = "All providers marked as problematic: {}".format([p.key for p in providers])
_artifactor_skip_providers(request, providers, skip_msg)
# If there is a provider already set up matching the user's requirements, reuse it
for provider in providers:
if provider.exists:
return provider
# If we have more than one provider, we create two separate groups of providers, preferred
# and not preferred, that we shuffle separately and then join together
if len(providers) > 1:
only_preferred_filter = ProviderFilter(required_fields=[("do_not_prefer", True)],
inverted=True)
preferred_providers = list_providers(
filters=filters + [only_preferred_filter], use_global_filters=use_global_filters)
not_preferred_providers = [p for p in providers if p not in preferred_providers]
random.shuffle(preferred_providers)
random.shuffle(not_preferred_providers)
providers = preferred_providers + not_preferred_providers
# Try to set up one of matching providers
non_existing = [prov for prov in providers if not prov.exists]
for provider in non_existing:
if _setup_provider_verbose(request, provider):
return provider
skip_msg = "Failed to set up any matching providers: {}", [p.key for p in providers]
_artifactor_skip_providers(request, non_existing, skip_msg)
def setup_one_or_skip(request, filters=None, use_global_filters=True):
""" Sets up one of matching providers or skips the test
Args:
filters: List of :py:class:`ProviderFilter` or None
request: Needed for logging a potential skip correctly in artifactor
use_global_filters: Will apply global filters as well if `True`, will not otherwise
"""
filters = filters or []
providers = list_providers(filters=filters, use_global_filters=use_global_filters)
# All providers filtered out?
if not providers:
global_providers = list_providers(filters=None, use_global_filters=use_global_filters)
if not global_providers:
# This can also mean that there simply are no providers in the yamls!
pytest.skip("No provider matching global filters found")
else:
pytest.skip("No provider matching test-specific filters found")
# Are all providers marked as problematic?
if _problematic_providers.issuperset(providers):
skip_msg = "All providers marked as problematic: {}".format([p.key for p in providers])
_artifactor_skip_providers(request, providers, skip_msg)
# If there is a provider already set up matching the user's requirements, reuse it
for provider in providers:
if provider.exists:
return provider
# If we have more than one provider, we create two separate groups of providers, preferred
# and not preferred, that we shuffle separately and then join together
if len(providers) > 1:
only_preferred_filter = ProviderFilter(required_fields=[("do_not_prefer", True)],
inverted=True)
preferred_providers = list_providers(
filters=filters + [only_preferred_filter], use_global_filters=use_global_filters)
not_preferred_providers = [p for p in providers if p not in preferred_providers]
random.shuffle(preferred_providers)
random.shuffle(not_preferred_providers)
providers = preferred_providers + not_preferred_providers
# Try to set up one of matching providers
non_existing = [prov for prov in providers if not prov.exists]
for provider in non_existing:
if _setup_provider_verbose(request, provider):
return provider
skip_msg = "Failed to set up any matching providers: {}", [p.key for p in providers]
_artifactor_skip_providers(request, non_existing, skip_msg)
def run(**kwargs):
thread_queue = []
for provider in list_providers("openstack"):
mgmt_sys = cfme_data['management_systems'][provider]
rhos_credentials = credentials[mgmt_sys['credentials']]
default_host_creds = credentials['host_default']
username = rhos_credentials['username']
password = rhos_credentials['password']
auth_url = mgmt_sys['auth_url']
rhosip = mgmt_sys['ipaddress']
sshname = default_host_creds['username']
sshpass = default_host_creds['password']
if not net.is_pingable(rhosip):
continue
if not net.net_check(ports.SSH, rhosip):
print("SSH connection to {}:{} failed, port unavailable".format(
provider, ports.SSH))
continue
thread = Thread(target=upload_template,
args=(rhosip, sshname, sshpass, username, password, auth_url, provider,
kwargs.get('image_url'), kwargs.get('template_name')))
thread.daemon = True
thread_queue.append(thread)
thread.start()
for thread in thread_queue:
thread.join()
def run(**kwargs):
try:
thread_queue = []
for provider in list_providers("virtualcenter"):
mgmt_sys = cfme_data['management_systems'][provider]
creds = credentials[mgmt_sys['credentials']]
hostname = mgmt_sys['hostname']
username = creds['username']
password = creds['password']
default_name = credentials['host_default']['username']
default_password = credentials['host_default']['password']
host_ip = mgmt_sys['ipaddress']
client = VMWareSystem(hostname, username, password)
if not net.is_pingable(host_ip):
continue
thread = Thread(target=upload_template,
args=(client, hostname, username, password, provider,
kwargs.get('image_url'), kwargs.get('template_name'),
default_name, default_password))
thread.daemon = True
thread_queue.append(thread)
thread.start()
for thread in thread_queue:
thread.join()
except Exception as e:
print(e)
return False
def test_containers_overview_data_integrity():
"""Test data integrity of status boxes in containers dashboard.
Steps:
* Go to Containers / Overview
* All cells should contain the correct relevant information
# of nodes
# of providers
# ...
"""
navigate_to(ContainersOverview, 'All')
# We should wait ~2 seconds for the StatusBox population
# (until we find a better solution)
time.sleep(2)
statusbox_values = {data_set.object: int(StatusBox(data_set.name).value())
for data_set in DATA_SETS}
api_values = get_api_object_counts(
map(lambda name: ContainersProvider(name, key=name),
list_providers('openshift'))
)
results = {}
for cls in DATA_SETS:
results[cls.object] = api_values[cls.object] == statusbox_values[cls.object]
if not all(results.values()):
pytest.fail('There is a mismatch between API and UI values:\n{}'.format(
'\n'.join(['{}: {} (API) != {} (UI)'.format(
obj.__name__, api_values[obj], statusbox_values[obj])
for obj, is_pass in results.items() if not is_pass])))
def run(**kwargs):
"""Calls all the functions needed to upload new template to RHEVM.
This is called either by template_upload_all script, or by main function.
Args:
**kwargs: Kwargs generated from cfme_data['template_upload']['template_upload_rhevm'].
"""
thread_queue = []
valid_providers = []
providers = list_providers("rhevm")
if kwargs['provider_data']:
mgmt_sys = providers = kwargs['provider_data']['management_systems']
for provider in providers:
if kwargs['provider_data']:
if mgmt_sys[provider]['type'] != 'rhevm':
continue
sshname = mgmt_sys[provider]['sshname']
sshpass = mgmt_sys[provider]['sshpass']
rhevip = mgmt_sys[provider]['ipaddress']
else:
mgmt_sys = cfme_data['management_systems']
ssh_rhevm_creds = mgmt_sys[provider]['ssh_creds']
sshname = credentials[ssh_rhevm_creds]['username']
sshpass = credentials[ssh_rhevm_creds]['password']
rhevip = mgmt_sys[provider]['ipaddress']
print("RHEVM:{} verifying provider's state before template upload".format(provider))
if not net.is_pingable(rhevip):
continue
elif not is_ovirt_engine_running(rhevip, sshname, sshpass):
print('RHEVM:{} ovirt-engine service not running..'.format(provider))
continue
valid_providers.append(provider)
for provider in valid_providers:
if kwargs['provider_data']:
sshname = mgmt_sys[provider]['sshname']
sshpass = mgmt_sys[provider]['sshpass']
username = mgmt_sys[provider]['username']
password = mgmt_sys[provider]['password']
else:
ssh_rhevm_creds = mgmt_sys[provider]['ssh_creds']
sshname = credentials[ssh_rhevm_creds]['username']
sshpass = credentials[ssh_rhevm_creds]['password']
rhevm_credentials = mgmt_sys[provider]['credentials']
username = credentials[rhevm_credentials]['username']
password = credentials[rhevm_credentials]['password']
rhevip = mgmt_sys[provider]['ipaddress']
thread = Thread(target=upload_template,
args=(rhevip, sshname, sshpass, username, password, provider,
kwargs.get('image_url'), kwargs.get('template_name'),
kwargs['provider_data']))
thread.daemon = True
thread_queue.append(thread)
thread.start()
for thread in thread_queue:
thread.join()
def deployments_in_mgmt(cls, provider=None, server=None):
if provider is None:
deployments = []
for _provider in list_providers('hawkular'):
deployments.extend(cls._deployments_in_mgmt(get_crud(_provider), server))
return deployments
else:
return cls._deployments_in_mgmt(provider, server)
def messagings_in_mgmt(cls, provider=None, server=None):
if provider is None:
messagings = []
for _provider in list_providers('hawkular'):
messagings.extend(cls._messagings_in_mgmt(get_crud(_provider), server))
return messagings
else:
return cls._messagings_in_mgmt(provider, server)
def datasources_in_mgmt(cls, provider=None, server=None):
if provider is None:
datasources = []
for _provider in list_providers('hawkular'):
datasources.extend(cls._datasources_in_mgmt(get_crud(_provider), server))
return datasources
else:
return cls._datasources_in_mgmt(provider, server)
def deployments_in_mgmt(cls, provider=None, server=None):
if provider is None:
deployments = []
for _provider in list_providers('hawkular'):
deployments.extend(cls._deployments_in_mgmt(get_crud(_provider), server))
return deployments
else:
return cls._deployments_in_mgmt(provider, server)
def servers_in_mgmt(cls, provider=None, server_group=None):
if provider is None:
servers = []
for _provider in list_providers('hawkular'):
servers.extend(cls._servers_in_mgmt(get_crud(_provider), server_group))
return servers
else:
return cls._servers_in_mgmt(provider, server_group)
def datasources_in_mgmt(cls, provider=None, server=None):
if provider is None:
datasources = []
for _provider in list_providers('hawkular'):
datasources.extend(
cls._datasources_in_mgmt(get_crud(_provider), server))
return datasources
else:
return cls._datasources_in_mgmt(provider, server)
def templates_uploaded_on_providers(api, stream, template):
if get_untested_templates(api, stream, template):
print('report will not be generated, proceed with the next untested provider')
sys.exit()
for temp in api.template.get(
limit=1, tested=False, group__name=stream).get('objects', []):
if 'template_rhevm' in images_uploaded(stream):
if not provider_in_the_list(list_providers('rhevm'), temp['providers']):
return False
if 'template_rhos' in images_uploaded(stream):
if not provider_in_the_list(list_providers('openstack'), temp['providers']):
return False
if 'template_vsphere' in images_uploaded(stream):
if not provider_in_the_list(list_providers('virtualcenter'), temp['providers']):
return False
if 'template_scvmm' in images_uploaded(stream):
if not provider_in_the_list(list_providers('scvmm'), temp['providers']):
return False
return True
def cleanup_vms(texts, max_hours=24, providers=None, prompt=True):
providers = providers or list_providers()
providers_data = cfme_data.get("management_systems", {})
delta = datetime.timedelta(hours=int(max_hours))
vms_to_delete = defaultdict(set)
thread_queue = []
# precompile regexes
matchers = [re.compile(text, re.IGNORECASE) for text in texts]
for provider_key in providers:
ipaddress = cfme_data['management_systems'][provider_key].get('ipaddress', None)
if ipaddress and not net.is_pingable(ipaddress):
continue
provider_type = providers_data[provider_key].get('type', None)
thread = Thread(target=process_provider_vms,
args=(provider_key, provider_type, matchers, delta, vms_to_delete))
# Mark as daemon thread for easy-mode KeyboardInterrupt handling
thread.daemon = True
thread_queue.append(thread)
thread.start()
# Join the queued calls
for thread in thread_queue:
thread.join()
for provider_key, vm_set in vms_to_delete.items():
print('{}:'.format(provider_key))
for vm_name, vm_delta in vm_set:
days, hours = vm_delta.days, vm_delta.seconds / 3600
print(' {} is {} days, {} hours old'.format(vm_name, days, hours))
if vms_to_delete and prompt:
yesno = raw_input('Delete these VMs? [y/N]: ')
if str(yesno).lower() != 'y':
print('Exiting.')
return 0
if not vms_to_delete:
print('No VMs to delete.')
thread_queue = []
for provider_key, vm_set in vms_to_delete.items():
thread = Thread(target=delete_provider_vms,
args=(provider_key, [name for name, t_delta in vm_set]))
# Mark as daemon thread for easy-mode KeyboardInterrupt handling
thread.daemon = True
thread_queue.append(thread)
thread.start()
for thread in thread_queue:
thread.join()
print("Deleting finished")
def ec2cleanup(max_hours, exclude_instances, exclude_volumes, exclude_eips):
for provider in list_providers("ec2"):
ec2provider = get_mgmt(provider)
logger.info("\n" + provider + ":\n")
logger.info("Deleted instances:")
delete_old_instances(
ec2provider=ec2provider, date=datetime.now(), maxhours=max_hours, excluded_instances=exclude_instances
)
sleep(120)
logger.info("\nReleased addresses:")
delete_disassociated_addresses(ec2provider=ec2provider, excluded_ips=exclude_eips)
logger.info("\nDeleted volumes:")
delete_unattached_volumes(ec2provider=ec2provider, excluded_volumes=exclude_volumes)
def ec2cleanup(texts, max_hours, exclude_instances, exclude_volumes, exclude_eips, output):
for provider in list_providers('ec2'):
ec2provider = get_mgmt(provider)
logger.info("\n" + provider + ":\n")
logger.info("Deleted instances:")
delete_old_instances(texts=texts, ec2provider=ec2provider, provider_key=provider,
date=datetime.datetime.now(), maxhours=max_hours,
excluded_instances=exclude_instances, output=output)
time.sleep(120)
logger.info("\nReleased addresses:")
delete_disassociated_addresses(ec2provider=ec2provider, excluded_eips=exclude_eips)
logger.info("\nDeleted volumes:")
delete_unattached_volumes(ec2provider=ec2provider, excluded_volumes=exclude_volumes)
def cleanup_vms(texts, max_hours=24, providers=None, prompt=True):
providers = providers or list_providers()
providers_data = cfme_data.get("management_systems", {})
delta = datetime.timedelta(hours=int(max_hours))
vms_to_delete = defaultdict(set)
thread_queue = []
# precompile regexes
matchers = [re.compile(text) for text in texts]
for provider_key in providers:
provider_type = providers_data[provider_key].get('type', None)
thread = Thread(target=process_provider_vms,
args=(provider_key, provider_type, matchers, delta, vms_to_delete))
# Mark as daemon thread for easy-mode KeyboardInterrupt handling
thread.daemon = True
thread_queue.append(thread)
thread.start()
# Join the queued calls
for thread in thread_queue:
thread.join()
for provider_key, vm_set in vms_to_delete.items():
print('{}:'.format(provider_key))
for vm_name, vm_delta in vm_set:
days, hours = vm_delta.days, vm_delta.seconds / 3600
print(' {} is {} days, {} hours old'.format(vm_name, days, hours))
if vms_to_delete and prompt:
yesno = raw_input('Delete these VMs? [y/N]: ')
if str(yesno).lower() != 'y':
print('Exiting.')
return 0
if not vms_to_delete:
print('No VMs to delete.')
thread_queue = []
for provider_key, vm_set in vms_to_delete.items():
thread = Thread(target=delete_provider_vms,
args=(provider_key, [name for name, t_delta in vm_set]))
# Mark as daemon thread for easy-mode KeyboardInterrupt handling
thread.daemon = True
thread_queue.append(thread)
thread.start()
for thread in thread_queue:
thread.join()
print("Deleting finished")
def main(*providers):
for provider_key in list_providers('openstack'):
print('Checking {}'.format(provider_key))
api = get_mgmt(provider_key).api
try:
fips = api.floating_ips.findall(fixed_ip=None)
except Exception:
print('Unable to get fips for {}:'.format(provider_key))
print(format_exc().splitlines()[-1])
continue
for fip in fips:
print('Deleting {} on {}'.format(fip.ip, provider_key))
fip.delete()
print('{} deleted'.format(fip.ip))
def run(**kwargs):
"""Calls all the functions needed to import new template to EC2.
This is called either by template_upload_all script, or by main function.
Args:
**kwargs: Kwargs are passed by template_upload_all.
"""
mgmt_sys = cfme_data['management_systems']
for provider in list_providers('ec2'):
ssh_rhevm_creds = mgmt_sys[provider]['credentials']
username = credentials[ssh_rhevm_creds]['username']
password = credentials[ssh_rhevm_creds]['password']
upload_bucket_name = mgmt_sys[provider]['upload_bucket_name']
upload_template(provider, username, password, upload_bucket_name, kwargs.get(
'image_url'), kwargs.get('template_name'))
def pytest_configure(config):
""" Filters the list of providers as part of pytest configuration
Note:
Additional filter is added to the global_filters dict of active filters here.
"""
cmd_filter = config.getvalueorskip('use_provider')
if not cmd_filter:
cmd_filter = ["default"]
new_filter = ProviderFilter(keys=cmd_filter, required_tags=cmd_filter, conjunctive=False)
global_filters['use_provider'] = new_filter
logger.debug('Filtering providers with {}, leaves {}'.format(
cmd_filter, [prov.key for prov in list_providers()]))
def clear_providers_by_class(prov_class, validate=True):
""" Removes all providers that are an instance of given class or one of it's subclasses
"""
from utils.providers import ProviderFilter, list_providers
pf = ProviderFilter(classes=[prov_class])
provs = list_providers(filters=[pf], use_global_filters=False)
# First, delete all
deleted_provs = []
for prov in provs:
existed = prov.delete_if_exists(cancel=False)
if existed:
deleted_provs.append(prov)
# Then, check that all were deleted
if validate:
for prov in deleted_provs:
prov.wait_for_delete()
def _setup_provider_verbose(request, provider):
try:
if request.config.option.provider_limit > 0:
# TODO: Use appliance.
existing_providers = [
p for p in list_providers(use_global_filters=False)
if p.exists and p.key != provider.key
]
random.shuffle(existing_providers)
maximum_current_providers = request.config.option.provider_limit - 1
if len(existing_providers) > maximum_current_providers:
providers_to_remove = existing_providers[
maximum_current_providers:]
store.terminalreporter.write_line(
'Removing extra providers: {}'.format(', '.join(
[p.key for p in providers_to_remove])))
for p in providers_to_remove:
logger.info('removing provider %r', p.key)
p.delete(cancel=False)
# Decoupled wait for better performance
for p in providers_to_remove:
logger.info('waiting for provider %r to disappear', p.key)
p.wait_for_delete()
store.terminalreporter.write_line(
"Trying to set up provider {}\n".format(provider.key), green=True)
provider.setup()
return True
except Exception as e:
logger.exception(e)
_setup_failures[provider] += 1
if _setup_failures[provider] >= SETUP_FAIL_LIMIT:
_problematic_providers.add(provider)
message = "Provider {} is now marked as problematic and won't be used again."\
" {}: {}".format(provider.key, type(e).__name__, str(e))
logger.warning(message)
store.terminalreporter.write_line(message + "\n", red=True)
if provider.exists:
# Remove it in order to not explode on next calls
provider.delete(cancel=False)
provider.wait_for_delete()
message = "Provider {} was deleted because it failed to set up.".format(
provider.key)
logger.warning(message)
store.terminalreporter.write_line(message + "\n", red=True)
return False
def run(**kwargs):
thread_queue = []
for provider in list_providers("gce"):
mgmt_sys = cfme_data['management_systems'][provider]
gce_credentials = credentials[mgmt_sys['credentials']]
service_account = gce_credentials['service_account']
project = mgmt_sys['project']
zone = mgmt_sys['zone']
thread = Thread(target=upload_template,
args=(project, zone, service_account, kwargs.get('image_url'),
kwargs.get('template_name'), kwargs.get('bucket_name'), provider))
thread.daemon = True
thread_queue.append(thread)
thread.start()
for thread in thread_queue:
thread.join()
def run(**kwargs):
thread_queue = []
for provider in list_providers("gce"):
mgmt_sys = cfme_data['management_systems'][provider]
gce_credentials = credentials[mgmt_sys['credentials']]
service_account = gce_credentials['service_account']
project = mgmt_sys['project']
zone = mgmt_sys['zone']
thread = Thread(target=upload_template,
args=(project, zone, service_account,
kwargs.get('image_url'),
kwargs.get('template_name'),
kwargs.get('bucket_name'), provider))
thread.daemon = True
thread_queue.append(thread)
thread.start()
for thread in thread_queue:
thread.join()
def run(**kwargs):
thread_queue = []
providers = list_providers("openstack")
if kwargs['provider_data']:
provider_data = kwargs['provider_data']
mgmt_sys = providers = provider_data['management_systems']
for provider in providers:
if kwargs['provider_data']:
if mgmt_sys[provider]['type'] != 'openstack':
continue
username = mgmt_sys[provider]['username']
password = mgmt_sys[provider]['password']
sshname = mgmt_sys[provider]['sshname']
sshpass = mgmt_sys[provider]['sshpass']
else:
mgmt_sys = cfme_data['management_systems']
rhos_credentials = credentials[mgmt_sys[provider]['credentials']]
default_host_creds = credentials['host_default']
username = rhos_credentials['username']
password = rhos_credentials['password']
sshname = default_host_creds['username']
sshpass = default_host_creds['password']
rhosip = mgmt_sys[provider]['ipaddress']
auth_url = mgmt_sys[provider]['auth_url']
if not net.is_pingable(rhosip):
continue
if not net.net_check(ports.SSH, rhosip):
print("SSH connection to {}:{} failed, port unavailable".format(
provider, ports.SSH))
continue
thread = Thread(target=upload_template,
args=(rhosip, sshname, sshpass, username, password, auth_url, provider,
kwargs.get('image_url'), kwargs.get('template_name'),
kwargs['provider_data']))
thread.daemon = True
thread_queue.append(thread)
thread.start()
for thread in thread_queue:
thread.join()
def providers(metafunc, filters=None):
""" Gets providers based on given (+ global) filters
Note:
Using the default 'function' scope, each test will be run individually for each provider
before moving on to the next test. To group all tests related to single provider together,
parametrize tests in the 'module' scope.
Note:
testgen for providers now requires the usage of test_flags for collection to work.
Please visit http://cfme-tests.readthedocs.org/guides/documenting.html#documenting-tests
for more details.
"""
filters = filters or []
argnames = []
argvalues = []
idlist = []
# Obtains the test's flags in form of a ProviderFilter
meta = getattr(metafunc.function, 'meta', None)
test_flag_str = getattr(meta, 'kwargs', {}).get('from_docs',
{}).get('test_flag')
if test_flag_str:
test_flags = test_flag_str.split(',')
flags_filter = ProviderFilter(required_flags=test_flags)
filters = filters + [flags_filter]
for provider in list_providers(filters):
argvalues.append([provider])
# Use the provider key for idlist, helps with readable parametrized test output
idlist.append(provider.key)
# Add provider to argnames if missing
if 'provider' in metafunc.fixturenames and 'provider' not in argnames:
metafunc.function = pytest.mark.uses_testgen()(metafunc.function)
argnames.append('provider')
if metafunc.config.getoption('sauce'):
break
return argnames, argvalues, idlist
def providers(metafunc, filters=None):
""" Gets providers based on given (+ global) filters
Note:
Using the default 'function' scope, each test will be run individually for each provider
before moving on to the next test. To group all tests related to single provider together,
parametrize tests in the 'module' scope.
Note:
testgen for providers now requires the usage of test_flags for collection to work.
Please visit http://cfme-tests.readthedocs.org/guides/documenting.html#documenting-tests
for more details.
"""
filters = filters or []
argnames = []
argvalues = []
idlist = []
# Obtains the test's flags in form of a ProviderFilter
meta = getattr(metafunc.function, 'meta', None)
test_flag_str = getattr(meta, 'kwargs', {}).get('from_docs', {}).get('test_flag')
if test_flag_str:
test_flags = test_flag_str.split(',')
flags_filter = ProviderFilter(required_flags=test_flags)
filters = filters + [flags_filter]
for provider in list_providers(filters):
argvalues.append([provider])
# Use the provider key for idlist, helps with readable parametrized test output
idlist.append(provider.key)
# Add provider to argnames if missing
if 'provider' in metafunc.fixturenames and 'provider' not in argnames:
metafunc.function = pytest.mark.uses_testgen()(metafunc.function)
argnames.append('provider')
if metafunc.config.getoption('sauce'):
break
return argnames, argvalues, idlist
def main(trackerbot_url, mark_usable=None):
api = trackerbot.api(trackerbot_url)
thread_q = []
thread_lock = Lock()
template_providers = defaultdict(list)
all_providers = set(list_providers())
unresponsive_providers = set()
# Queue up list_template calls
for provider_key in all_providers:
thread = Thread(target=get_provider_templates,
args=(provider_key, template_providers,
unresponsive_providers, thread_lock))
thread_q.append(thread)
thread.start()
# Join the queued calls
for thread in thread_q:
thread.join()
seen_templates = set()
if mark_usable is None:
usable = {}
else:
usable = {'usable': mark_usable}
existing_provider_templates = [
pt['id'] for pt in trackerbot.depaginate(
api, api.providertemplate.get())['objects']
]
# Find some templates and update the API
for template_name, providers in template_providers.items():
template_name = str(template_name)
group_name, datestamp, stream = trackerbot.parse_template(
template_name)
# Don't want sprout templates
if group_name in ('sprout', 'rhevm-internal'):
print('Ignoring {} from group {}'.format(template_name,
group_name))
continue
seen_templates.add(template_name)
group = trackerbot.Group(group_name, stream=stream)
template = trackerbot.Template(template_name, group, datestamp)
for provider_key in providers:
provider = trackerbot.Provider(provider_key)
if '{}_{}'.format(template_name,
provider_key) in existing_provider_templates:
print('Template {} already tracked for provider {}'.format(
template_name, provider_key))
continue
try:
trackerbot.mark_provider_template(api, provider, template,
**usable)
print('Added {} template {} on provider {} (datestamp: {})'.
format(group_name, template_name, provider_key,
datestamp))
except SlumberHttpBaseException as ex:
print("{}\t{}".format(ex.response.status_code, ex.content))
# Remove provider relationships where they no longer exist, skipping unresponsive providers,
# and providers not known to this environment
for pt in trackerbot.depaginate(api,
api.providertemplate.get())['objects']:
provider_key, template_name = pt['provider']['key'], pt['template'][
'name']
if provider_key not in template_providers[template_name] \
and provider_key not in unresponsive_providers:
if provider_key in all_providers:
print("Cleaning up template {} on {}".format(
template_name, provider_key))
trackerbot.delete_provider_template(api, provider_key,
template_name)
else:
print("Skipping template cleanup {} on unknown provider {}".
format(template_name, provider_key))
# Remove templates that aren't on any providers anymore
for template in trackerbot.depaginate(api, api.template.get())['objects']:
if not template['providers']:
print("Deleting template {} (no providers)".format(
template['name']))
api.template(template['name']).delete()
def generate_html_report(api, stream, filename, appliance_template):
status = 'PASSED'
number_of_images_before = len(images_uploaded(stream))
if get_untested_templates(api, stream, appliance_template):
print('report will not be generated, proceed with the next untested provider')
sys.exit()
stream_data = get_latest_tested_template_on_stream(api, stream, appliance_template)
if len(images_uploaded(stream)) > number_of_images_before:
print("new images are uploaded on latest directory, wait for upload on providers")
wait_for_templates_on_providers(api, stream, appliance_template)
if appliance_template and appliance_template != stream_data['template_name']:
print("the report will be generated only for the latest templates")
sys.exit()
if stream_data and not get_untested_templates(api, stream_data['group_name'],
appliance_template):
print("Found tested template for {}".format(stream))
print("Gathering tested template data for {}".format(stream))
print("Updating the template log")
stream_html = [stream_data['template_name'], stream_data['passed_on_providers'],
stream_data['failed_on_providers'], stream_data['group_name'],
stream_data['datestamp']]
if 'html' in filename:
data = template_env.get_template('template_tester_report.html').render(
upstream=stream_html)
with open(filename, 'w') as report:
report.write(data)
else:
with open(filename, 'a+') as report:
if 'template_rhos' not in images_uploaded(stream):
print('\n\nMISSING: Image for OpenStack in latest directory')
report.write('\n\nMISSING: Image for OpenStack in latest directory')
elif provider_in_the_list(list_providers('openstack'),
stream_data['passed_on_providers']):
report.write('\n\nPASSED: {}'.format(images_uploaded(stream)['template_rhos']))
map(lambda (x): report.write('\n{}: Passed'.format(x)), provider_in_the_list(
list_providers('openstack'), stream_data['passed_on_providers']))
elif provider_in_the_list(list_providers('openstack'),
stream_data['failed_on_providers']):
report.write('\n\nFAILED: {}'.format(images_uploaded(stream)['template_rhos']))
map(lambda (x): report.write('\n{}: Failed'.format(x)),
provider_in_the_list(list_providers('openstack'),
stream_data['failed_on_providers']))
else:
print('\n\nMISSING: OpenStack template is not available on any '
'rhos providers yet')
report.write('\n\nMISSING: OpenStack template is not available on any '
'rhos providers yet')
if 'template_rhevm' not in images_uploaded(stream):
print('\n\nMISSING: Image for RHEVM in latest directory')
report.write('\n\nMISSING: Image for RHEVM in latest directory')
elif provider_in_the_list(list_providers('rhevm'),
stream_data['passed_on_providers']):
report.write('\n\nPASSED: {}'.format(
images_uploaded(stream)['template_rhevm']))
map(lambda(x): report.write('\n{}: Passed'.format(x)), provider_in_the_list(
list_providers('rhevm'), stream_data['passed_on_providers']))
elif provider_in_the_list(list_providers('rhevm'),
stream_data['failed_on_providers']):
report.write('\n\nFAILED: {}'.format(
images_uploaded(stream)['template_rhevm']))
map(lambda(x): report.write('\n{}: Failed'.format(x)),
provider_in_the_list(list_providers('rhevm'),
stream_data['failed_on_providers']))
else:
print('\n\nMISSING: RHEVM template is not available on any '
'rhevm providers yet')
report.write('\n\nMISSING: RHEVM template is not available on any '
'rhevm providers yet')
if 'template_vsphere' not in images_uploaded(stream):
print('\n\nMISSING: Image for VIRTUALCENTER in latest directory')
report.write('\n\nMISSING: Image for VIRTUALCENTER in latest directory')
elif provider_in_the_list(list_providers('virtualcenter'),
stream_data['passed_on_providers']):
report.write('\n\nPASSED: {}'.format(
images_uploaded(stream)['template_vsphere']))
map(lambda (x): report.write('\n{}: Passed'.format(x)), provider_in_the_list(
list_providers('virtualcenter'), stream_data['passed_on_providers']))
elif provider_in_the_list(list_providers('virtualcenter'),
stream_data['failed_on_providers']):
report.write('\n\nFAILED: {}'.format(
images_uploaded(stream)['template_vsphere']))
map(lambda (x): report.write('\n{}: Failed'.format(x)),
provider_in_the_list(list_providers('virtualcenter'),
stream_data['failed_on_providers']))
else:
print('\n\nMISSING: VIRTUALCENTER template is not available on any '
'vmware providers yet')
report.write('\n\nMISSING: VIRTUALCENTER template is not available on any '
'vmware providers yet')
if 'template_scvmm' not in images_uploaded(stream):
print('\n\nMISSING: Image for SCVMM in latest directory')
report.write('\n\nMISSING: Image for SCVMM in latest directory')
elif provider_in_the_list(list_providers('scvmm'),
stream_data['passed_on_providers']):
report.write('\n\nPASSED: {}'.format(
images_uploaded(stream)['template_scvmm']))
map(lambda (x): report.write('\n{}: Passed'.format(x)), provider_in_the_list(
list_providers('scvmm'), stream_data['passed_on_providers']))
elif provider_in_the_list(list_providers('scvmm'),
stream_data['failed_on_providers']):
report.write('\n\nFAILED: {}'.format(
images_uploaded(stream)['template_scvmm']))
map(lambda (x): report.write('\n{}: Failed'.format(x)),
provider_in_the_list(list_providers('scvmm'),
stream_data['failed_on_providers']))
else:
print('\n\nMISSING: SCVMM template is not available on any '
'scvmm providers yet')
report.write('\n\nMISSING: SCVMM template is not available on any '
'scvmm providers yet')
report.seek(0, 0)
lines = report.readlines()
template_missing = filter(lambda (x): "MISSING" in x, lines)
template_passed = filter(lambda (x): "PASSED" in x, lines)
template_failed = filter(lambda (x): "FAILED" in x, lines)
if template_failed:
status = "FAILED"
if template_missing and not (template_passed or template_failed):
report.close()
sys.exit("Template is MISSING....Please verify uploads....")
print("template_tester_results report generated:{}".format(status))
else:
print("No Templates tested on: {}".format(datetime.datetime.now()))
"""
import sys
from traceback import format_exc
from utils.providers import list_providers, get_mgmt
def main(*providers):
for provider_key in list_providers('openstack'):
print('Checking {}'.format(provider_key))
api = get_mgmt(provider_key).api
try:
fips = api.floating_ips.findall(fixed_ip=None)
except Exception:
print('Unable to get fips for {}:'.format(provider_key))
print(format_exc().splitlines()[-1])
continue
for fip in fips:
print('Deleting {} on {}'.format(fip.ip, provider_key))
fip.delete()
print('{} deleted'.format(fip.ip))
if __name__ == "__main__":
provs = sys.argv[1:]
if provs:
main(*provs)
else:
main(*list_providers("openstack"))
def get_orphaned_vmware_files(provider=None):
providers = [provider] if provider else list_providers("virtualcenter")
for provider_key in providers:
# we can add thread here
get_datastores_per_host(provider_key)
def run(**kwargs):
for provider in list_providers("scvmm"):
kwargs = make_kwargs_scvmm(cfme_data, provider,
kwargs.get('image_url'),
kwargs.get('template_name'))
check_kwargs(**kwargs)
mgmt_sys = cfme_data['management_systems'][provider]
hostname_fqdn = mgmt_sys['hostname']
creds = credentials[mgmt_sys['credentials']]
# For powershell to work, we need to extract the User Name from the Domain
user = creds['username'].split('\\')
if len(user) == 2:
username_powershell = user[1]
else:
username_powershell = user[0]
username_scvmm = creds['domain'] + "\\" + creds['username']
scvmm_args = {
"hostname": mgmt_sys['ipaddress'],
"username": username_powershell,
"password": creds['password'],
"domain": creds['domain'],
}
client = SCVMMSystem(**scvmm_args)
url = kwargs.get('image_url', None)
# Template name equals either user input of we extract the name from the url
new_template_name = kwargs.get('template_name', None)
if new_template_name is None:
new_template_name = os.path.basename(url)[:-4]
print("SCVMM:{} started template {} upload".format(
provider, new_template_name))
print("SCVMM:{} Make Template out of the VHD {}".format(
provider, new_template_name))
# use_library is either user input or we use the cfme_data value
use_library = kwargs.get('library', None)
if use_library is None:
use_library = mgmt_sys['template_upload'].get('library',
None) + "\\VHDS\\"
print("SCVMM:{} Template Library: {}".format(provider, use_library))
# The VHD name changed, match the template_name.
new_vhd_name = new_template_name + '.vhd'
use_network = mgmt_sys['template_upload'].get('network', None)
use_os_type = mgmt_sys['template_upload'].get('os_type', None)
cores = mgmt_sys['template_upload'].get('cores', None)
ram = mgmt_sys['template_upload'].get('ram', None)
# Uses PowerShell Get-SCVMTemplate to return a list of templates and aborts if exists.
if not check_template_exists(client, new_template_name):
if kwargs.get('upload'):
upload_vhd(client, url, use_library, new_vhd_name)
if kwargs.get('template'):
print("SCVMM:{} Make Template out of the VHD {}".format(
provider, new_template_name))
make_template(client, hostname_fqdn, new_template_name,
use_library, use_network, use_os_type,
username_scvmm, cores, ram)
try:
wait_for(check_template_exists, [client, new_template_name],
fail_condition=False,
delay=5)
print("SCVMM:{} template {} uploaded successfully".format(
provider, new_template_name))
except Exception as e:
print(e)
print(
"SCVMM:{} Exception occured while verifying the template {} upload"
.format(provider, new_template_name))
else:
print(
"SCVMM: A Template with that name already exists in the SCVMMLibrary"
)
def providers(metafunc, filters=None, selector=ALL):
""" Gets providers based on given (+ global) filters
Note:
Using the default 'function' scope, each test will be run individually for each provider
before moving on to the next test. To group all tests related to single provider together,
parametrize tests in the 'module' scope.
Note:
testgen for providers now requires the usage of test_flags for collection to work.
Please visit http://cfme-tests.readthedocs.org/guides/documenting.html#documenting-tests
for more details.
"""
filters = filters or []
argnames = []
argvalues = []
idlist = []
# Obtains the test's flags in form of a ProviderFilter
meta = getattr(metafunc.function, 'meta', None)
test_flag_str = getattr(meta, 'kwargs', {}).get('from_docs', {}).get('test_flag')
if test_flag_str:
test_flags = test_flag_str.split(',')
flags_filter = ProviderFilter(required_flags=test_flags)
filters = filters + [flags_filter]
potential_providers = list_providers(filters)
if selector == ONE:
allowed_providers = [potential_providers[0]]
elif selector == LATEST:
allowed_providers = [sorted(
potential_providers, key=lambda k:LooseVersion(
str(k.data.get('version', 0))), reverse=True
)[0]]
elif selector == ONE_PER_TYPE:
types = set()
def add_prov(prov):
types.add(prov.type)
return prov
allowed_providers = [
add_prov(prov) for prov in potential_providers if prov.type not in types
]
elif selector == ONE_PER_CATEGORY:
categories = set()
def add_prov(prov):
categories.add(prov.category)
return prov
allowed_providers = [
add_prov(prov) for prov in potential_providers if prov.category not in categories
]
elif selector == ONE_PER_VERSION:
versions = set()
def add_prov(prov):
versions.add(prov.data.get('version', 0))
return prov
allowed_providers = [
add_prov(prov) for prov in potential_providers if prov.data.get(
'version', 0) not in versions
]
else:
allowed_providers = potential_providers
for provider in allowed_providers:
argvalues.append([provider])
# Use the provider key for idlist, helps with readable parametrized test output
idlist.append(provider.key)
# Add provider to argnames if missing
if 'provider' in metafunc.fixturenames and 'provider' not in argnames:
metafunc.function = pytest.mark.uses_testgen()(metafunc.function)
argnames.append('provider')
if metafunc.config.getoption('sauce') or selector == ONE:
break
return argnames, argvalues, idlist
def run(**kwargs):
for provider in list_providers("scvmm"):
kwargs = make_kwargs_scvmm(cfme_data, provider,
kwargs.get('image_url'), kwargs.get('template_name'))
check_kwargs(**kwargs)
mgmt_sys = cfme_data['management_systems'][provider]
hostname_fqdn = mgmt_sys['hostname']
creds = credentials[mgmt_sys['credentials']]
# For powershell to work, we need to extract the User Name from the Domain
user = creds['username'].split('\\')
if len(user) == 2:
username_powershell = user[1]
else:
username_powershell = user[0]
username_scvmm = creds['domain'] + "\\" + creds['username']
scvmm_args = {
"hostname": mgmt_sys['ipaddress'],
"username": username_powershell,
"password": creds['password'],
"domain": creds['domain'],
}
client = SCVMMSystem(**scvmm_args)
url = kwargs.get('image_url', None)
# Template name equals either user input of we extract the name from the url
new_template_name = kwargs.get('template_name', None)
if new_template_name is None:
new_template_name = os.path.basename(url)[:-4]
print("SCVMM:{} started template {} upload".format(provider, new_template_name))
print("SCVMM:{} Make Template out of the VHD {}".format(provider, new_template_name))
# use_library is either user input or we use the cfme_data value
use_library = kwargs.get('library', None)
if use_library is None:
use_library = mgmt_sys['template_upload'].get('library', None) + "\\VHDS\\"
print("SCVMM:{} Template Library: {}".format(provider, use_library))
# The VHD name changed, match the template_name.
new_vhd_name = new_template_name + '.vhd'
use_network = mgmt_sys['template_upload'].get('network', None)
use_os_type = mgmt_sys['template_upload'].get('os_type', None)
cores = mgmt_sys['template_upload'].get('cores', None)
ram = mgmt_sys['template_upload'].get('ram', None)
# Uses PowerShell Get-SCVMTemplate to return a list of templates and aborts if exists.
if not check_template_exists(client, new_template_name):
if kwargs.get('upload'):
upload_vhd(client, url, use_library, new_vhd_name)
if kwargs.get('template'):
print("SCVMM:{} Make Template out of the VHD {}".format(
provider, new_template_name))
make_template(
client,
hostname_fqdn,
new_template_name,
use_library,
use_network,
use_os_type,
username_scvmm,
cores,
ram
)
try:
wait_for(check_template_exists,
[client, new_template_name], fail_condition=False, delay=5)
print("SCVMM:{} template {} uploaded successfully".format(
provider, new_template_name))
except Exception as e:
print(e)
print("SCVMM:{} Exception occured while verifying the template {} upload".format(
provider, new_template_name))
else:
print("SCVMM: A Template with that name already exists in the SCVMMLibrary")
def main(trackerbot_url, mark_usable=None):
api = trackerbot.api(trackerbot_url)
thread_q = []
thread_lock = Lock()
template_providers = defaultdict(list)
all_providers = set(list_providers())
unresponsive_providers = set()
# Queue up list_template calls
for provider_key in all_providers:
thread = Thread(target=get_provider_templates,
args=(provider_key, template_providers, unresponsive_providers, thread_lock))
thread_q.append(thread)
thread.start()
# Join the queued calls
for thread in thread_q:
thread.join()
seen_templates = set()
if mark_usable is None:
usable = {}
else:
usable = {'usable': mark_usable}
existing_provider_templates = [
pt['id']
for pt
in trackerbot.depaginate(api, api.providertemplate.get())['objects']]
# Find some templates and update the API
for template_name, providers in template_providers.items():
template_name = str(template_name)
group_name, datestamp, stream = trackerbot.parse_template(template_name)
# Don't want sprout templates
if group_name in ('sprout', 'rhevm-internal'):
print('Ignoring {} from group {}'.format(template_name, group_name))
continue
seen_templates.add(template_name)
group = trackerbot.Group(group_name, stream=stream)
template = trackerbot.Template(template_name, group, datestamp)
for provider_key in providers:
provider = trackerbot.Provider(provider_key)
if '{}_{}'.format(template_name, provider_key) in existing_provider_templates:
print('Template {} already tracked for provider {}'.format(
template_name, provider_key))
continue
try:
trackerbot.mark_provider_template(api, provider, template, **usable)
print('Added {} template {} on provider {} (datestamp: {})'.format(
group_name, template_name, provider_key, datestamp))
except SlumberHttpBaseException as ex:
print("{}\t{}".format(ex.response.status_code, ex.content))
# Remove provider relationships where they no longer exist, skipping unresponsive providers,
# and providers not known to this environment
for pt in trackerbot.depaginate(api, api.providertemplate.get())['objects']:
provider_key, template_name = pt['provider']['key'], pt['template']['name']
if provider_key not in template_providers[template_name] \
and provider_key not in unresponsive_providers:
if provider_key in all_providers:
print("Cleaning up template {} on {}".format(template_name, provider_key))
trackerbot.delete_provider_template(api, provider_key, template_name)
else:
print("Skipping template cleanup {} on unknown provider {}".format(
template_name, provider_key))
# Remove templates that aren't on any providers anymore
for template in trackerbot.depaginate(api, api.template.get())['objects']:
if not template['providers']:
print("Deleting template {} (no providers)".format(template['name']))
api.template(template['name']).delete()
