def setup_one_or_skip(request, filters=None, use_global_filters=True):
""" Sets up one of matching providers or skips the test
Args:
filters: List of :py:class:`ProviderFilter` or None
request: Needed for logging a potential skip correctly in artifactor
use_global_filters: Will apply global filters as well if `True`, will not otherwise
"""
filters = filters or []
providers = list_providers(filters=filters,
use_global_filters=use_global_filters)
# All providers filtered out?
if not providers:
global_providers = list_providers(
filters=None, use_global_filters=use_global_filters)
if not global_providers:
# This can also mean that there simply are no providers in the yamls!
pytest.skip("No provider matching global filters found")
else:
pytest.skip("No provider matching test-specific filters found")
# Are all providers marked as problematic?
if _problematic_providers.issuperset(providers):
skip_msg = f"All providers marked as problematic: {[p.key for p in providers]}"
_artifactor_skip_providers(request, providers, skip_msg)
# If there is a provider already set up matching the user's requirements, reuse it
for provider in providers:
if provider.exists:
return provider
# If we have more than one provider, we create two separate groups of providers, preferred
# and not preferred, that we shuffle separately and then join together
if len(providers) > 1:
only_preferred_filter = ProviderFilter(required_fields=[
("do_not_prefer", True)
],
inverted=True)
preferred_providers = list_providers(
filters=filters + [only_preferred_filter],
use_global_filters=use_global_filters)
not_preferred_providers = [
p for p in providers if p not in preferred_providers
]
random.shuffle(preferred_providers)
random.shuffle(not_preferred_providers)
providers = preferred_providers + not_preferred_providers
# Try to set up one of matching providers
non_existing = [prov for prov in providers if not prov.exists]
for provider in non_existing:
if _setup_provider_verbose(request, provider):
return provider
skip_msg = f"Failed to set up any matching providers: {[p.key for p in providers]}"
_artifactor_skip_providers(request, non_existing, skip_msg)
def setup_one_or_skip(request, filters=None, use_global_filters=True):
""" Sets up one of matching providers or skips the test
Args:
filters: List of :py:class:`ProviderFilter` or None
request: Needed for logging a potential skip correctly in artifactor
use_global_filters: Will apply global filters as well if `True`, will not otherwise
"""
filters = filters or []
providers = list_providers(filters=filters, use_global_filters=use_global_filters)
# All providers filtered out?
if not providers:
global_providers = list_providers(filters=None, use_global_filters=use_global_filters)
if not global_providers:
# This can also mean that there simply are no providers in the yamls!
pytest.skip("No provider matching global filters found")
else:
pytest.skip("No provider matching test-specific filters found")
# Are all providers marked as problematic?
if _problematic_providers.issuperset(providers):
skip_msg = "All providers marked as problematic: {}".format([p.key for p in providers])
_artifactor_skip_providers(request, providers, skip_msg)
# If there is a provider already set up matching the user's requirements, reuse it
for provider in providers:
if provider.exists:
return provider
# If we have more than one provider, we create two separate groups of providers, preferred
# and not preferred, that we shuffle separately and then join together
if len(providers) > 1:
only_preferred_filter = ProviderFilter(required_fields=[("do_not_prefer", True)],
inverted=True)
preferred_providers = list_providers(
filters=filters + [only_preferred_filter], use_global_filters=use_global_filters)
not_preferred_providers = [p for p in providers if p not in preferred_providers]
random.shuffle(preferred_providers)
random.shuffle(not_preferred_providers)
providers = preferred_providers + not_preferred_providers
# Try to set up one of matching providers
non_existing = [prov for prov in providers if not prov.exists]
for provider in non_existing:
if _setup_provider_verbose(request, provider):
return provider
skip_msg = "Failed to set up any matching providers: {}", [p.key for p in providers]
_artifactor_skip_providers(request, non_existing, skip_msg)
def test_azure_multiple_subscription(appliance, request, soft_assert):
"""
Verifies that different azure providers have different resources access
Steps:
1. Add all Azure providers
2. Compare their VMs/Templates
Metadata:
test_flag: crud
"""
pf = ProviderFilter(classes=[AzureProvider], required_flags=['crud'])
providers = list_providers([pf])
if len(providers) < 2:
pytest.skip("this test needs at least 2 AzureProviders")
prov_inventory = []
for provider in providers:
request.addfinalizer(provider.clear_providers)
provider.create()
provider.validate_stats()
prov_inventory.append((provider.name,
provider.num_vm(),
provider.num_template()))
for index, prov_a in enumerate(prov_inventory[:-1]):
for prov_b in prov_inventory[index + 1:]:
soft_assert(prov_a[1] != prov_b[1], "Same num_vms for {} and {}".format(prov_a[0],
prov_b[0]))
soft_assert(prov_a[2] != prov_b[2], "Same num_templates for {} and {}".format(prov_a[0],
prov_b[0]))
def test_azure_multiple_subscription(appliance, request, soft_assert):
"""
Verifies that different azure providers have different resources access
Steps:
1. Add all Azure providers
2. Compare their VMs/Templates
Metadata:
test_flag: crud
"""
pf = ProviderFilter(classes=[AzureProvider], required_flags=['crud'])
providers = list_providers([pf])
if len(providers) < 2:
pytest.skip("this test needs at least 2 AzureProviders")
prov_inventory = []
for provider in providers:
request.addfinalizer(provider.clear_providers)
provider.create()
provider.validate_stats()
prov_inventory.append(
(provider.name, provider.num_vm(), provider.num_template()))
for index, prov_a in enumerate(prov_inventory[:-1]):
for prov_b in prov_inventory[index + 1:]:
soft_assert(
prov_a[1] != prov_b[1],
"Same num_vms for {} and {}".format(prov_a[0], prov_b[0]))
soft_assert(
prov_a[2] != prov_b[2],
"Same num_templates for {} and {}".format(
prov_a[0], prov_b[0]))
def pytest_configure(config):
""" Filters the list of providers as part of pytest configuration
Note:
Additional filter is added to the global_filters dict of active filters here.
"""
cmd_filter = config.getvalueorskip('use_provider')
if not cmd_filter:
cmd_filter = ["default"]
new_filter = ProviderFilter(keys=cmd_filter, required_tags=cmd_filter, conjunctive=False)
global_filters['use_provider'] = new_filter
logger.debug('Filtering providers with {}, leaves {}'.format(
cmd_filter, [prov.key for prov in list_providers()]))
def az_pwsh_vm(appliance):
"""
azure_pwsh contains powershell and necessary scripts to upload vhd, create VM, get ip of the
resource and delete the VM.
Find the provider that contains that template.
The example of the configuration can be found in data/az_pwsh_cloudinit.cfg
"""
filter_pwsh_template = ProviderFilter(
required_fields=[['templates', 'powershell_vm']])
providers = list_providers(filters=[filter_pwsh_template])
if not providers:
pytest.skip(
"There's no provider that contains a template with powershell")
# If there's more than 1 provider that has the template, we select the first
provider = providers[0]
vm_name = random_vm_name(context="pwsh")
pwsh_vm = provider.data.templates.powershell_vm.name
collection = provider.appliance.provider_based_collection(provider)
try:
vm = collection.instantiate(vm_name, provider, pwsh_vm)
vm.create_on_provider(allow_skip="default")
except IndexError:
from cfme.exceptions import ItemNotFound
raise ItemNotFound('VM with powershell not found!')
vm.mgmt.ensure_state(VmState.RUNNING)
@wait_for_decorator(timeout="10m", delay=5)
def get_ip_address():
ip = vm.ip_address
return ip is not None
yield vm
vm.cleanup_on_provider()
def providers(metafunc, filters=None):
""" Gets providers based on given (+ global) filters
Note:
Using the default 'function' scope, each test will be run individually for each provider
before moving on to the next test. To group all tests related to single provider together,
parametrize tests in the 'module' scope.
Note:
testgen for providers now requires the usage of test_flags for collection to work.
Please visit http://cfme-tests.readthedocs.org/guides/documenting.html#documenting-tests
for more details.
"""
filters = filters or []
argnames = []
argvalues = []
idlist = []
# Obtains the test's flags in form of a ProviderFilter
meta = getattr(metafunc.function, 'meta', None)
test_flag_str = getattr(meta, 'kwargs', {}).get('from_docs',
{}).get('test_flag')
if test_flag_str:
test_flags = test_flag_str.split(',')
flags_filter = ProviderFilter(required_flags=test_flags)
filters = filters + [flags_filter]
for provider in list_providers(filters):
argvalues.append([provider])
# Use the provider key for idlist, helps with readable parametrized test output
idlist.append(provider.key)
# Add provider to argnames if missing
if 'provider' in metafunc.fixturenames and 'provider' not in argnames:
metafunc.function = pytest.mark.uses_testgen()(metafunc.function)
argnames.append('provider')
if metafunc.config.getoption('sauce'):
break
return argnames, argvalues, idlist
def providers(metafunc, filters=None):
""" Gets providers based on given (+ global) filters
Note:
Using the default 'function' scope, each test will be run individually for each provider
before moving on to the next test. To group all tests related to single provider together,
parametrize tests in the 'module' scope.
Note:
testgen for providers now requires the usage of test_flags for collection to work.
Please visit http://cfme-tests.readthedocs.org/guides/documenting.html#documenting-tests
for more details.
"""
filters = filters or []
argnames = []
argvalues = []
idlist = []
# Obtains the test's flags in form of a ProviderFilter
meta = getattr(metafunc.function, 'meta', None)
test_flag_str = getattr(meta, 'kwargs', {}).get('from_docs', {}).get('test_flag')
if test_flag_str:
test_flags = test_flag_str.split(',')
flags_filter = ProviderFilter(required_flags=test_flags)
filters = filters + [flags_filter]
for provider in list_providers(filters):
argvalues.append([provider])
# Use the provider key for idlist, helps with readable parametrized test output
idlist.append(provider.key)
# Add provider to argnames if missing
if 'provider' in metafunc.fixturenames and 'provider' not in argnames:
metafunc.function = pytest.mark.uses_testgen()(metafunc.function)
argnames.append('provider')
if metafunc.config.getoption('sauce'):
break
return argnames, argvalues, idlist
def az_pwsh_vm(appliance):
"""
azure_pwsh contains powershell and necessary scripts to upload vhd, create VM, get ip of the
resource and delete the VM.
Find the provider that contains that template.
The example of the configuration can be found in data/az_pwsh_cloudinit.cfg
"""
filter_pwsh_template = ProviderFilter(required_fields=[['templates', 'powershell_vm']])
providers = list_providers(filters=[filter_pwsh_template])
if not providers:
pytest.skip("There's no provider that contains a template with powershell")
# If there's more than 1 provider that has the template, we select the first
provider = providers[0]
vm_name = random_vm_name(context="pwsh")
pwsh_vm = provider.data.templates.powershell_vm.name
collection = provider.appliance.provider_based_collection(provider)
try:
vm = collection.instantiate(vm_name, provider, pwsh_vm)
vm.create_on_provider(allow_skip="default")
except IndexError:
from cfme.exceptions import ItemNotFound
raise ItemNotFound('VM with powershell not found!')
vm.mgmt.ensure_state(VmState.RUNNING)
@wait_for_decorator(timeout="10m", delay=5)
def get_ip_address():
ip = vm.mgmt.ip
return ip is not None
yield vm
vm.cleanup_on_provider()
def cleanup_vms(texts, max_hours=24, providers=None, tags=None, dryrun=True):
"""
Main method for the cleanup process
Generates regex match objects
Checks providers for cleanup boolean in yaml
Checks provider connectivity (using ping)
Process Pool for provider scanning
Each provider process will thread vm scanning and deletion
Args:
texts (list): List of regex strings to match with
max_hours (int): age limit for deletion
providers (list): List of provider keys to scan and cleanup
tags (list): List of tags to filter providers by
dryrun (bool): Whether or not to actually delete VMs or just report
Returns:
int: return code, 0 on success, otherwise raises exception
"""
logger.info('Matching VM names against the following case-insensitive strings: %r', texts)
# Compile regex, strip leading/trailing single quotes from cli arg
matchers = [re.compile(text.strip("'"), re.IGNORECASE) for text in texts]
# setup provider filter with cleanup (default), tags, and providers (from cli opts)
filters = [ProviderFilter(required_fields=[('cleanup', True)])]
if tags:
logger.info('Adding required_tags ProviderFilter for: %s', tags)
filters.append(ProviderFilter(required_tags=tags))
if providers:
logger.info('Adding keys ProviderFilter for: %s', providers)
filters.append(ProviderFilter(keys=providers))
# Just want keys, use list_providers with no global filters to include disabled.
with DummyAppliance():
providers_to_scan = [prov.key for prov in list_providers(filters, use_global_filters=False)]
logger.info('Potential providers for cleanup, filtered with given tags and provider keys: \n%s',
'\n'.join(providers_to_scan))
# scan providers for vms with name matches
scan_fail_queue = manager.Queue()
with Pool(4) as pool:
deleted_vms = pool.starmap(
cleanup_provider,
((provider_key, matchers, scan_fail_queue, max_hours, dryrun)
for provider_key in providers_to_scan)
)
# flatten deleted_vms list, as its top level is by provider process
# at same time remove None responses
deleted_vms = [report
for prov_list in deleted_vms if prov_list is not None
for report in prov_list]
scan_fail_vms = []
# add the scan failures into deleted vms for reporting sake
while not scan_fail_queue.empty():
scan_fail_vms.append(scan_fail_queue.get())
with open(args.outfile, 'a') as report:
report.write('## VM/Instances deleted via:\n'
'## text matches: {}\n'
'## age matches: {}\n'
.format(texts, max_hours))
message = tabulate(
sorted(scan_fail_vms + deleted_vms, key=attrgetter('result')),
headers=['Provider', 'Name', 'Age', 'Status Before', 'Delete RC'],
tablefmt='orgtbl'
)
report.write(message + '\n')
logger.info(message)
return 0
provider.mgmt.remove_sa_from_scc(scc_name=scc_name,
namespace=sa_namespace,
sa=sa_name)
else:
logger.debug(
"skipping sa %s in scc %s because project exists "
"or it doesn't match any pattern", sa, scc_name)
if __name__ == "__main__":
args = parse_cmd_line()
errors = 0
pf = ProviderFilter(classes=[OpenshiftProvider],
required_fields=[('use_for_sprout', True)])
with DummyAppliance():
providers = list_providers(filters=[pf], use_global_filters=False)
for prov in providers:
# ping provider
try:
prov.mgmt.list_project()
except Exception as e:
logger.error('Connection to provider %s cannot be estabilished',
prov.key)
logger.error('Error: %s', e)
errors += 1
continue
# remove all sa records from scc
if args.cleanup_scc:
try:
text_to_match = [re.compile(r) for r in args.text_to_match]
def providers(metafunc, filters=None, selector=ALL, fixture_name='provider'):
""" Gets providers based on given (+ global) filters
Note:
Using the default 'function' scope, each test will be run individually for each provider
before moving on to the next test. To group all tests related to single provider together,
parametrize tests in the 'module' scope.
Note:
testgen for providers now requires the usage of test_flags for collection to work.
Please visit http://cfme-tests.readthedocs.org/guides/documenting.html#documenting-tests
for more details.
"""
filters = filters or []
argnames = []
argvalues = []
idlist = []
# Obtains the test's flags in form of a ProviderFilter
meta = getattr(metafunc.function, 'meta', None)
test_flag_str = getattr(meta, 'kwargs', {}).get('from_docs', {}).get('test_flag')
if test_flag_str:
test_flags = test_flag_str.split(',')
flags_filter = ProviderFilter(required_flags=test_flags)
filters = filters + [flags_filter]
# available_providers are the ones "available" from the yamls after all of the global and
# local filters have been applied. It will be a list of crud objects.
available_providers = list_providers(filters)
# supported_providers are the ones "supported" in the supportability.yaml file. It will
# be a list of DataProvider objects and will be filtered based upon what the test has asked for
holder = metafunc.config.pluginmanager.get_plugin('appliance-holder')
series = holder.held_appliance.version.series()
supported_providers = all_required(series, filters)
def get_valid_providers(provider):
# We now search through all the available providers looking for one that matches the
# criteria. If we don't find one, we return None
prov_tuples = []
for a_prov in available_providers:
try:
if not a_prov.version:
raise ValueError("provider {p} has no version".format(p=a_prov))
elif (a_prov.version == provider.version and
a_prov.type == provider.type_name and
a_prov.category == provider.category):
prov_tuples.append((provider, a_prov))
except (KeyError, ValueError):
if (a_prov.type == provider.type_name and
a_prov.category == provider.category):
prov_tuples.append((provider, a_prov))
return prov_tuples
# A small routine to check if we need to supply the idlist a provider type or
# a real type/version
need_prov_keys = False
for filter in filters:
if isinstance(filter, ProviderFilter) and filter.classes:
for filt in filter.classes:
if hasattr(filt, 'type_name'):
need_prov_keys = True
break
matching_provs = [valid_provider
for prov in supported_providers
for valid_provider in get_valid_providers(prov)]
# Now we run through the selectors and build up a list of supported providers which match our
# requirements. This then forms the providers that the test should run against.
if selector == ONE:
if matching_provs:
allowed_providers = [matching_provs[0]]
else:
allowed_providers = []
elif selector == LATEST:
allowed_providers = [sorted(
matching_provs, key=lambda k:LooseVersion(
str(k[0].version)), reverse=True
)[0]]
elif selector == ONE_PER_TYPE:
types = set()
def add_prov(prov):
types.add(prov[0].type_name)
return prov
allowed_providers = [
add_prov(prov) for prov in matching_provs if prov[0].type_name not in types
]
elif selector == ONE_PER_CATEGORY:
categories = set()
def add_prov(prov):
categories.add(prov[0].category)
return prov
allowed_providers = [
add_prov(prov) for prov in matching_provs if prov[0].category not in categories
]
elif selector == ONE_PER_VERSION:
# This needs to handle versions per type
versions = defaultdict(set)
def add_prov(prov):
versions[prov[0].type_name].add(prov[0].version)
return prov
allowed_providers = [
add_prov(prov)
for prov in matching_provs
if prov[0].version not in versions[prov[0].type_name]
]
else:
# If there are no selectors, then the allowed providers are whichever are supported
allowed_providers = matching_provs
# Now we iterate through the required providers and try to match them to the available ones
for data_prov, real_prov in allowed_providers:
data_prov.key = real_prov.key
argvalues.append(pytest.param(data_prov))
# Use the provider key for idlist, helps with readable parametrized test output
the_id = str(data_prov.key) if metafunc.config.getoption('legacy_ids') else data_prov.the_id
# Now we modify the id based on what selector we chose
if metafunc.config.getoption('disable_selectors'):
idlist.append(the_id)
else:
if selector == ONE:
if need_prov_keys:
idlist.append(data_prov.type_name)
else:
idlist.append(data_prov.category)
elif selector == ONE_PER_CATEGORY:
idlist.append(data_prov.category)
elif selector == ONE_PER_TYPE:
idlist.append(data_prov.type_name)
else:
idlist.append(the_id)
# Add provider to argnames if missing
if fixture_name in metafunc.fixturenames and fixture_name not in argnames:
metafunc.function = pytest.mark.uses_testgen()(metafunc.function)
argnames.append(fixture_name)
if metafunc.config.getoption('sauce') or selector == ONE:
break
return argnames, argvalues, idlist
sa_namespace, sa_name = sa.split(':')[-2:]
if match(text_to_match, sa_namespace) and not provider.mgmt.does_vm_exist(sa_namespace):
logger.info('removing sa %s from scc %s', sa, scc_name)
provider.mgmt.remove_sa_from_scc(scc_name=scc_name, namespace=sa_namespace,
sa=sa_name)
else:
logger.debug("skipping sa %s in scc %s because project exists "
"or it doesn't match any pattern", sa, scc_name)
if __name__ == "__main__":
args = parse_cmd_line()
errors = 0
pf = ProviderFilter(classes=[OpenshiftProvider], required_fields=[('use_for_sprout', True)])
with DummyAppliance():
providers = list_providers(filters=[pf], use_global_filters=False)
for prov in providers:
# ping provider
try:
prov.mgmt.list_project()
except Exception as e:
logger.error('Connection to provider %s cannot be estabilished', prov.key)
logger.error('Error: %s', e)
errors += 1
continue
# remove all sa records from scc
if args.cleanup_scc:
try:
text_to_match = [re.compile(r) for r in args.text_to_match]
delete_stale_sa(prov, text_to_match)
def providers(metafunc, filters=None, selector=ONE_PER_VERSION, fixture_name='provider'):
""" Gets providers based on given (+ global) filters
Note:
Using the default 'function' scope, each test will be run individually for each provider
before moving on to the next test. To group all tests related to single provider together,
parametrize tests in the 'module' scope.
Note:
testgen for providers now requires the usage of test_flags for collection to work.
Please visit http://cfme-tests.readthedocs.org/guides/documenting.html#documenting-tests
for more details.
"""
filters = filters or []
argnames = []
argvalues = []
idlist = []
# Obtains the test's flags in form of a ProviderFilter
meta = getattr(metafunc.function, 'meta', None)
test_flag_str = getattr(meta, 'kwargs', {}).get('from_docs', {}).get('test_flag')
if test_flag_str:
test_flags = test_flag_str.split(',')
flags_filter = ProviderFilter(required_flags=test_flags)
filters = filters + [flags_filter]
# available_providers are the ones "available" from the yamls after all of the aal and
# local filters have been applied. It will be a list of crud objects.
available_providers = list_providers(filters)
# supported_providers are the ones "supported" in the supportability.yaml file. It will
# be a list of DataProvider objects and will be filtered based upon what the test has asked for
holder = metafunc.config.pluginmanager.get_plugin('appliance-holder')
series = holder.held_appliance.version.series()
supported_providers = all_required(series, filters)
def get_valid_providers(data_provider):
"""Search through all available providers in yaml for ones that match the criteria in
data_provider."""
valid_providers = []
for available_provider in available_providers:
try:
if available_provider.version != data_provider.version:
continue
except KeyError:
pass
if (
available_provider.type == data_provider.type_name and
available_provider.category == data_provider.category
):
data_provider_with_key = DataProvider.get_instance(data_provider.category,
data_provider.type_name, data_provider.version, available_provider.key)
valid_providers.append((data_provider_with_key, available_provider))
return valid_providers
# Check whether we need to supply a provider type or a real type / version to idlist.
need_prov_keys = False
for filter in filters:
if isinstance(filter, ProviderFilter) and filter.classes:
for filt in filter.classes:
if hasattr(filt, 'type_name'):
need_prov_keys = True
break
matching_provs = [valid_provider
for data_provider in supported_providers
for valid_provider in get_valid_providers(data_provider)]
# Run through the selectors and build a list of supported providers which match our
# requirements. This then forms the providers against which the test should run.
if selector == ONE:
if matching_provs:
allowed_providers = [matching_provs[0]]
else:
allowed_providers = []
elif selector == SECOND:
if matching_provs:
try:
allowed_providers = [matching_provs[1]]
except IndexError:
pytest.skip("no second provider was found")
else:
allowed_providers = []
elif selector == LATEST:
allowed_providers = [sorted(
matching_provs, key=lambda k:LooseVersion(
str(k[0].version)), reverse=True
)[0]]
elif selector == ONE_PER_TYPE:
types = set()
def add_prov(prov):
types.add(prov[0].type_name)
return prov
allowed_providers = [
add_prov(prov) for prov in matching_provs if prov[0].type_name not in types
]
elif selector == ONE_PER_CATEGORY:
categories = set()
def add_prov(prov):
categories.add(prov[0].category)
return prov
allowed_providers = [
add_prov(prov) for prov in matching_provs if prov[0].category not in categories
]
elif selector == ONE_PER_VERSION:
# This needs to handle versions per type
versions = defaultdict(set)
def add_prov(prov):
versions[prov[0].type_name].add(prov[0].version)
return prov
allowed_providers = [
add_prov(prov)
for prov in matching_provs
if prov[0].version not in versions[prov[0].type_name]
]
else:
# If there are no selectors, then the allowed providers are whichever are supported
allowed_providers = matching_provs
# Iterate through the required providers and try to match them to the available ones
for data_prov, real_prov in allowed_providers:
argvalues.append(pytest.param(data_prov))
# Use the provider key for idlist, helps with readable parametrized test output
use_legacy_ids = metafunc.config.getoption('legacy_ids')
legacy_key = data_prov.key
the_id = legacy_key if use_legacy_ids else data_prov.the_id
# Modify the test id based on what selector we chose.
if metafunc.config.getoption('disable_selectors'):
selected_id = the_id
else:
if selector == ONE:
if need_prov_keys:
selected_id = data_prov.type_name
else:
selected_id = data_prov.category
elif selector == ONE_PER_CATEGORY:
selected_id = data_prov.category
elif selector == ONE_PER_TYPE:
selected_id = data_prov.type_name
elif selector == ALL:
if use_legacy_ids:
# if we're already using legacy there shouldn't be duplicate ids
selected_id = legacy_key
else:
# include the key to differentiate multiple providers of same type+version
selected_id = f'{the_id}-{legacy_key}'
else:
selected_id = the_id
idlist.append(selected_id)
# Add provider to argnames if missing
if fixture_name in metafunc.fixturenames and fixture_name not in argnames:
metafunc.function = pytest.mark.uses_testgen()(metafunc.function)
argnames.append(fixture_name)
if metafunc.config.getoption('sauce') or selector == ONE:
break
return argnames, argvalues, idlist
def cleanup_vms(texts, max_hours=24, providers=None, tags=None, prompt=True):
"""
Main method for the cleanup process
Generates regex match objects
Checks providers for cleanup boolean in yaml
Checks provider connectivity (using ping)
Threads process_provider_vms to build list of vms to delete
Prompts user to continue with delete
Threads deleting of the vms
Args:
texts (list): List of regex strings to match with
max_hours (int): age limit for deletion
providers (list): List of provider keys to scan and cleanup
tags (list): List of tags to filter providers by
prompt (bool): Whether or not to prompt the user before deleting vms
Returns:
int: return code, 0 on success, otherwise raises exception
"""
logger.info('Matching VM names against the following case-insensitive strings: %r', texts)
# Compile regex, strip leading/trailing single quotes from cli arg
matchers = [re.compile(text.strip("'"), re.IGNORECASE) for text in texts]
# setup provider filter with cleanup (default), tags, and providers (from cli opts)
filters = [ProviderFilter(required_fields=[('cleanup', True)])]
if tags:
logger.info('Adding required_tags ProviderFilter for: %s', tags)
filters.append(ProviderFilter(required_tags=tags))
if providers:
logger.info('Adding keys ProviderFilter for: %s', providers)
filters.append(ProviderFilter(keys=providers))
# Just want keys, use list_providers with no global filters to include disabled.
providers_to_scan = [prov.key for prov in list_providers(filters, use_global_filters=False)]
logger.info('Potential providers for cleanup, filtered with given tags and provider keys: \n%s',
'\n'.join(providers_to_scan))
# scan providers for vms with name matches
# manager = Manager()
text_match_queue = manager.Queue()
scan_fail_queue = manager.Queue()
provider_scan_args = [
(provider_key, matchers, text_match_queue, scan_fail_queue)
for provider_key in providers_to_scan]
pool_manager(scan_provider, provider_scan_args)
text_matched = []
while not text_match_queue.empty():
text_matched.append(text_match_queue.get())
# scan vms for age matches
age_match_queue = manager.Queue()
vm_scan_args = [
(provider_key, vm_name, timedelta(hours=int(max_hours)), age_match_queue, scan_fail_queue)
for provider_key, vm_name in text_matched]
pool_manager(scan_vm, vm_scan_args)
vms_to_delete = []
while not age_match_queue.empty():
vms_to_delete.append(age_match_queue.get())
scan_fail_vms = []
# add the scan failures into deleted vms for reporting sake
while not scan_fail_queue.empty():
scan_fail_vms.append(scan_fail_queue.get())
if vms_to_delete and prompt:
yesno = raw_input('Delete these VMs? [y/N]: ')
if str(yesno).lower() != 'y':
logger.info('Exiting.')
return 0
# initialize this even if we don't have anything to delete, for report consistency
deleted_vms = []
if vms_to_delete:
delete_queue = manager.Queue()
delete_vm_args = [(provider_key, vm_name, age, delete_queue)
for provider_key, vm_name, age in vms_to_delete]
pool_manager(delete_vm, delete_vm_args)
while not delete_queue.empty():
deleted_vms.append(delete_queue.get()) # Each item is a VmReport tuple
else:
logger.info('No VMs to delete.')
with open(args.outfile, 'a') as report:
report.write('## VM/Instances deleted via:\n'
'## text matches: {}\n'
'## age matches: {}\n'
.format(texts, max_hours))
message = tabulate(sorted(scan_fail_vms + deleted_vms, key=attrgetter('result')),
headers=['Provider', 'Name', 'Age', 'Status Before', 'Delete RC'],
tablefmt='orgtbl')
report.write(message + '\n')
logger.info(message)
return 0
def cleanup_vms(texts, max_hours=24, providers=None, tags=None, prompt=True):
"""
Main method for the cleanup process
Generates regex match objects
Checks providers for cleanup boolean in yaml
Checks provider connectivity (using ping)
Threads process_provider_vms to build list of vms to delete
Prompts user to continue with delete
Threads deleting of the vms
Args:
texts (list): List of regex strings to match with
max_hours (int): age limit for deletion
providers (list): List of provider keys to scan and cleanup
tags (list): List of tags to filter providers by
prompt (bool): Whether or not to prompt the user before deleting vms
Returns:
int: return code, 0 on success, otherwise raises exception
"""
logger.info(
'Matching VM names against the following case-insensitive strings: %r',
texts)
# Compile regex, strip leading/trailing single quotes from cli arg
matchers = [re.compile(text.strip("'"), re.IGNORECASE) for text in texts]
# setup provider filter with cleanup (default), tags, and providers (from cli opts)
filters = [ProviderFilter(required_fields=[('cleanup', True)])]
if tags:
logger.info('Adding required_tags ProviderFilter for: %s', tags)
filters.append(ProviderFilter(required_tags=tags))
if providers:
logger.info('Adding keys ProviderFilter for: %s', providers)
filters.append(ProviderFilter(keys=providers))
# Just want keys, use list_providers with no global filters to include disabled.
with DummyAppliance():
providers_to_scan = [
prov.key
for prov in list_providers(filters, use_global_filters=False)
]
logger.info(
'Potential providers for cleanup, filtered with given tags and provider keys: \n%s',
'\n'.join(providers_to_scan))
# scan providers for vms with name matches
# manager = Manager()
text_match_queue = manager.Queue()
scan_fail_queue = manager.Queue()
provider_scan_args = [(provider_key, matchers, text_match_queue,
scan_fail_queue)
for provider_key in providers_to_scan]
pool_manager(scan_provider, provider_scan_args)
text_matched = []
while not text_match_queue.empty():
text_matched.append(text_match_queue.get())
# scan vms for age matches
age_match_queue = manager.Queue()
vm_scan_args = [(provider_key, vm_name, timedelta(hours=int(max_hours)),
age_match_queue, scan_fail_queue)
for provider_key, vm_name in text_matched]
pool_manager(scan_vm, vm_scan_args)
vms_to_delete = []
while not age_match_queue.empty():
vms_to_delete.append(age_match_queue.get())
scan_fail_vms = []
# add the scan failures into deleted vms for reporting sake
while not scan_fail_queue.empty():
scan_fail_vms.append(scan_fail_queue.get())
if vms_to_delete and prompt:
yesno = raw_input('Delete these VMs? [y/N]: ')
if str(yesno).lower() != 'y':
logger.info('Exiting.')
return 0
# initialize this even if we don't have anything to delete, for report consistency
deleted_vms = []
if vms_to_delete:
delete_queue = manager.Queue()
delete_vm_args = [(provider_key, vm_name, age, delete_queue)
for provider_key, vm_name, age in vms_to_delete]
pool_manager(delete_vm, delete_vm_args)
while not delete_queue.empty():
deleted_vms.append(
delete_queue.get()) # Each item is a VmReport tuple
else:
logger.info('No VMs to delete.')
with open(args.outfile, 'a') as report:
report.write('## VM/Instances deleted via:\n'
'## text matches: {}\n'
'## age matches: {}\n'.format(texts, max_hours))
message = tabulate(
sorted(scan_fail_vms + deleted_vms, key=attrgetter('result')),
headers=['Provider', 'Name', 'Age', 'Status Before', 'Delete RC'],
tablefmt='orgtbl')
report.write(message + '\n')
logger.info(message)
return 0
output_queue.put(output_list)
return
if __name__ == "__main__":
args = parse_cmd_line()
# providers as a set when processing tags to ensure unique entries
filters = []
if args.provider:
filters.append(ProviderFilter(keys=args.provider))
if args.tag:
filters.append(ProviderFilter(required_tags=args.tag))
# don't include global filter to keep disabled in the list
providers = [prov.key for prov in list_providers(filters, use_global_filters=False)]
queue = Queue() # for MP output
proc_list = [
Process(target=list_vms, args=(provider, queue), name='list_vms:{}'.format(provider))
for provider in providers
]
for proc in proc_list:
proc.start()
for proc in proc_list:
proc.join()
print('Done processing providers, assembling report...')
# Now pull all the results off of the queue
# Stacking the generator this way is equivalent to using list.extend instead of list.append
def providers(metafunc, filters=None, selector=ALL):
""" Gets providers based on given (+ global) filters
Note:
Using the default 'function' scope, each test will be run individually for each provider
before moving on to the next test. To group all tests related to single provider together,
parametrize tests in the 'module' scope.
Note:
testgen for providers now requires the usage of test_flags for collection to work.
Please visit http://cfme-tests.readthedocs.org/guides/documenting.html#documenting-tests
for more details.
"""
filters = filters or []
argnames = []
argvalues = []
idlist = []
# Obtains the test's flags in form of a ProviderFilter
meta = getattr(metafunc.function, 'meta', None)
test_flag_str = getattr(meta, 'kwargs', {}).get('from_docs', {}).get('test_flag')
if test_flag_str:
test_flags = test_flag_str.split(',')
flags_filter = ProviderFilter(required_flags=test_flags)
filters = filters + [flags_filter]
potential_providers = list_providers(filters)
if selector == ONE:
if potential_providers:
allowed_providers = [potential_providers[0]]
else:
allowed_providers = []
elif selector == LATEST:
allowed_providers = [sorted(
potential_providers, key=lambda k:LooseVersion(
str(k.data.get('version', 0))), reverse=True
)[0]]
elif selector == ONE_PER_TYPE:
types = set()
def add_prov(prov):
types.add(prov.type)
return prov
allowed_providers = [
add_prov(prov) for prov in potential_providers if prov.type not in types
]
elif selector == ONE_PER_CATEGORY:
categories = set()
def add_prov(prov):
categories.add(prov.category)
return prov
allowed_providers = [
add_prov(prov) for prov in potential_providers if prov.category not in categories
]
elif selector == ONE_PER_VERSION:
versions = set()
def add_prov(prov):
versions.add(prov.data.get('version', 0))
return prov
allowed_providers = [
add_prov(prov) for prov in potential_providers if prov.data.get(
'version', 0) not in versions
]
else:
allowed_providers = potential_providers
for provider in allowed_providers:
argvalues.append([provider])
# Use the provider key for idlist, helps with readable parametrized test output
idlist.append(provider.key)
# Add provider to argnames if missing
if 'provider' in metafunc.fixturenames and 'provider' not in argnames:
metafunc.function = pytest.mark.uses_testgen()(metafunc.function)
argnames.append('provider')
if metafunc.config.getoption('sauce') or selector == ONE:
break
return argnames, argvalues, idlist
def cleanup_vms(texts, max_hours=24, providers=None, tags=None, dryrun=True):
"""
Main method for the cleanup process
Generates regex match objects
Checks providers for cleanup boolean in yaml
Checks provider connectivity (using ping)
Process Pool for provider scanning
Each provider process will thread vm scanning and deletion
Args:
texts (list): List of regex strings to match with
max_hours (int): age limit for deletion
providers (list): List of provider keys to scan and cleanup
tags (list): List of tags to filter providers by
dryrun (bool): Whether or not to actually delete VMs or just report
Returns:
int: return code, 0 on success, otherwise raises exception
"""
logger.info('Matching VM names against the following case-insensitive strings: %r', texts)
# Compile regex, strip leading/trailing single quotes from cli arg
matchers = [re.compile(text.strip("'"), re.IGNORECASE) for text in texts]
# setup provider filter with cleanup (default), tags, and providers (from cli opts)
filters = [ProviderFilter(required_fields=[('cleanup', True)])]
if tags:
logger.info('Adding required_tags ProviderFilter for: %s', tags)
filters.append(ProviderFilter(required_tags=tags))
if providers:
logger.info('Adding keys ProviderFilter for: %s', providers)
filters.append(ProviderFilter(keys=providers))
# Just want keys, use list_providers with no global filters to include disabled.
with DummyAppliance():
providers_to_scan = [prov.key for prov in list_providers(filters, use_global_filters=False)]
logger.info('Potential providers for cleanup, filtered with given tags and provider keys: \n%s',
'\n'.join(providers_to_scan))
# scan providers for vms with name matches
scan_fail_queue = manager.Queue()
with Pool(4) as pool:
deleted_vms = pool.starmap(
cleanup_provider,
((provider_key, matchers, scan_fail_queue, max_hours, dryrun)
for provider_key in providers_to_scan)
)
# flatten deleted_vms list, as its top level is by provider process
# at same time remove None responses
deleted_vms = [report
for prov_list in deleted_vms if prov_list is not None
for report in prov_list]
scan_fail_vms = []
# add the scan failures into deleted vms for reporting sake
while not scan_fail_queue.empty():
scan_fail_vms.append(scan_fail_queue.get())
with open(args.outfile, 'a') as report:
report.write('## VM/Instances deleted via:\n'
'## text matches: {}\n'
'## age matches: {}\n'
.format(texts, max_hours))
message = tabulate(
sorted(scan_fail_vms + deleted_vms, key=attrgetter('result')),
headers=['Provider', 'Name', 'Age', 'Status Before', 'Delete RC'],
tablefmt='orgtbl'
)
report.write(message + '\n')
logger.info(message)
return 0
def providers(metafunc, filters=None, selector=ALL):
""" Gets providers based on given (+ global) filters
Note:
Using the default 'function' scope, each test will be run individually for each provider
before moving on to the next test. To group all tests related to single provider together,
parametrize tests in the 'module' scope.
Note:
testgen for providers now requires the usage of test_flags for collection to work.
Please visit http://cfme-tests.readthedocs.org/guides/documenting.html#documenting-tests
for more details.
"""
filters = filters or []
argnames = []
argvalues = []
idlist = []
# Obtains the test's flags in form of a ProviderFilter
meta = getattr(metafunc.function, 'meta', None)
test_flag_str = getattr(meta, 'kwargs', {}).get('from_docs',
{}).get('test_flag')
if test_flag_str:
test_flags = test_flag_str.split(',')
flags_filter = ProviderFilter(required_flags=test_flags)
filters = filters + [flags_filter]
potential_providers = list_providers(filters)
if selector == ONE:
if potential_providers:
allowed_providers = [potential_providers[0]]
else:
allowed_providers = []
elif selector == LATEST:
allowed_providers = [
sorted(potential_providers,
key=lambda k: LooseVersion(str(k.data.get('version', 0))),
reverse=True)[0]
]
elif selector == ONE_PER_TYPE:
types = set()
def add_prov(prov):
types.add(prov.type)
return prov
allowed_providers = [
add_prov(prov) for prov in potential_providers
if prov.type not in types
]
elif selector == ONE_PER_CATEGORY:
categories = set()
def add_prov(prov):
categories.add(prov.category)
return prov
allowed_providers = [
add_prov(prov) for prov in potential_providers
if prov.category not in categories
]
elif selector == ONE_PER_VERSION:
versions = set()
def add_prov(prov):
versions.add(prov.data.get('version', 0))
return prov
allowed_providers = [
add_prov(prov) for prov in potential_providers
if prov.data.get('version', 0) not in versions
]
else:
allowed_providers = potential_providers
for provider in allowed_providers:
argvalues.append([provider])
# Use the provider key for idlist, helps with readable parametrized test output
idlist.append(provider.key)
# Add provider to argnames if missing
if 'provider' in metafunc.fixturenames and 'provider' not in argnames:
metafunc.function = pytest.mark.uses_testgen()(metafunc.function)
argnames.append('provider')
if metafunc.config.getoption('sauce') or selector == ONE:
break
return argnames, argvalues, idlist
def providers(metafunc, filters=None, selector=ALL, fixture_name='provider'):
""" Gets providers based on given (+ global) filters
Note:
Using the default 'function' scope, each test will be run individually for each provider
before moving on to the next test. To group all tests related to single provider together,
parametrize tests in the 'module' scope.
Note:
testgen for providers now requires the usage of test_flags for collection to work.
Please visit http://cfme-tests.readthedocs.org/guides/documenting.html#documenting-tests
for more details.
"""
filters = filters or []
argnames = []
argvalues = []
idlist = []
# Obtains the test's flags in form of a ProviderFilter
meta = getattr(metafunc.function, 'meta', None)
test_flag_str = getattr(meta, 'kwargs', {}).get('from_docs',
{}).get('test_flag')
if test_flag_str:
test_flags = test_flag_str.split(',')
flags_filter = ProviderFilter(required_flags=test_flags)
filters = filters + [flags_filter]
# available_providers are the ones "available" from the yamls after all of the global and
# local filters have been applied. It will be a list of crud objects.
available_providers = list_providers(filters)
# supported_providers are the ones "supported" in the supportability.yaml file. It will
# be a list of DataProvider objects and will be filtered based upon what the test has asked for
holder = metafunc.config.pluginmanager.get_plugin('appliance-holder')
series = holder.held_appliance.version.series()
supported_providers = all_required(series, filters)
def get_valid_providers(provider):
# We now search through all the available providers looking for one that matches the
# criteria. If we don't find one, we return None
prov_tuples = []
for a_prov in available_providers:
try:
if not a_prov.version:
raise ValueError(
"provider {p} has no version".format(p=a_prov))
elif (a_prov.version == provider.version
and a_prov.type == provider.type_name
and a_prov.category == provider.category):
prov_tuples.append((provider, a_prov))
except (KeyError, ValueError):
if (a_prov.type == provider.type_name
and a_prov.category == provider.category):
prov_tuples.append((provider, a_prov))
return prov_tuples
# A small routine to check if we need to supply the idlist a provider type or
# a real type/version
need_prov_keys = False
for filter in filters:
if isinstance(filter, ProviderFilter) and filter.classes:
for filt in filter.classes:
if hasattr(filt, 'type_name'):
need_prov_keys = True
break
matching_provs = [
valid_provider for prov in supported_providers
for valid_provider in get_valid_providers(prov)
]
# Now we run through the selectors and build up a list of supported providers which match our
# requirements. This then forms the providers that the test should run against.
if selector == ONE:
if matching_provs:
allowed_providers = [matching_provs[0]]
else:
allowed_providers = []
elif selector == LATEST:
allowed_providers = [
sorted(matching_provs,
key=lambda k: LooseVersion(str(k[0].version)),
reverse=True)[0]
]
elif selector == ONE_PER_TYPE:
types = set()
def add_prov(prov):
types.add(prov[0].type_name)
return prov
allowed_providers = [
add_prov(prov) for prov in matching_provs
if prov[0].type_name not in types
]
elif selector == ONE_PER_CATEGORY:
categories = set()
def add_prov(prov):
categories.add(prov[0].category)
return prov
allowed_providers = [
add_prov(prov) for prov in matching_provs
if prov[0].category not in categories
]
elif selector == ONE_PER_VERSION:
# This needs to handle versions per type
versions = defaultdict(set)
def add_prov(prov):
versions[prov[0].type_name].add(prov[0].version)
return prov
allowed_providers = [
add_prov(prov) for prov in matching_provs
if prov[0].version not in versions[prov[0].type_name]
]
else:
# If there are no selectors, then the allowed providers are whichever are supported
allowed_providers = matching_provs
# Now we iterate through the required providers and try to match them to the available ones
for data_prov, real_prov in allowed_providers:
data_prov.key = real_prov.key
argvalues.append(pytest.param(data_prov))
# Use the provider key for idlist, helps with readable parametrized test output
if metafunc.config.getoption('legacy_ids'):
the_id = "{}".format(data_prov.key)
else:
ver = data_prov.version if data_prov.version else None
if ver:
the_id = "{}-{}".format(data_prov.type_name, data_prov.version)
else:
the_id = "{}".format(data_prov.type_name)
# Now we modify the id based on what selector we chose
if metafunc.config.getoption('disable_selectors'):
idlist.append(the_id)
else:
if selector == ONE:
if need_prov_keys:
idlist.append(data_prov.type_name)
else:
idlist.append(data_prov.category)
elif selector == ONE_PER_CATEGORY:
idlist.append(data_prov.category)
elif selector == ONE_PER_TYPE:
idlist.append(data_prov.type_name)
else:
idlist.append(the_id)
# Add provider to argnames if missing
if fixture_name in metafunc.fixturenames and fixture_name not in argnames:
metafunc.function = pytest.mark.uses_testgen()(metafunc.function)
argnames.append(fixture_name)
if metafunc.config.getoption('sauce') or selector == ONE:
break
return argnames, argvalues, idlist
output_queue.put(output_list)
return
if __name__ == "__main__":
args = parse_cmd_line()
# providers as a set when processing tags to ensure unique entries
filters = []
if args.provider:
filters.append(ProviderFilter(keys=args.provider))
if args.tag:
filters.append(ProviderFilter(required_tags=args.tag))
# don't include global filter to keep disabled in the list
providers = [prov.key for prov in list_providers(filters, use_global_filters=False)]
queue = Queue() # for MP output
proc_list = [
Process(target=list_vms, args=(provider, queue), name='list_vms:{}'.format(provider))
for provider in providers
]
for proc in proc_list:
proc.start()
for proc in proc_list:
proc.join()
print('Done processing providers, assembling report...')
# Now pull all the results off of the queue
# Stacking the generator this way is equivalent to using list.extend instead of list.append
