def from_config(cls):
url = cfme_data.get("bugzilla", {}).get("url")
product = cfme_data.get("bugzilla", {}).get("product")
if url is None:
raise Exception("No Bugzilla URL specified!")
cr_root = cfme_data.get("bugzilla", {}).get("credentials")
username = credentials.get(cr_root, {}).get("username")
password = credentials.get(cr_root, {}).get("password")
return cls(
url=url, user=username, password=password, cookiefile=None,
tokenfile=None, product=product)
def from_config(cls):
url = cfme_data.get("bugzilla", {}).get("url", None)
product = cfme_data.get("bugzilla", {}).get("product", None)
if url is None:
raise Exception("No Bugzilla URL specified!")
cr_root = cfme_data.get("bugzilla", {}).get("credentials", None)
username = credentials.get(cr_root, {}).get("username", None)
password = credentials.get(cr_root, {}).get("password", None)
return cls(
url=url, user=username, password=password, cookiefile=None,
tokenfile=None, product=product)
def bug(request):
"""Fixture, that when called provides specific bug. No machinery that changes the ID is involved
Usage:
.. code-block:: python
@pytest.mark.bugzilla(1234)
# or just @pytest.mark.bugzilla if you want no generic skipping and so
def test_something(bug):
if bug(345).status is "blabla":
foo()
bar()
baz()
It works only on ``bugzilla``-marked tests so far. After I find some neat 'global' store in
py.test, I will modify it to be usable everywhere.
If bugzilla integration is disabled, it returns BugMock instance which answers False on each
comparison, equality or attribute.
"""
try:
return lambda bug_id: getbug_cached(
request.node._bugzilla, bug_id, cfme_data.get("bugzilla", {}).get("loose", []))
except AttributeError:
return lambda *args, **kwargs: BugMock()
def _skip_test_flags(data, metafunc, required_fields):
# Test to see the test has meta data, if it does and that metadata contains
# a test_flag kwarg, then check to make sure the provider contains that test_flag
# if not, do not collect the provider for this particular test.
# Obtain the tests flags
meta = getattr(metafunc.function, 'meta', None)
test_flags = getattr(meta, 'kwargs', {}) \
.get('from_docs', {}).get('test_flag', '').split(',')
if test_flags != ['']:
test_flags = [flag.strip() for flag in test_flags]
defined_flags = cfme_data.get('test_flags', '').split(',')
defined_flags = [flag.strip() for flag in defined_flags]
excluded_flags = data.get('excluded_test_flags', '').split(',')
excluded_flags = [flag.strip() for flag in excluded_flags]
allowed_flags = set(defined_flags) - set(excluded_flags)
if set(test_flags) - allowed_flags:
logger.info("Skipping Provider %s for test %s in module %s because "
"it does not have the right flags, "
"%s does not contain %s",
data['name'], metafunc.function.func_name, metafunc.function.__module__,
list(allowed_flags), list(set(test_flags) - allowed_flags))
return True
return False
def get_vm_config_modified_time(name, vm_name, datastore_url, provider_key):
try:
providers_data = cfme_data.get("management_systems", {})
hosts = providers_data[provider_key]['hosts']
host_creds = providers_data[provider_key].get('host_credentials', 'host_default')
hostname = [host['name'] for host in hosts if name in host['name']]
if not hostname:
hostname = re.findall(r'[0-9]+(?:\.[0-9]+){3}', name)
connect_kwargs = {
'username': credentials[host_creds]['username'],
'password': credentials[host_creds]['password'],
'hostname': hostname[0]
}
datastore_path = re.findall(r'([^ds:`/*].*)', str(datastore_url))
ssh_client = SSHClient(**connect_kwargs)
command = 'find ~/{}/{} -name {} | xargs date -r'.format(
datastore_path[0], str(vm_name), str(vm_name) + '.vmx')
exit_status, output = ssh_client.run_command(command)
ssh_client.close()
modified_time = parser.parse(output.rstrip())
modified_time = modified_time.astimezone(pytz.timezone(str(get_localzone())))
return modified_time.replace(tzinfo=None)
except Exception as e:
logger.error(e)
return False
def auth_groups(metafunc, auth_mode):
"""Provides two test params based on the 'auth_modes' and 'group_roles' in cfme_data:
``group_name``:
expected group name in provided by the backend specified in ``auth_mode``
``group_data``:
list of nav destinations that should be visible as a member of ``group_name``
Args:
auth_mode: One of the auth_modes specified in ``cfme_data.get('auth_modes', {})``
"""
argnames = ['group_name', 'group_data']
argvalues = []
idlist = []
if auth_mode in cfme_data.get('auth_modes', {}):
# If auth_modes exists, group_roles is assumed to exist as well
gdata = group_data()
for group in gdata:
argvalues.append([group, sorted(gdata[group])])
idlist.append(group)
return argnames, argvalues, idlist
def pytest_generate_tests(metafunc):
# Filter out providers without provisioning data or hosts defined
argnames, argvalues, idlist = testgen.infra_providers(metafunc, 'provisioning')
argnames = argnames + ['cloud_init_template']
new_idlist = []
new_argvalues = []
for i, argvalue_tuple in enumerate(argvalues):
args = dict(zip(argnames, argvalue_tuple))
if not args['provisioning']:
# No provisioning data available
continue
if args['provider_type'] == "scvmm" or args['provider_type'] == 'virtualcenter':
continue
# required keys should be a subset of the dict keys set
if not {'ci-template', 'ci-username', 'ci-pass', 'template'}.issubset(
args['provisioning'].viewkeys()):
continue
cloud_init_template = args['provisioning']['ci-template']
if cloud_init_template not in cfme_data.get('customization_templates', {}).keys():
continue
argvalues[i].append(get_template_from_config(cloud_init_template))
new_idlist.append(idlist[i])
new_argvalues.append(argvalues[i])
testgen.parametrize(metafunc, argnames, new_argvalues, ids=new_idlist, scope="module")
def direct_connection(self):
"""Returns an API from mgmt_system.py targeted at this provider"""
# Find the credentials entry
name = str(self.host_name)
from utils.conf import cfme_data, credentials
for prov_id, provider in cfme_data.get("management_systems", {}).iteritems():
if provider.get("hostname", None) == name or provider.get("region", None) == name:
credentials = credentials.get(provider["credentials"], {})
provider_id = prov_id
break
else:
raise NameError("Could not find provider %s in the credentials!" % name)
ptype = str(self.type).lower()
if ptype == "emsredhat":
from utils.mgmt_system import RHEVMSystem
return RHEVMSystem(self.host_name, credentials["username"], credentials["password"])
elif ptype == "emsvmware":
from utils.mgmt_system import VMWareSystem
return VMWareSystem(self.host_name, credentials["username"], credentials["password"])
elif ptype == "emsamazon":
from utils.mgmt_system import EC2System
return EC2System(**credentials)
elif ptype == "emsopenstack":
from utils.mgmt_system import OpenstackSystem
credentials.update(
{"auth_url": cfme_data["management_systems"][provider_id]["auth_url"]}
)
return OpenstackSystem(**credentials)
else:
TypeError("Unknown Provider type!")
def pytest_generate_tests(metafunc):
# Filter out providers without provisioning data or hosts defined
argnames, argvalues, idlist = testgen.infra_providers(metafunc, required_fields=[
'iso_datastore',
['provisioning', 'host'],
['provisioning', 'datastore'],
['provisioning', 'iso_template'],
['provisioning', 'iso_file'],
['provisioning', 'iso_kickstart'],
['provisioning', 'iso_root_password'],
['provisioning', 'iso_image_type'],
['provisioning', 'vlan'],
])
argnames = argnames + ['iso_cust_template', 'iso_datastore']
new_idlist = []
new_argvalues = []
for i, argvalue_tuple in enumerate(argvalues):
args = dict(zip(argnames, argvalue_tuple))
iso_cust_template = args['provider'].data['provisioning']['iso_kickstart']
if iso_cust_template not in cfme_data.get('customization_templates', {}).keys():
continue
argvalues[i].append(get_template_from_config(iso_cust_template))
argvalues[i].append(ISODatastore(args['provider'].name))
new_idlist.append(idlist[i])
new_argvalues.append(argvalues[i])
testgen.parametrize(metafunc, argnames, new_argvalues, ids=new_idlist, scope="module")
def pytest_generate_tests(metafunc):
# Filter out providers without templates defined
argnames, argvalues, idlist = testgen.cloud_providers(metafunc, 'provisioning')
argnames = argnames + ['cloud_init_template']
new_argvalues = []
new_idlist = []
for i, argvalue_tuple in enumerate(argvalues):
args = dict(zip(argnames, argvalue_tuple))
if not args['provisioning']:
# Don't know what type of instance to provision, move on
continue
if not {'ci-template', 'ci-username', 'ci-pass', 'image'}.issubset(
args['provisioning'].viewkeys()):
# Need all for template provisioning
continue
cloud_init_template = args['provisioning']['ci-template']
if cloud_init_template not in cfme_data.get('customization_templates', {}).keys():
continue
argvalues[i].append(get_template_from_config(cloud_init_template))
new_idlist.append(idlist[i])
new_argvalues.append(argvalues[i])
testgen.parametrize(metafunc, argnames, new_argvalues, ids=new_idlist, scope="module")
def _uncollect_test_flags(data, metafunc, required_fields):
# Test to see the test has meta data, if it does and that metadata contains
# a test_flag kwarg, then check to make sure the provider contains that test_flag
# if not, do not collect the provider for this particular test.
# Obtain the tests flags
meta = getattr(metafunc.function, 'meta', None)
test_flags = getattr(meta, 'kwargs', {}) \
.get('from_docs', {}).get('test_flag', '').split(',')
if test_flags != ['']:
test_flags = [flag.strip() for flag in test_flags]
defined_flags = cfme_data.get('test_flags', '').split(',')
defined_flags = [flag.strip() for flag in defined_flags]
excluded_flags = data.get('excluded_test_flags', '').split(',')
excluded_flags = [flag.strip() for flag in excluded_flags]
allowed_flags = set(defined_flags) - set(excluded_flags)
if set(test_flags) - allowed_flags:
logger.info(
"Uncollecting Provider %s for test %s in module %s because "
"it does not have the right flags, "
"%s does not contain %s", data['name'],
metafunc.function.func_name, metafunc.function.__module__,
list(allowed_flags), list(set(test_flags) - allowed_flags))
return True
return False
def pytest_generate_tests(metafunc):
# Filter out providers without provisioning data or hosts defined
argnames, argvalues, idlist = testgen.infra_providers(metafunc, 'provisioning')
argnames = argnames + ['cloud_init_template']
new_idlist = []
new_argvalues = []
for i, argvalue_tuple in enumerate(argvalues):
args = dict(zip(argnames, argvalue_tuple))
if not args['provisioning']:
# No provisioning data available
continue
if args['provider'].type == "scvmm" or args['provider'].type == 'virtualcenter':
continue
# required keys should be a subset of the dict keys set
if not {'ci-template', 'ci-username', 'ci-pass', 'template'}.issubset(
args['provisioning'].viewkeys()):
continue
cloud_init_template = args['provisioning']['ci-template']
if cloud_init_template not in cfme_data.get('customization_templates', {}).keys():
continue
argvalues[i].append(get_template_from_config(cloud_init_template))
new_idlist.append(idlist[i])
new_argvalues.append(argvalues[i])
testgen.parametrize(metafunc, argnames, new_argvalues, ids=new_idlist, scope="module")
def set_host_credentials(request, provider, vm_analysis_data):
# Add credentials to host
test_host = host.Host(name=vm_analysis_data['host'], provider=provider)
wait_for(lambda: test_host.exists, delay=10, num_sec=120)
host_list = cfme_data.get('management_systems',
{})[provider.key].get('hosts', [])
host_data = [x for x in host_list if x.name == vm_analysis_data['host']][0]
# has valid creds appears broken
if not test_host.has_valid_credentials:
test_host.update(updates={
'credentials':
host.get_credentials_from_config(host_data['credentials'])
},
validate_credentials=True)
# Remove creds after test
@request.addfinalizer
def _host_remove_creds():
test_host.update(updates={
'credentials':
host.Host.Credential(principal="", secret="", verify_secret="")
},
validate_credentials=False)
def pytest_generate_tests(metafunc):
# Filter out providers without templates defined
argnames, argvalues, idlist = testgen.cloud_providers(
metafunc, 'provisioning')
argnames = argnames + ['cloud_init_template']
new_argvalues = []
new_idlist = []
for i, argvalue_tuple in enumerate(argvalues):
args = dict(zip(argnames, argvalue_tuple))
if not args['provisioning']:
# Don't know what type of instance to provision, move on
continue
if not {'ci-template', 'ci-username', 'ci-pass', 'image'}.issubset(
args['provisioning'].viewkeys()):
# Need all for template provisioning
continue
cloud_init_template = args['provisioning']['ci-template']
if cloud_init_template not in cfme_data.get('customization_templates',
{}).keys():
continue
argvalues[i].append(get_template_from_config(cloud_init_template))
new_idlist.append(idlist[i])
new_argvalues.append(argvalues[i])
testgen.parametrize(metafunc,
argnames,
new_argvalues,
ids=new_idlist,
scope="module")
def auth_groups(metafunc, auth_mode):
"""Provides two test params based on the 'auth_modes' and 'group_roles' in cfme_data:
``group_name``:
expected group name in provided by the backend specified in ``auth_mode``
``group_data``:
list of nav destinations that should be visible as a member of ``group_name``
Args:
auth_mode: One of the auth_modes specified in ``cfme_data.get('auth_modes', {})``
"""
argnames = ['group_name', 'group_data']
argvalues = []
idlist = []
if auth_mode in cfme_data.get('auth_modes', {}):
# If auth_modes exists, group_roles is assumed to exist as well
gdata = group_data()
for group in gdata:
argvalues.append([group, sorted(gdata[group])])
idlist.append(group)
return argnames, argvalues, idlist
def _uncollect_test_flags(data, metafunc, required_fields):
# Test to see the test has meta data, if it does and that metadata contains
# a test_flag kwarg, then check to make sure the provider contains that test_flag
# if not, do not collect the provider for this particular test.
# Obtain the tests flags
meta = getattr(metafunc.function, "meta", None)
test_flags = getattr(meta, "kwargs", {}).get("from_docs", {}).get("test_flag", "").split(",")
if test_flags != [""]:
test_flags = [flag.strip() for flag in test_flags]
defined_flags = cfme_data.get("test_flags", "").split(",")
defined_flags = [flag.strip() for flag in defined_flags]
excluded_flags = data.get("excluded_test_flags", "").split(",")
excluded_flags = [flag.strip() for flag in excluded_flags]
allowed_flags = set(defined_flags) - set(excluded_flags)
if set(test_flags) - allowed_flags:
logger.info(
"Uncollecting Provider %s for test %s in module %s because "
"it does not have the right flags, "
"%s does not contain %s",
data["name"],
metafunc.function.func_name,
metafunc.function.__module__,
list(allowed_flags),
list(set(test_flags) - allowed_flags),
)
return True
return False
def main():
parser = argparse.ArgumentParser(
epilog=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument('--address',
help='hostname or ip address of target appliance',
default=None)
parser.add_argument('--sdk_url',
help='url to download sdk pkg',
default=cfme_data.get("basic_info",
{}).get("netapp_sdk_url"))
parser.add_argument('--restart',
help='restart evmserverd after installation ' +
'(required for proper operation)',
action="store_true")
args = parser.parse_args()
if not args.address:
appliance = get_or_create_current_appliance()
else:
appliance = IPAppliance(address=args.address)
print('Address: {}'.format(appliance.address))
print('SDK URL: {}'.format(args.sdk_url))
print('Restart: {}'.format(args.restart))
appliance.install_netapp_sdk(sdk_url=args.sdk_url,
reboot=args.restart,
log_callback=log)
def get_vm_config_modified_time(name, vm_name, datastore_url, provider_key):
try:
providers_data = cfme_data.get("management_systems", {})
hosts = providers_data[provider_key]['hosts']
host_creds = providers_data[provider_key].get('host_credentials',
'host_default')
hostname = [host['name'] for host in hosts if name in host['name']]
if not hostname:
hostname = re.findall(r'[0-9]+(?:\.[0-9]+){3}', name)
connect_kwargs = {
'username': credentials[host_creds]['username'],
'password': credentials[host_creds]['password'],
'hostname': hostname[0]
}
datastore_path = re.findall(r'([^ds:`/*].*)', str(datastore_url))
ssh_client = SSHClient(**connect_kwargs)
command = 'find ~/{}/{} -name {} | xargs date -r'.format(
datastore_path[0], str(vm_name),
str(vm_name) + '.vmx')
exit_status, output = ssh_client.run_command(command)
ssh_client.close()
modified_time = parser.parse(output.rstrip())
modified_time = modified_time.astimezone(
pytz.timezone(str(get_localzone())))
return modified_time.replace(tzinfo=None)
except Exception as e:
logger.error(e)
return False
def pytest_generate_tests(metafunc):
# Filter out providers without provisioning data or hosts defined
argnames, argvalues, idlist = testgen.providers_by_class(
metafunc, [InfraProvider], required_fields=[
('iso_datastore', True),
['provisioning', 'host'],
['provisioning', 'datastore'],
['provisioning', 'iso_template'],
['provisioning', 'iso_file'],
['provisioning', 'iso_kickstart'],
['provisioning', 'iso_root_password'],
['provisioning', 'iso_image_type'],
['provisioning', 'vlan'],
])
argnames = argnames + ['iso_cust_template', 'iso_datastore']
new_idlist = []
new_argvalues = []
for i, argvalue_tuple in enumerate(argvalues):
args = dict(zip(argnames, argvalue_tuple))
if args['provider'].type == "scvmm":
continue
iso_cust_template = args['provider'].data['provisioning']['iso_kickstart']
if iso_cust_template not in cfme_data.get('customization_templates', {}).keys():
continue
argvalues[i].append(get_template_from_config(iso_cust_template))
argvalues[i].append(ISODatastore(args['provider'].name))
new_idlist.append(idlist[i])
new_argvalues.append(argvalues[i])
testgen.parametrize(metafunc, argnames, new_argvalues, ids=new_idlist, scope="module")
def pytest_generate_tests(metafunc):
# Filter out providers without provisioning data or hosts defined
argnames, argvalues, idlist = testgen.infra_providers(
metafunc, 'provisioning')
argnames = argnames + ['iso_cust_template', 'iso_datastore']
new_idlist = []
new_argvalues = []
for i, argvalue_tuple in enumerate(argvalues):
args = dict(zip(argnames, argvalue_tuple))
if not args['provisioning']:
# No provisioning data available
continue
if args['provider'].type == "scvmm":
continue
provider_data = cfme_data.get(
'management_systems',
{})[argvalue_tuple[argnames.index('provider')].key]
if not provider_data.get('iso_datastore', False):
continue
# required keys should be a subset of the dict keys set
if not {
'iso_template', 'host', 'datastore', 'iso_file',
'iso_kickstart', 'iso_root_password', 'iso_image_type', 'vlan'
}.issubset(args['provisioning'].viewkeys()):
# Need all for template provisioning
continue
iso_cust_template = args['provisioning']['iso_kickstart']
if iso_cust_template not in cfme_data.get('customization_templates',
{}).keys():
continue
argvalues[i].append(get_template_from_config(iso_cust_template))
argvalues[i].append(ISODatastore(provider_data['name']))
new_idlist.append(idlist[i])
new_argvalues.append(argvalues[i])
testgen.parametrize(metafunc,
argnames,
new_argvalues,
ids=new_idlist,
scope="module")
def exists(self):
sel.force_navigate('clouds_tenants')
provider_name = cfme_data.get('management_systems', {})[self.provider_key]['name']
res = list_page.tenant_table.find_row_by_cells({'Name': self.name,
'Cloud Provider': provider_name})
if res:
return True
else:
return False
def pytest_generate_tests(metafunc):
argnames, argvalues, idlist = ['db_url', 'db_version', 'db_desc'], [], []
db_backups = cfme_data.get('db_backups', {})
if not db_backups:
return []
for key, data in db_backups.iteritems():
argvalues.append((data.url, data.version, data.desc))
idlist.append(key)
return metafunc.parametrize(argnames=argnames, argvalues=argvalues, ids=idlist)
def pytest_generate_tests(metafunc):
argnames, argvalues, idlist = ['db_url', 'db_version', 'db_desc'], [], []
db_backups = cfme_data.get('db_backups', {})
if not db_backups:
return []
for key, data in db_backups.iteritems():
argvalues.append((data.url, data.version, data.desc))
idlist.append(key)
return metafunc.parametrize(argnames=argnames, argvalues=argvalues, ids=idlist)
def pytest_generate_tests(metafunc):
# Filter out providers without provisioning data or hosts defined
argnames, argvalues, idlist = testgen.infra_providers(metafunc, "provisioning")
argnames = argnames + ["iso_cust_template", "iso_datastore"]
new_idlist = []
new_argvalues = []
for i, argvalue_tuple in enumerate(argvalues):
args = dict(zip(argnames, argvalue_tuple))
if not args["provisioning"]:
# No provisioning data available
continue
if args["provider_type"] == "scvmm":
continue
provider_data = cfme_data.get("management_systems", {})[argvalue_tuple[argnames.index("provider_key")]]
if not provider_data.get("iso_datastore", False):
continue
# required keys should be a subset of the dict keys set
if not {
"iso_template",
"host",
"datastore",
"iso_file",
"iso_kickstart",
"iso_root_password",
"iso_image_type",
"vlan",
}.issubset(args["provisioning"].viewkeys()):
# Need all for template provisioning
continue
iso_cust_template = args["provisioning"]["iso_kickstart"]
if iso_cust_template not in cfme_data.get("customization_templates", {}).keys():
continue
argvalues[i].append(get_template_from_config(iso_cust_template))
argvalues[i].append(ISODatastore(provider_data["name"]))
new_idlist.append(idlist[i])
new_argvalues.append(argvalues[i])
testgen.parametrize(metafunc, argnames, new_argvalues, ids=new_idlist, scope="module")
def pytest_generate_tests(metafunc):
# Filter out providers without host provisioning data defined
argnames, argvalues, idlist = testgen.providers_by_class(
metafunc, [InfraProvider],
required_fields=[
['host_provisioning', 'pxe_server'],
['host_provisioning', 'pxe_image'],
['host_provisioning', 'pxe_image_type'],
['host_provisioning', 'pxe_kickstart'],
['host_provisioning', 'datacenter'],
['host_provisioning', 'cluster'],
['host_provisioning', 'datastores'],
['host_provisioning', 'hostname'],
['host_provisioning', 'root_password'],
['host_provisioning', 'ip_addr'],
['host_provisioning', 'subnet_mask'],
['host_provisioning', 'gateway'],
['host_provisioning', 'dns'],
])
pargnames, pargvalues, pidlist = testgen.pxe_servers(metafunc)
argnames = argnames + ['pxe_server', 'pxe_cust_template']
pxe_server_names = [pval[0] for pval in pargvalues]
new_idlist = []
new_argvalues = []
for i, argvalue_tuple in enumerate(argvalues):
args = dict(zip(argnames, argvalue_tuple))
try:
prov_data = args['provider'].data['host_provisioning']
except KeyError:
# No host provisioning data available
continue
stream = prov_data.get('runs_on_stream', '')
if not version.current_version().is_in_series(str(stream)):
continue
pxe_server_name = prov_data.get('pxe_server', '')
if pxe_server_name not in pxe_server_names:
continue
pxe_cust_template = prov_data.get('pxe_kickstart', '')
if pxe_cust_template not in cfme_data.get('customization_templates',
{}).keys():
continue
argvalues[i].append(get_pxe_server_from_config(pxe_server_name))
argvalues[i].append(get_template_from_config(pxe_cust_template))
new_idlist.append(idlist[i])
new_argvalues.append(argvalues[i])
testgen.parametrize(metafunc,
argnames,
new_argvalues,
ids=new_idlist,
scope="module")
def direct_connection(self):
"""Returns an API from mgmt_system.py targeted at this provider"""
# Find the credentials entry
name = str(self.host_name)
from utils.conf import cfme_data, credentials
for prov_id, provider in cfme_data.get("management_systems",
{}).iteritems():
if provider.get("hostname", None) == name or provider.get(
"region", None) == name:
credentials = credentials.get(provider["credentials"], {})
provider_id = prov_id
break
else:
raise NameError("Could not find provider %s in the credentials!" %
name)
ptype = str(self.type).lower()
if ptype == "emsredhat":
from utils.mgmt_system import RHEVMSystem
return RHEVMSystem(self.host_name, credentials["username"],
credentials["password"])
elif ptype == "emsvmware":
from utils.mgmt_system import VMWareSystem
return VMWareSystem(self.host_name, credentials["username"],
credentials["password"])
elif ptype == "emsmicrosoft":
from utils.mgmt_system import SCVMMSystem
return SCVMMSystem(hostname=self.host_name,
username=credentials["username"],
password=credentials["password"],
domain=credentials["domain"])
elif ptype == "emsamazon":
from utils.mgmt_system import EC2System
return EC2System(**credentials)
elif ptype == "emsopenstack":
from utils.mgmt_system import OpenstackSystem
credentials.update({
"auth_url":
cfme_data.get("management_systems",
{})[provider_id]["auth_url"]
})
return OpenstackSystem(**credentials)
else:
TypeError("Unknown Provider type!")
def configure_openldap_auth_mode(browser, available_auth_modes):
"""Configure LDAP authentication mode"""
if 'miq_openldap' in available_auth_modes:
server_data = cfme_data.get('auth_modes', {})['miq_openldap']
configuration.set_auth_mode(**server_data)
yield
current_appliance.server.login_admin()
configuration.set_auth_mode(mode='database')
else:
yield
def group():
data = cfme_data.get("openldap_test", {})
if not data:
pytest.skip("No openldap_test section in yaml")
credentials = Credential(
principal=data["username"],
secret=data["password"],
)
return Group(description=data["group_name"], role="EvmRole-user",
user_to_lookup=data['username'], ldap_credentials=credentials)
def objects_from_config(*keys):
result = {}
for key, data in cfme_data.get("storage", {}).get("managers", {}).iteritems():
if keys and key not in keys:
continue
data = copy(data)
if "credentials" in data:
data["credentials"] = StorageManager.Credential(
**credentials.get(data["credentials"], {}))
result[key] = StorageManager(**data)
return result
def group_data():
roles = cfme_data.get("group_roles", {})
# These roles are not present on 5.2 appliance in these groups
if version.current_version() < "5.3":
_remove_page(roles, "evmgroup-super_administrator", ["clouds_tenants"])
_remove_page(roles, "evmgroup-user", ["services_requests", "infrastructure_requests"])
if version.current_version() < "5.4":
_remove_from_all(roles, "clouds_stacks")
_remove_from_all(roles, "infrastructure_config_management")
return roles
def cleanup_vms(texts, max_hours=24, providers=None, prompt=True):
providers = providers or list_providers()
providers_data = cfme_data.get("management_systems", {})
delta = datetime.timedelta(hours=int(max_hours))
vms_to_delete = defaultdict(set)
thread_queue = []
# precompile regexes
matchers = [re.compile(text, re.IGNORECASE) for text in texts]
for provider_key in providers:
ipaddress = cfme_data['management_systems'][provider_key].get('ipaddress', None)
if ipaddress and not net.is_pingable(ipaddress):
continue
provider_type = providers_data[provider_key].get('type', None)
thread = Thread(target=process_provider_vms,
args=(provider_key, provider_type, matchers, delta, vms_to_delete))
# Mark as daemon thread for easy-mode KeyboardInterrupt handling
thread.daemon = True
thread_queue.append(thread)
thread.start()
# Join the queued calls
for thread in thread_queue:
thread.join()
for provider_key, vm_set in vms_to_delete.items():
print('{}:'.format(provider_key))
for vm_name, vm_delta in vm_set:
days, hours = vm_delta.days, vm_delta.seconds / 3600
print(' {} is {} days, {} hours old'.format(vm_name, days, hours))
if vms_to_delete and prompt:
yesno = raw_input('Delete these VMs? [y/N]: ')
if str(yesno).lower() != 'y':
print('Exiting.')
return 0
if not vms_to_delete:
print('No VMs to delete.')
thread_queue = []
for provider_key, vm_set in vms_to_delete.items():
thread = Thread(target=delete_provider_vms,
args=(provider_key, [name for name, t_delta in vm_set]))
# Mark as daemon thread for easy-mode KeyboardInterrupt handling
thread.daemon = True
thread_queue.append(thread)
thread.start()
for thread in thread_queue:
thread.join()
print("Deleting finished")
def configure_openldap_auth_mode_default_groups(browser, available_auth_modes):
"""Configure LDAP authentication mode"""
if 'miq_openldap' in available_auth_modes:
server_data = cfme_data.get('auth_modes', {})['miq_openldap']
server_data['get_groups'] = False
server_data['default_groups'] = 'EvmRole-user'
configuration.set_auth_mode(**server_data)
yield
current_appliance.server.login_admin()
configuration.set_auth_mode(mode='database')
else:
yield
def objects_from_config(*keys):
result = {}
for key, data in cfme_data.get("storage", {}).get("managers",
{}).iteritems():
if keys and key not in keys:
continue
data = copy(data)
if "credentials" in data:
data["credentials"] = StorageManager.Credential(
**credentials.get(data["credentials"], {}))
result[key] = StorageManager(**data)
return result
def roles(request, all_possible_roles, appliance):
result = {}
for role in all_possible_roles:
result[role] = role in cfme_data.get("server_roles",
{})["sets"][request.param]
# Hard-coded protection
result["user_interface"] = True
# ansible role introduced in CFME 5.8
if appliance.version < '5.8' and result.get('embedded_ansible'):
del result['embedded_ansible']
return result
def cleanup_vms(texts, max_hours=24, providers=None, prompt=True):
providers = providers or list_all_providers()
providers_data = cfme_data.get("management_systems", {})
delta = datetime.timedelta(hours=int(max_hours))
vms_to_delete = defaultdict(set)
thread_queue = []
# precompile regexes
matchers = [re.compile(text) for text in texts]
for provider_key in providers:
provider_type = providers_data[provider_key].get('type', None)
thread = Thread(target=process_provider_vms,
args=(provider_key, provider_type, matchers, delta,
vms_to_delete))
# Mark as daemon thread for easy-mode KeyboardInterrupt handling
thread.daemon = True
thread_queue.append(thread)
thread.start()
# Join the queued calls
for thread in thread_queue:
thread.join()
for provider_key, vm_set in vms_to_delete.items():
print('{}:'.format(provider_key))
for vm_name, vm_delta in vm_set:
days, hours = vm_delta.days, vm_delta.seconds / 3600
print(' {} is {} days, {} hours old'.format(vm_name, days, hours))
if vms_to_delete and prompt:
yesno = raw_input('Delete these VMs? [y/N]: ')
if str(yesno).lower() != 'y':
print('Exiting.')
return 0
if not vms_to_delete:
print('No VMs to delete.')
thread_queue = []
for provider_key, vm_set in vms_to_delete.items():
thread = Thread(target=delete_provider_vms,
args=(provider_key, [name
for name, t_delta in vm_set]))
# Mark as daemon thread for easy-mode KeyboardInterrupt handling
thread.daemon = True
thread_queue.append(thread)
thread.start()
for thread in thread_queue:
thread.join()
print("Deleting finished")
def configure_aws_iam_auth_mode(browser, available_auth_modes):
"""Configure AWS IAM authentication mode"""
if 'miq_aws_iam' in available_auth_modes:
aws_iam_data = dict(cfme_data.get('auth_modes', {})['miq_aws_iam'])
aws_iam_creds = credentials[aws_iam_data.pop('credentials')]
aws_iam_data['access_key'] = aws_iam_creds['username']
aws_iam_data['secret_key'] = aws_iam_creds['password']
configuration.set_auth_mode(**aws_iam_data)
yield
current_appliance.server.login_admin()
configuration.set_auth_mode(mode='database')
else:
yield
def pytest_generate_tests(metafunc):
# Filter out providers without provisioning data or hosts defined
argnames, argvalues, idlist = testgen.infra_providers(metafunc, 'provisioning')
argnames = argnames + ['iso_cust_template', 'iso_datastore']
new_idlist = []
new_argvalues = []
for i, argvalue_tuple in enumerate(argvalues):
args = dict(zip(argnames, argvalue_tuple))
if not args['provisioning']:
# No provisioning data available
continue
if args['provider_type'] == "scvmm":
continue
provider_data = cfme_data.get('management_systems', {})[
argvalue_tuple[argnames.index('provider_key')]]
if not provider_data.get('iso_datastore', False):
continue
# required keys should be a subset of the dict keys set
if not {'iso_template', 'host', 'datastore',
'iso_file', 'iso_kickstart',
'iso_root_password',
'iso_image_type', 'vlan'}.issubset(args['provisioning'].viewkeys()):
# Need all for template provisioning
continue
iso_cust_template = args['provisioning']['iso_kickstart']
if iso_cust_template not in cfme_data.get('customization_templates', {}).keys():
continue
argvalues[i].append(get_template_from_config(iso_cust_template))
argvalues[i].append(ISODatastore(provider_data['name']))
new_idlist.append(idlist[i])
new_argvalues.append(argvalues[i])
testgen.parametrize(metafunc, argnames, new_argvalues, ids=new_idlist, scope="module")
def is_datastore_banned(datastore_name):
"""Checks whether the datastore is in the list of datastores not allowed to use
Args:
datastore_name: Name of the datastore
Returns: :py:class:`bool`
"""
banned_datastores = cfme_data.get("datastores_not_for_provision", [])
for banned_datastore in banned_datastores:
if banned_datastore in datastore_name:
return True
else:
return False
def pytest_configure(config):
path = cfme_data.get("cfme_annotations_path")
if path:
to_parse = project_path.join(path)
parsed = parse(to_parse)
if not parsed:
store.terminalreporter.line("no test annotation found in {}".format(to_parse), yellow=True)
else:
store.terminalreporter.line("no test annotation found in {}".format(path), yellow=True)
parsed = []
config.pluginmanager.register(MarkFromMap.from_parsed_list(parsed, "tier", pytest.mark.tier))
config.pluginmanager.register(MarkFromMap.from_parsed_list(parsed, "requirement", pytest.mark.requirement))
config.pluginmanager.register(MarkFromMap.from_parsed_list(parsed, "type", pytest.mark.__getattr__))
def config_managers(metafunc):
"""Provides config managers
"""
argnames = ['config_manager_obj']
argvalues = []
idlist = []
data = cfme_data.get('configuration_managers', {})
for cfg_mgr_key in data:
argvalues.append([get_config_manager_from_config(cfg_mgr_key)])
idlist.append(cfg_mgr_key)
return argnames, argvalues, idlist
def test_external_auth_ipa(request, setup_first_provider, configure_external_auth_ipa_module):
try:
data = cfme_data.get("ipa_test", {})
except KeyError:
pytest.skip("No ipa_test section in yaml")
group = Group(description='cfme', role="EvmRole-user")
request.addfinalizer(group.delete)
group.create()
user = User(name=data["fullname"])
request.addfinalizer(user.delete)
request.addfinalizer(login.login_admin)
login.login(data["username"], data["password"])
assert login.current_full_name() == data["fullname"]
def config_managers(metafunc):
"""Provides config managers
"""
argnames = ["config_manager_obj"]
argvalues = []
idlist = []
data = cfme_data.get("configuration_managers", {})
for cfg_mgr_key in data:
argvalues.append([get_config_manager_from_config(cfg_mgr_key)])
idlist.append(cfg_mgr_key)
return argnames, argvalues, idlist
def is_datastore_banned(datastore_name):
"""Checks whether the datastore is in the list of datastores not allowed to use
Args:
datastore_name: Name of the datastore
Returns: :py:class:`bool`
"""
banned_datastores = cfme_data.get("datastores_not_for_provision", [])
for banned_datastore in banned_datastores:
if banned_datastore in datastore_name:
return True
else:
return False
def list_provider_vms(provider_key):
try:
provider = get_mgmt(provider_key)
vm_list = provider.list_vm()
provider_type = cfme_data.get("management_systems", {})[provider_key].get('type', None)
now = datetime.datetime.now()
for vm_name in vm_list:
creation = provider.vm_creation_time(vm_name)
providers_vm_list.append([provider_type, provider_key, vm_name,
(now - creation), provider.vm_type(vm_name),
provider.vm_status(vm_name)])
except Exception as e:
logger.error('failed to list vms from provider {}'.format(provider_key))
logger.exception(e)
def pytest_generate_tests(metafunc):
# Filter out providers without host provisioning data defined
argnames, argvalues, idlist = testgen.providers_by_class(
metafunc, [InfraProvider], required_fields=[
['host_provisioning', 'pxe_server'],
['host_provisioning', 'pxe_image'],
['host_provisioning', 'pxe_image_type'],
['host_provisioning', 'pxe_kickstart'],
['host_provisioning', 'datacenter'],
['host_provisioning', 'cluster'],
['host_provisioning', 'datastores'],
['host_provisioning', 'hostname'],
['host_provisioning', 'root_password'],
['host_provisioning', 'ip_addr'],
['host_provisioning', 'subnet_mask'],
['host_provisioning', 'gateway'],
['host_provisioning', 'dns'],
])
pargnames, pargvalues, pidlist = testgen.pxe_servers(metafunc)
argnames = argnames + ['pxe_server', 'pxe_cust_template']
pxe_server_names = [pval[0] for pval in pargvalues]
new_idlist = []
new_argvalues = []
for i, argvalue_tuple in enumerate(argvalues):
args = dict(zip(argnames, argvalue_tuple))
try:
prov_data = args['provider'].data['host_provisioning']
except KeyError:
# No host provisioning data available
continue
stream = prov_data.get('runs_on_stream', '')
if not version.current_version().is_in_series(str(stream)):
continue
pxe_server_name = prov_data.get('pxe_server', '')
if pxe_server_name not in pxe_server_names:
continue
pxe_cust_template = prov_data.get('pxe_kickstart', '')
if pxe_cust_template not in cfme_data.get('customization_templates', {}).keys():
continue
argvalues[i].append(get_pxe_server_from_config(pxe_server_name))
argvalues[i].append(get_template_from_config(pxe_cust_template))
new_idlist.append(idlist[i])
new_argvalues.append(argvalues[i])
testgen.parametrize(metafunc, argnames, new_argvalues, ids=new_idlist, scope="module")
def pytest_addoption(parser):
group = parser.getgroup('GitHub Issues integration')
group.addoption('--github',
action='store_true',
default=cfme_data.get("github", {}).get("enabled", False),
dest='github',
help='Enable GitHub Issue blockers integration.')
group.addoption('--github-default-repo',
action='store',
default=cfme_data.get("github", {}).get("default_repo", None),
dest='github_default_repo',
help='Default repo for GitHub queries')
cr_root = cfme_data.get("github", {}).get("credentials", None)
group.addoption('--github-user',
action='store',
default=credentials.get(cr_root, {}).get("username", None),
dest='github_user',
help='GH Username.')
group.addoption('--github-password',
action='store',
default=credentials.get(cr_root, {}).get("password", None),
dest='github_password',
help='GH Password.')
def test_openldap_auth(request, setup_first_provider, configure_openldap_auth_mode):
data = cfme_data.get("openldap_test", {})
if not data:
pytest.skip("No openldap_test section in yaml")
group = Group(description=data["group_name"], role="EvmRole-user")
request.addfinalizer(group.delete)
group.create()
credentials = Credential(principal=data["username"], secret=data["password"], verify_secret=data["password"])
user = User(name=data["fullname"], credential=credentials)
request.addfinalizer(user.delete)
request.addfinalizer(login.login_admin)
with user:
login.login(user)
assert login.current_full_name() == data["fullname"]
def pytest_addoption(parser):
group = parser.getgroup('Bugzilla integration')
group.addoption('--bugzilla',
action='store_true',
default=cfme_data.get("bugzilla", {}).get("enabled", False),
dest='bugzilla',
help='Enable Bugzilla support.')
group.addoption('--bugzilla-url',
action='store',
default=cfme_data.get("bugzilla", {}).get("url", None),
dest='bugzilla_url',
help='Bugzilla XMLRPC url.')
cr_root = cfme_data.get("bugzilla", {}).get("credentials", None)
group.addoption('--bugzilla-user',
action='store',
default=credentials.get(cr_root, {}).get("username", None),
dest='bugzilla_user',
help='Bugzilla user id.')
group.addoption('--bugzilla-password',
action='store',
default=credentials.get(cr_root, {}).get("password", None),
dest='bugzilla_password',
help='Bugzilla password.')
def exists(self):
sel.force_navigate('clouds_tenants')
provider_name = cfme_data.get('management_systems',
{})[self.provider_key]['name']
res = list_page.tenant_table.find_row_by_cells({
'Name':
self.name,
'Cloud Provider':
provider_name
})
if res:
return True
else:
return False
def pytest_generate_tests(metafunc):
arg_names = "provider", "provider_data", "original_provider_key"
arg_values = []
arg_ids = []
for provider_key, provider in cfme_data.get("management_systems", {}).iteritems():
if provider["type"] != "virtualcenter":
continue
hosts = provider.get("hosts", [])
if not hosts:
continue
version = provider.get("version", None)
if version is None:
# No version, no test
continue
if Version(version) < "5.0":
# Ignore lesser than 5
continue
host = random.choice(hosts)
creds = credentials[host["credentials"]]
ip_address = resolve_hostname(host["name"])
cred = VMwareProvider.Credential(
principal=creds["username"],
secret=creds["password"],
verify_secret=creds["password"]
)
# Mock provider data
provider_data = {}
provider_data.update(provider)
provider_data["name"] = host["name"]
provider_data["hostname"] = host["name"]
provider_data["ipaddress"] = ip_address
provider_data["credentials"] = host["credentials"]
provider_data.pop("host_provisioning", None)
provider_data["hosts"] = [host]
provider_data["discovery_range"] = {}
provider_data["discovery_range"]["start"] = ip_address
provider_data["discovery_range"]["end"] = ip_address
host_provider = VMwareProvider(
name=host["name"],
hostname=host["name"],
ip_address=ip_address,
credentials={'default': cred},
provider_data=provider_data,
)
arg_values.append([host_provider, provider_data, provider_key])
arg_ids.append("{}/random_host".format(provider_key))
metafunc.parametrize(arg_names, arg_values, ids=arg_ids, scope="module")
def pytest_generate_tests(metafunc):
# Filter out providers without provisioning data or hosts defined
argnames, argvalues, idlist = testgen.providers_by_class(
metafunc, [InfraProvider],
required_fields=[['provisioning', 'pxe_server'],
['provisioning', 'pxe_image'],
['provisioning', 'pxe_image_type'],
['provisioning', 'pxe_kickstart'],
['provisioning', 'pxe_template'],
['provisioning', 'datastore'],
['provisioning', 'host'],
['provisioning', 'pxe_root_password'],
['provisioning', 'vlan']])
pargnames, pargvalues, pidlist = testgen.pxe_servers(metafunc)
argnames = argnames + ['pxe_server', 'pxe_cust_template']
pxe_server_names = [pval[0] for pval in pargvalues]
new_idlist = []
new_argvalues = []
for i, argvalue_tuple in enumerate(argvalues):
args = dict(zip(argnames, argvalue_tuple))
provider = args['provider']
if provider.one_of(SCVMMProvider):
continue
provisioning_data = provider.data['provisioning']
pxe_server_name = provisioning_data['pxe_server']
if pxe_server_name not in pxe_server_names:
continue
pxe_cust_template = provisioning_data['pxe_kickstart']
if pxe_cust_template not in cfme_data.get('customization_templates',
{}).keys():
continue
argvalues[i].append(get_pxe_server_from_config(pxe_server_name))
argvalues[i].append(get_template_from_config(pxe_cust_template))
new_idlist.append(idlist[i])
new_argvalues.append(argvalues[i])
testgen.parametrize(metafunc,
argnames,
new_argvalues,
ids=new_idlist,
scope="module")
def pytest_collection_modifyitems(session, config, items):
if not config.getvalue("bugzilla"):
return
loose = cfme_data.get("bugzilla", {}).get("loose", [])
if isinstance(loose, basestring):
loose = [i.strip() for i in loose.strip().split(",")]
terminalreporter = reporter(config)
terminalreporter.write("\nChecking bugs in Bugzilla...\n")
bugz = _bz.Bugzilla(
url=config.getvalue("bugzilla_url"),
user=config.getvalue("bugzilla_user"),
password=config.getvalue("bugzilla_password"))
progress = ("-", "\\", "|", "/") # Very simple eye-candy to not pollute tests output
progressbar = 0
last_line_length = 0
try:
for item in filter(lambda item: item.get_marker("bugzilla") is not None, items):
marker = item.get_marker("bugzilla")
item._bugzilla = bugz
terminalreporter.write("\r{}".format(last_line_length * " "))
terminalreporter.write("\r{}: {}".format(progress[progressbar], item.name))
progressbar = (progressbar + 1) % len(progress)
last_line_length = 3 + len(item.name)
item._bugzilla_bugs = resolve_bugs(bugz, loose, *marker.args)
item._skip_func = kwargify(marker.kwargs.get("skip_when", None))
item._xfail_func = kwargify(marker.kwargs.get("xfail_when", None))
item._unskip_dict = {}
for bug_id, function in marker.kwargs.get("unskip", {}).iteritems():
item._unskip_dict[bug_id] = kwargify(function)
terminalreporter.write("\r{} bugs retrieved\n".format(len(_bugs_cache)))
terminalreporter.write("All bugs summary:\n")
for bug_id, bug in _bugs_cache.iteritems():
terminalreporter.write("#{} {} {}\n".format(bug_id, bug.status, bug.summary))
except xmlrpclib.Fault as exception:
# It can happen that the user account does not have required rights.
if exception.faultCode == 102:
terminalreporter.write("\n\n======= !!!BAILING OUT. NOT ENOUGH RIGHTS!!! =======\n")
# remove any possible bugzilla markings in the test items so that does not get tested
for item in filter(lambda item: item.get_marker("bugzilla") is not None, items):
if hasattr(item, "_bugzilla"):
delattr(item, "_bugzilla")
if hasattr(item, "_bugzilla_bugs"):
delattr(item, "_bugzilla_bugs")
terminalreporter.write("======= !!!BUGZILLA INTEGRATION DISABLED!!! =======\n")
def pytest_generate_tests(metafunc):
# Filter out providers without provisioning data or hosts defined
argnames, argvalues, idlist = testgen.infra_providers(
metafunc, 'provisioning')
pargnames, pargvalues, pidlist = testgen.pxe_servers(metafunc)
argnames = argnames + ['pxe_server', 'pxe_cust_template']
pxe_server_names = [pval[0] for pval in pargvalues]
new_idlist = []
new_argvalues = []
for i, argvalue_tuple in enumerate(argvalues):
args = dict(zip(argnames, argvalue_tuple))
if not args['provisioning']:
# No provisioning data available
continue
if args['provider'].type == "scvmm":
continue
# required keys should be a subset of the dict keys set
if not {
'pxe_template', 'host', 'datastore', 'pxe_server', 'pxe_image',
'pxe_kickstart', 'pxe_root_password', 'pxe_image_type', 'vlan'
}.issubset(args['provisioning'].viewkeys()):
# Need all for template provisioning
continue
pxe_server_name = args['provisioning']['pxe_server']
if pxe_server_name not in pxe_server_names:
continue
pxe_cust_template = args['provisioning']['pxe_kickstart']
if pxe_cust_template not in cfme_data.get('customization_templates',
{}).keys():
continue
argvalues[i].append(get_pxe_server_from_config(pxe_server_name))
argvalues[i].append(get_template_from_config(pxe_cust_template))
new_idlist.append(idlist[i])
new_argvalues.append(argvalues[i])
testgen.parametrize(metafunc,
argnames,
new_argvalues,
ids=new_idlist,
scope="module")
