def check_user_roles(self, roles):
"""Check if roles provided by user (or the default one) exist.
:param roles: value returned by roles_client.list_roles
:type roles: dict
:return: List of the existing roles given by user (or by defaults)
:rtype: list
"""
roles_names = [r['name'] for r in roles['roles']]
user_roles = self._conf.get('auth', 'tempest_roles').split(',')
available_roles = []
for r in user_roles:
if r in roles_names:
available_roles.append(r)
else:
LOG.debug("Provided %s role is not present in the system.", r)
if len(available_roles) == 0:
# try 'member' or 'Member', they might present in a system
if 'member' in roles_names:
self._conf.set('auth', 'tempest_roles', 'member')
elif 'Member' in roles_names:
self._conf.set('auth', 'tempest_roles', 'Member')
else:
LOG.debug("Setting auth.tempest_roles to an empty list "
"because none of the provided roles exists.")
self._conf.set('auth', 'tempest_roles', "")
return available_roles
def do_get(self, url, top_level=False, top_level_path=""):
parts = list(urllib.parse.urlparse(url))
# 2 is the path offset
if top_level:
parts[2] = '/' + top_level_path
parts[2] = MULTIPLE_SLASH.sub('/', parts[2])
url = urllib.parse.urlunparse(parts)
try:
if self.disable_ssl_validation:
urllib3.disable_warnings()
http = urllib3.PoolManager(cert_reqs='CERT_NONE')
else:
http = urllib3.PoolManager()
r = http.request('GET', url, headers=self.headers)
except Exception as e:
LOG.error("Request on service '%s' with url '%s' failed",
self.s_type, url)
raise e
if r.status == 403:
raise exceptions.Forbidden("Request on service '%s' with url '%s' "
"failed with code 403" %
(self.s_type, url))
if r.status >= 400:
raise ServiceError("Request on service '%s' with url '%s' failed"
" with code %d" % (self.s_type, url, r.status))
return r.data.decode('utf-8')
def set_identity_v3_extensions(self):
"""Returns discovered identity v3 extensions
As keystone V3 uses a JSON Home to store the extensions.
This method implements a different discovery method.
:return: A list with the discovered extensions
"""
try:
r = requests.get(self.service_url,
verify=False,
headers={'Accept': 'application/json-home'})
# check for http status
r.raise_for_status()
except requests.exceptions.HTTPError:
LOG.warning(
"Request on service '%s' with url '%s' failed, "
"checking for v3", 'identity', self.service_url)
if 'v3' not in self.service_url:
self.service_url = self.service_url + '/v3'
r = requests.get(self.service_url,
verify=False,
headers={'Accept': 'application/json-home'})
ext_h = 'https://docs.openstack.org/api/openstack-identity/3/ext/'
content = r.content.decode('utf-8')
res = [x for x in json.loads(content)['resources'].keys()]
ext = [ex for ex in res if 'ext' in ex]
ext = [str(e).replace(ext_h, '').split('/')[0] for e in ext]
self.extensions_v3 = list(set(ext))
def set_cloud_config_values(non_admin, cloud_creds, conf):
"""Set values from client's cloud config file.
Set admin and non-admin credentials and uri from cloud credentials.
Note: the values may be later overridden by values specified in CLI.
:type non_admin: Boolean
:param cloud_creds: auth data from openstacksdk
:type cloud_creds: dict
:param conf: TempestConf object
"""
try:
if non_admin:
# Tempest doesn't have non-admin credentials, but we're gonna
# keep them under identity for future usage
conf.set('identity', 'username', cloud_creds['username'])
conf.set('identity', 'project_name', cloud_creds['project_name'])
conf.set('identity', 'password', cloud_creds['password'])
else:
# admin credentials are under auth section
conf.set('auth', 'admin_username', cloud_creds['username'])
conf.set('auth', 'admin_project_name', cloud_creds['project_name'])
conf.set('auth', 'admin_password', cloud_creds['password'])
conf.set('identity', 'uri', cloud_creds['auth_url'])
if 'region_name' in cloud_creds:
conf.set('identity', 'region', cloud_creds['region_name'])
except cfg.NoSuchOptError:
LOG.warning(
'Could not load some identity options from cloud config file')
def set_default_tempest_options(self, conf):
try:
uri = conf.get('identity', 'uri')
if 'v3' not in uri:
return
sec = 'heat_plugin'
# Tempest doesn't differentiate between admin or demo creds anymore
username = conf.get('auth', 'admin_username')
password = conf.get('auth', 'admin_password')
conf.set(sec, 'username', username)
conf.set(sec, 'password', password)
conf.set(sec, 'admin_username', username)
conf.set(sec, 'admin_password', password)
conf.set(sec, 'project_name', conf.get('identity', 'project_name'))
conf.set(sec, 'region', conf.get('identity', 'region'))
conf.set(sec, 'auth_url', uri)
v = '3' if conf.get('identity', 'auth_version') == 'v3' else '2'
conf.set(sec, 'auth_version', v)
domain_name = conf.get('auth', 'admin_domain_name')
conf.set(sec, 'project_domain_name', domain_name)
conf.set(sec, 'user_domain_name', domain_name)
conf.set(sec, 'image_ssh_user', 'root')
conf.set(sec, 'network_for_ssh', 'public')
conf.set(sec, 'fixed_network_name', 'public')
# should be set to True if using self-signed SSL certificates which
# is a general case
conf.set(sec, 'disable_ssl_certificate_validation', 'True')
except configparser.NoOptionError:
LOG.warning("Be aware that an option required for "
"heat_tempest_plugin cannot be set!")
def _check_health_check(self, path):
try:
self.client.accounts.skip_path()
resp, _ = self.client.accounts.get(path, {})
return resp['status'] == '200'
except Exception as e:
LOG.warning('Healthcheck API not discovered giving %s', e)
return False
def set_logging(debug, verbose):
"""Set logging based on the arguments.
:type debug: Boolean
:type verbose: Boolean
"""
log_format = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
logging.basicConfig(format=log_format)
if debug:
LOG.setLevel(logging.DEBUG)
elif verbose:
LOG.setLevel(logging.INFO)
def _supplied_network(self):
LOG.info("Looking for existing network id: {0}"
"".format(self._public_network_id))
# check if network exists
network_list = self.client.list_networks()
for network in network_list['networks']:
if network['id'] == self._public_network_id:
self._public_network_name = network['name']
break
else:
raise ValueError('provided network id: {0} was not found.'
''.format(self._public_network_id))
def read_deployer_input(deployer_input_file, conf):
"""Read deployer-input file and set values in conf accordingly.
:param deployer_input_file: Path to the deployer inut file
:type deployer_input_file: String
:param conf: TempestConf object
"""
LOG.info("Adding options from deployer-input file '%s'",
deployer_input_file)
if six.PY3:
deployer_input = configparser.ConfigParser()
else:
deployer_input = configparser.SafeConfigParser()
deployer_input.read(deployer_input_file)
for section in deployer_input.sections():
# There are no deployer input options in DEFAULT
for (key, value) in deployer_input.items(section):
conf.set(section, key, value, priority=True)
def give_role_to_user(self, username, role_name, role_required=True):
"""Give the user a role in the project.
:type username: string
:type role_name: string
:type role_required: boolean
"""
project_name = self._conf.get('identity', 'project_name')
proj_id = self.projects_client.get_project_by_name(project_name)['id']
users = self.users_client.list_users()
user_ids = [u['id'] for u in users['users'] if u['name'] == username]
user_id = user_ids[0]
roles = self.roles_client.list_roles()
self.check_user_roles(roles)
role_ids = [r['id'] for r in roles['roles'] if r['name'] == role_name]
if not role_ids:
if role_required:
raise Exception("required role %s not found" % role_name)
LOG.debug("%s role not required", role_name)
return
role_id = role_ids[0]
try:
self.roles_client.create_user_role_on_project(
proj_id, user_id, role_id)
LOG.debug("User '%s' was given the '%s' role in project '%s'",
username, role_name, project_name)
except exceptions.Conflict:
LOG.debug(
"(no change) User '%s' already has the '%s' role in"
" project '%s'", username, role_name, project_name)
def create_user_with_project(self, username, password, project_name):
"""Create a user and a project if it doesn't exist.
:type username: string
:type password: string
:type project_name: string
"""
LOG.info("Creating user '%s' with project '%s' and password '%s'",
username, project_name, password)
project_description = "Project for Tempest %s user" % username
email = "*****@*****.**" % username
# create a project
try:
self.projects_client.create_project(
name=project_name, description=project_description)
except exceptions.Conflict:
LOG.info("(no change) Project '%s' already exists", project_name)
proj_id = self.projects_client.get_project_by_name(project_name)['id']
params = {
'name': username,
'password': password,
'tenantId': proj_id,
'email': email
}
# create a user
try:
self.users_client.create_user(**params)
except exceptions.Conflict:
LOG.info("User '%s' already exists.", username)
def load_basic_defaults(conf):
"""Load basic default options into conf file.
:type conf: TempestConf object
"""
LOG.debug("Setting basic default values")
default_values = {
"DEFAULT": [("debug", "true"), ("use_stderr", "false"),
("log_file", "tempest.log")],
"identity": [("username", "demo_tempestconf"), ("password", "secrete"),
("project_name", "demo"),
("alt_username", "alt_demo_tempestconf"),
("alt_password", "secrete"),
("alt_project_name", "alt_demo"),
("disable_ssl_certificate_validation", "true")],
"scenario": [("img_dir", "etc")],
"auth": [("tempest_roles", "_member_"), ("admin_username", "admin"),
("admin_project_name", "admin"),
("admin_domain_name", "Default")],
"object-storage": [("reseller_admin_role", "ResellerAdmin")],
"oslo-concurrency": [("lock_path", "/tmp")],
"compute-feature-enabled": [
# Default deployment does not use shared storage
("live_migration", "false"),
("live_migrate_paused_instances", "true"),
("preserve_ports", "true")
],
"network-feature-enabled": [("ipv6_subnet_attributes", "true")]
}
for section in default_values.keys():
if section != "DEFAULT" and not conf.has_section(section):
conf.add_section(section)
for key, value in default_values[section]:
if not conf.has_option(section, key):
conf.set(section, key, value)
def _discover_network(self):
LOG.info("No network supplied, trying auto discover for network")
network_list = self.client.list_networks()
for network in network_list['networks']:
if network['router:external'] and network['subnets']:
LOG.info("Found network, using: {0}".format(network['id']))
self._public_network_id = network['id']
self._public_network_name = network['name']
break
# Couldn't find an existing external network
else:
LOG.error("No external networks found. "
"Please note that any test that relies on external "
"connectivity would most likely fail.")
def list_create_roles(self, conf, client):
try:
roles = client.list_roles()['roles']
for section_key in ["operator_role", "reseller_admin_role"]:
key_value = conf.get_defaulted("object-storage", section_key)
if key_value not in [r['name'] for r in roles]:
LOG.info("Creating %s role", key_value)
try:
client.create_role(name=key_value)
except exceptions.Conflict:
LOG.info("Role %s already exists", key_value)
conf.set('object-storage', 'operator_role', 'admin')
except exceptions.Forbidden:
LOG.info("Roles can't be listed or created. The user doesn't have "
"permissions.")
# If is not admin, we set the operator_role to Member
# otherwise we set to admin
conf.set('object-storage', 'operator_role', 'Member')
def config_tempest(**kwargs):
# convert a list of remove values to a dict
remove = parse_values_to_remove(kwargs.get('remove', []))
add = parse_values_to_append(kwargs.get('append', []))
set_logging(kwargs.get('debug', False), kwargs.get('verbose', False))
accounts_path = kwargs.get('test_accounts')
if kwargs.get('create_accounts_file') is not None:
accounts_path = kwargs.get('create_accounts_file')
conf = TempestConf(write_credentials=accounts_path is None)
set_options(conf, kwargs.get('deployer_input'),
kwargs.get('non_admin', False),
kwargs.get('image_path', C.DEFAULT_IMAGE),
kwargs.get('overrides', []), accounts_path,
kwargs.get('cloud_creds'))
credentials = Credentials(conf, not kwargs.get('non_admin', False))
clients = ClientManager(conf, credentials)
services = Services(clients, conf, credentials)
if kwargs.get('create', False) and kwargs.get('test_accounts') is None:
users = Users(clients.projects, clients.roles, clients.users, conf)
users.create_tempest_users()
if services.is_service(**{"type": "compute"}):
flavors = Flavors(clients.flavors,
kwargs.get('create', False),
conf,
kwargs.get('flavor_min_mem', C.DEFAULT_FLAVOR_RAM),
kwargs.get('flavor_min_disk', C.DEFAULT_FLAVOR_DISK),
no_rng=kwargs.get('no_rng', False))
flavors.create_tempest_flavors()
if services.is_service(**{"type": "image"}):
image = services.get_service('image')
image.set_image_preferences(kwargs.get('image_disk_format',
C.DEFAULT_IMAGE_FORMAT),
kwargs.get('non_admin', False),
no_rng=kwargs.get('no_rng', False),
convert=kwargs.get('convert_to_raw',
False))
image.create_tempest_images(conf)
if services.is_service(**{"type": "network"}):
network = services.get_service("network")
network.create_tempest_networks(conf, kwargs.get('network_id'))
services.post_configuration()
services.set_supported_api_versions()
services.set_service_extensions()
if accounts_path is not None and kwargs.get('test_accounts') is None:
LOG.info("Creating an accounts.yaml file in: %s", accounts_path)
accounts.create_accounts_file(kwargs.get('create', False),
accounts_path, conf)
# remove all unwanted values if were specified
if remove != {}:
LOG.info("Removing configuration: %s", str(remove))
conf.remove_values(remove)
if add != {}:
LOG.info("Adding configuration: %s", str(add))
conf.append_values(add)
out_path = kwargs.get('out', 'etc/tempest.conf')
conf.write(out_path)
def set_options(conf,
deployer_input,
non_admin,
image_path,
overrides=[],
accounts_path=None,
cloud_creds=None,
no_default_deployer=False):
"""Set options in conf provided by different source.
1. read the default values
2. read a file provided by --deployer-input argument
3. read default DEPLOYER_INPUT if --no-deployer-input is False and no
deployer_input was passed
4. set values from client's config (openstacksdk support) if provided
5. set overrides - may override values which were set in the steps above
:param conf: TempestConf object
:param deployer_input: Path to the deployer inut file
:type deployer_input: string
:type non_admin: boolean
:param image_path: An image to be uploaded to glance
:type image_path: string
:param overrides: list of tuples: [(section, key, value)]
:type overrides: list
:param accounts_path: A path where accounts.yaml is or will be created.
:type accounts_path: string
:param cloud_creds: Cloud credentials from client's config
:type cloud_creds: dict
"""
load_basic_defaults(conf)
# image.image_path is a python-tempestconf option which defines which
# image will be uploaded to glance
conf.set('image', 'image_path', image_path)
if deployer_input and os.path.isfile(deployer_input):
LOG.info("Reading deployer input from file {}".format(deployer_input))
read_deployer_input(deployer_input, conf)
elif os.path.isfile(C.DEPLOYER_INPUT) and not no_default_deployer:
LOG.info("Reading deployer input from file {}".format(
C.DEPLOYER_INPUT))
read_deployer_input(C.DEPLOYER_INPUT, conf)
if non_admin:
# non admin, so delete auth admin values which were set
# in load_basic_defaults method
conf.set("auth", "admin_username", "")
conf.set("auth", "admin_project_name", "")
conf.set("auth", "admin_password", "")
conf.set("auth", "use_dynamic_credentials", "False", priority=True)
# get and set auth data from client's config
if cloud_creds:
set_cloud_config_values(non_admin, cloud_creds, conf)
if accounts_path:
# new way for running using accounts file
conf.set("auth", "use_dynamic_credentials", "False", priority=True)
conf.set("auth", "test_accounts_file", os.path.abspath(accounts_path))
# set overrides - values specified in CLI
for section, key, value in overrides:
conf.set(section, key, value, priority=True)
uri = conf.get("identity", "uri")
if "v3" in uri:
conf.set("identity", "auth_version", "v3")
conf.set("identity", "uri_v3", uri)
else:
# TODO(arxcruz) make a check if v3 is enabled
conf.set("identity", "uri_v3", uri.replace("v2.0", "v3"))
def create_tempest_networks(self, conf, network_id):
LOG.info("Setting up network")
self.client = self.client.get_neutron_client()
self.create_tempest_networks_neutron(conf, network_id)
