def get_hash_dict(cls, accounts, admin_role,
object_storage_operator_role=None,
object_storage_reseller_admin_role=None):
hash_dict = {'roles': {}, 'creds': {}, 'networks': {}}
# Loop over the accounts read from the yaml file
for account in accounts:
roles = []
types = []
resources = []
if 'roles' in account:
roles = account.pop('roles')
if 'types' in account:
types = account.pop('types')
if 'resources' in account:
resources = account.pop('resources')
temp_hash = hashlib.md5()
account_for_hash = dict((k, v) for (k, v) in account.items()
if k in cls.HASH_CRED_FIELDS)
temp_hash.update(six.text_type(account_for_hash).encode('utf-8'))
temp_hash_key = temp_hash.hexdigest()
hash_dict['creds'][temp_hash_key] = account
for role in roles:
hash_dict = cls._append_role(role, temp_hash_key,
hash_dict)
# If types are set for the account append the matching role
# subdict with the hash
for type in types:
if type == 'admin':
hash_dict = cls._append_role(admin_role, temp_hash_key,
hash_dict)
elif type == 'operator':
if object_storage_operator_role:
hash_dict = cls._append_role(
object_storage_operator_role, temp_hash_key,
hash_dict)
else:
msg = ("Type 'operator' configured, but no "
"object_storage_operator_role specified")
raise lib_exc.InvalidCredentials(msg)
elif type == 'reseller_admin':
if object_storage_reseller_admin_role:
hash_dict = cls._append_role(
object_storage_reseller_admin_role,
temp_hash_key,
hash_dict)
else:
msg = ("Type 'reseller_admin' configured, but no "
"object_storage_reseller_admin_role specified")
raise lib_exc.InvalidCredentials(msg)
# Populate the network subdict
for resource in resources:
if resource == 'network':
hash_dict['networks'][temp_hash_key] = resources[resource]
else:
LOG.warning(
'Unknown resource type %s, ignoring this field',
resource
)
return hash_dict
def _get_creds(self, roles=None):
if self.use_default_creds:
raise lib_exc.InvalidCredentials("Account file %s doesn't exist" %
self.test_accounts_file)
useable_hashes = self._get_match_hash_list(roles)
if len(useable_hashes) == 0:
msg = 'No users configured for type/roles %s' % roles
raise lib_exc.InvalidCredentials(msg)
free_hash = self._get_free_hash(useable_hashes)
clean_creds = self._sanitize_creds(self.hash_dict['creds'][free_hash])
LOG.info('%s allocated creds:\n%s' % (self.name, clean_creds))
return self._wrap_creds_with_network(free_hash)
def _apply_credentials(self, attr):
for (key1, key2) in self.COLLISIONS:
val1 = attr.get(key1)
val2 = attr.get(key2)
if val1 and val2 and val1 != val2:
msg = ('Cannot have conflicting values for %s and %s' %
(key1, key2))
raise exceptions.InvalidCredentials(msg)
for key in attr:
if key in self.ATTRIBUTES:
setattr(self, key, attr[key])
else:
msg = '%s is not a valid attr for %s' % (key, self.__class__)
raise exceptions.InvalidCredentials(msg)
def _get_match_hash_list(self, roles=None):
hashes = []
if roles:
# Loop over all the creds for each role in the subdict and generate
# a list of cred lists for each role
for role in roles:
temp_hashes = self.hash_dict['roles'].get(role, None)
if not temp_hashes:
raise lib_exc.InvalidCredentials(
"No credentials with role: %s specified in the "
"accounts "
"file" % role)
hashes.append(temp_hashes)
# Take the list of lists and do a boolean and between each list to
# find the creds which fall under all the specified roles
temp_list = set(hashes[0])
for hash_list in hashes[1:]:
temp_list = temp_list & set(hash_list)
hashes = temp_list
else:
hashes = self.hash_dict['creds'].keys()
# NOTE(mtreinish): admin is a special case because of the increased
# privilege set which could potentially cause issues on tests where
# that is not expected. So unless the admin role isn't specified do
# not allocate admin.
admin_hashes = self.hash_dict['roles'].get(self.admin_role, None)
if ((not roles or self.admin_role not in roles) and admin_hashes):
useable_hashes = [x for x in hashes if x not in admin_hashes]
else:
useable_hashes = hashes
return useable_hashes
def __init__(self, credentials, scope='project'):
"""Auth provider __init__
:param credentials: credentials for authentication
:param scope: the default scope to be used by the credential providers
when requesting a token. Valid values depend on the
AuthProvider class implementation, and are defined in
the set SCOPES. Default value is 'project'.
"""
if self.check_credentials(credentials):
self.credentials = credentials
else:
if isinstance(credentials, Credentials):
password = credentials.get('password')
message = "Credentials are: " + str(credentials)
if password is None:
message += " Password is not defined."
else:
message += " Password is defined."
raise exceptions.InvalidCredentials(message)
else:
raise TypeError("credentials object is of type %s, which is"
" not a valid Credentials object type." %
credentials.__class__.__name__)
self._scope = None
self.scope = scope
self.cache = None
self.alt_auth_data = None
self.alt_part = None
def is_multi_user(self):
# Default credentials is not a valid option with locking Account
if self.use_default_creds:
raise lib_exc.InvalidCredentials("Account file %s doesn't exist" %
self.test_accounts_file)
else:
return len(self.hash_dict['creds']) > 1
def __init__(self,
identity_version,
test_accounts_file,
accounts_lock_dir,
name=None,
credentials_domain=None,
admin_role=None,
object_storage_operator_role=None,
object_storage_reseller_admin_role=None,
identity_uri=None):
super(PreProvisionedCredentialProvider,
self).__init__(identity_version=identity_version,
name=name,
admin_role=admin_role,
credentials_domain=credentials_domain,
identity_uri=identity_uri)
self.test_accounts_file = test_accounts_file
if test_accounts_file:
accounts = read_accounts_yaml(self.test_accounts_file)
else:
raise lib_exc.InvalidCredentials("No accounts file specified")
self.hash_dict = self.get_hash_dict(
accounts, admin_role, object_storage_operator_role,
object_storage_reseller_admin_role)
self.accounts_dir = accounts_lock_dir
self._creds = {}
def _apply_credentials(self, attr):
for key in attr.keys():
if key in self.ATTRIBUTES:
setattr(self, key, attr[key])
else:
msg = '%s is not a valid attr for %s' % (key, self.__class__)
raise exceptions.InvalidCredentials(msg)
def _get_creds(self, roles=None):
useable_hashes = self._get_match_hash_list(roles)
if len(useable_hashes) == 0:
msg = 'No users configured for type/roles %s' % roles
raise lib_exc.InvalidCredentials(msg)
free_hash = self._get_free_hash(useable_hashes)
clean_creds = self._sanitize_creds(self.hash_dict['creds'][free_hash])
LOG.info('%s allocated creds:\n%s', self.name, clean_creds)
return self._wrap_creds_with_network(free_hash)
def _apply_credentials(self, attr):
for (key1, key2) in COLLISIONS:
val1 = attr.get(key1)
val2 = attr.get(key2)
if val1 and val2 and val1 != val2:
msg = ('Cannot have conflicting values for %s and %s' %
(key1, key2))
raise exceptions.InvalidCredentials(msg)
super(KeystoneV3Credentials, self)._apply_credentials(attr)
def __init__(self,
identity_version,
test_accounts_file,
accounts_lock_dir,
name=None,
credentials_domain=None,
admin_role=None,
object_storage_operator_role=None,
object_storage_reseller_admin_role=None,
identity_uri=None):
"""Credentials provider using pre-provisioned accounts
This credentials provider loads the details of pre-provisioned
accounts from a YAML file, in the format specified by
`etc/accounts.yaml.sample`. It locks accounts while in use, using the
external locking mechanism, allowing for multiple python processes
to share a single account file, and thus running tests in parallel.
The accounts_lock_dir must be generated using `lockutils.get_lock_path`
from the oslo.concurrency library. For instance:
accounts_lock_dir = os.path.join(lockutils.get_lock_path(CONF),
'test_accounts')
Role names for object storage are optional as long as the
`operator` and `reseller_admin` credential types are not used in the
accounts file.
:param identity_version: identity version of the credentials
:param admin_role: name of the admin role
:param test_accounts_file: path to the accounts YAML file
:param accounts_lock_dir: the directory for external locking
:param name: name of the hash file (optional)
:param credentials_domain: name of the domain credentials belong to
(if no domain is configured)
:param object_storage_operator_role: name of the role
:param object_storage_reseller_admin_role: name of the role
:param identity_uri: Identity URI of the target cloud
"""
super(PreProvisionedCredentialProvider,
self).__init__(identity_version=identity_version,
name=name,
admin_role=admin_role,
credentials_domain=credentials_domain,
identity_uri=identity_uri)
self.test_accounts_file = test_accounts_file
if test_accounts_file:
accounts = read_accounts_yaml(self.test_accounts_file)
else:
raise lib_exc.InvalidCredentials("No accounts file specified")
self.hash_dict = self.get_hash_dict(
accounts, admin_role, object_storage_operator_role,
object_storage_reseller_admin_role)
self.accounts_dir = accounts_lock_dir
self._creds = {}
def get_auth_provider(credentials, pre_auth=False, scope='project'):
# kwargs for auth provider match the common ones used by service clients
default_params = config.service_client_config()
if credentials is None:
raise lib_exc.InvalidCredentials('Credentials must be specified')
auth_provider_class, auth_url = get_auth_provider_class(credentials)
_auth_provider = auth_provider_class(credentials,
auth_url,
scope=scope,
**default_params)
if pre_auth:
_auth_provider.set_auth()
return _auth_provider
def __init__(self, identity_client, projects_client, users_client,
roles_client, domains_client, domain_name):
super(V3CredsClient, self).__init__(identity_client, projects_client,
users_client, roles_client)
self.domains_client = domains_client
try:
# Domain names must be unique, in any case a list is returned,
# selecting the first (and only) element
self.creds_domain = self.domains_client.list_domains(
name=domain_name)['domains'][0]
except lib_exc.NotFound:
# TODO(andrea) we could probably create the domain on the fly
msg = "Requested domain %s could not be found" % domain_name
raise lib_exc.InvalidCredentials(msg)
def get_client_manager(cls,
credential_type=None,
roles=None,
force_new=None,
scope=None):
"""Returns an OpenStack client manager
Returns an OpenStack client manager based on either credential_type
or a list of roles. If neither is specified, it defaults to
credential_type 'primary'
:param credential_type: string - primary, alt or admin
:param roles: list of roles
:param scope: scope for the test user
:returns: the created client manager
:raises skipException: if the requested credentials are not available
"""
if all([roles, credential_type]):
msg = "Cannot get credentials by type and roles at the same time"
raise ValueError(msg)
if not any([roles, credential_type]):
credential_type = 'primary'
cred_provider = cls._get_credentials_provider()
if roles:
for role in roles:
if not cred_provider.is_role_available(role):
skip_msg = (
"%s skipped because the configured credential provider"
" is not able to provide credentials with the %s role "
"assigned." % (cls.__name__, role))
raise cls.skipException(skip_msg)
params = dict(roles=roles, scope=scope)
if force_new is not None:
params.update(force_new=force_new)
creds = cred_provider.get_creds_by_roles(**params)
else:
credentials_method = 'get_%s_creds' % credential_type
if hasattr(cred_provider, credentials_method):
creds = getattr(cred_provider, credentials_method)()
else:
raise lib_exc.InvalidCredentials(
"Invalid credentials type %s" % credential_type)
manager = cls.client_manager(credentials=creds.credentials)
# NOTE(andreaf) Ensure credentials have user and project id fields.
# It may not be the case when using pre-provisioned credentials.
manager.auth_provider.set_auth()
return manager
def _get_free_hash(self, hashes):
# Cast as a list because in some edge cases a set will be passed in
hashes = list(hashes)
if not os.path.isdir(self.accounts_dir):
os.mkdir(self.accounts_dir)
# Create File from first hash (since none are in use)
self._create_hash_file(hashes[0])
return hashes[0]
names = []
for _hash in hashes:
res = self._create_hash_file(_hash)
if res:
return _hash
else:
path = os.path.join(os.path.join(self.accounts_dir, _hash))
with open(path, 'r') as fd:
names.append(fd.read())
msg = ('Insufficient number of users provided. %s have allocated all '
'the credentials for this allocation request' % ','.join(names))
raise lib_exc.InvalidCredentials(msg)
def _get_match_hash_list(self, roles=None):
temp_hashes = super(ExactRoleMatchingPreProvisionedCredentialProvider,
self)._get_match_hash_list(roles=roles)
if roles:
# search through the roles we don't want because if we find a user
# in those roles, we don't want the user. We need to make an exact
# match
unwanted_roles = set(self.hash_dict['roles'].keys()) - set(roles)
for role in unwanted_roles:
unwanted_users = self.hash_dict['roles'].get(role)
users_to_remove = set(temp_hashes) & set(unwanted_users)
temp_hashes = list(set(temp_hashes) - set(users_to_remove))
if not temp_hashes:
raise lib_exc.InvalidCredentials(
"No credentials with exact roles %s specified in the "
"accounts file" % roles)
return temp_hashes
def __init__(self, credentials):
"""Auth provider __init__
:param credentials: credentials for authentication
"""
if self.check_credentials(credentials):
self.credentials = credentials
else:
if isinstance(credentials, Credentials):
password = credentials.get('password')
message = "Credentials are: " + str(credentials)
if password is None:
message += " Password is not defined."
else:
message += " Password is defined."
raise exceptions.InvalidCredentials(message)
else:
raise TypeError("credentials object is of type %s, which is"
" not a valid Credentials object type." %
credentials.__class__.__name__)
self.cache = None
self.alt_auth_data = None
self.alt_part = None
def __init__(self,
credentials,
identity_uri,
region=None,
scope='project',
disable_ssl_certificate_validation=True,
ca_certs=None,
trace_requests='',
client_parameters=None):
"""Service Clients provider
Instantiate a `ServiceClients` object, from a set of credentials and an
identity URI. The identity version is inferred from the credentials
object. Optionally auth scope can be provided.
A few parameters can be given a value which is applied as default
for all service clients: region, dscv, ca_certs, trace_requests.
Parameters dscv, ca_certs and trace_requests all apply to the auth
provider as well as any service clients provided by this manager.
Any other client parameter should be set via ClientsRegistry.
Client parameter used to be set via client_parameters, but this is
deprecated, and it is actually already not honoured
anymore: https://launchpad.net/bugs/1680915.
The list of available parameters is defined in the service clients
interfaces. For reference, most clients will accept 'region',
'service', 'endpoint_type', 'build_timeout' and 'build_interval', which
are all inherited from RestClient.
The `config` module in Tempest exposes an helper function
`service_client_config` that can be used to extract from configuration
a dictionary ready to be injected in kwargs.
Exceptions are:
- Token clients for 'identity' must be given an 'auth_url' parameter
- Volume client for 'volume' accepts 'default_volume_size'
- Servers client from 'compute' accepts 'enable_instance_password'
If Tempest configuration is used, parameters will be loaded in the
Registry automatically for all service client (Tempest stable ones
and plugins).
Examples::
identity_params = config.service_client_config('identity')
params = {
'identity': identity_params,
'compute': {'region': 'region2'}}
manager = lib_manager.Manager(
my_creds, identity_uri, client_parameters=params)
:param credentials: An instance of `auth.Credentials`
:param identity_uri: URI of the identity API. This should be a
mandatory parameter, and it will so soon.
:param region: Default value of region for service clients.
:param scope: default scope for tokens produced by the auth provider
:param disable_ssl_certificate_validation: Applies to auth and to all
service clients.
:param ca_certs: Applies to auth and to all service clients.
:param trace_requests: Applies to auth and to all service clients.
:param client_parameters: Dictionary with parameters for service
clients. Keys of the dictionary are the service client service
name, as declared in `service_clients.available_modules()` except
for the version. Values are dictionaries of parameters that are
going to be passed to all clients in the service client module.
"""
self._registered_services = set([])
self.credentials = credentials
self.identity_uri = identity_uri
if not identity_uri:
raise exceptions.InvalidCredentials(
'ServiceClients requires a non-empty identity_uri.')
self.region = region
# Check if passed or default credentials are valid
if not self.credentials.is_valid():
raise exceptions.InvalidCredentials()
# Get the identity classes matching the provided credentials
# TODO(andreaf) Define a new interface in Credentials to get
# the API version from an instance
identity = [
(k, auth.IDENTITY_VERSION[k][1])
for k in auth.IDENTITY_VERSION.keys()
if isinstance(self.credentials, auth.IDENTITY_VERSION[k][0])
]
# Zero matches or more than one are both not valid.
if len(identity) != 1:
raise exceptions.InvalidCredentials()
self.auth_version, auth_provider_class = identity[0]
self.dscv = disable_ssl_certificate_validation
self.ca_certs = ca_certs
self.trace_requests = trace_requests
# Creates an auth provider for the credentials
self.auth_provider = auth_provider_class(
self.credentials,
self.identity_uri,
scope=scope,
disable_ssl_certificate_validation=self.dscv,
ca_certs=self.ca_certs,
trace_requests=self.trace_requests)
# Setup some defaults for client parameters of registered services
client_parameters = client_parameters or {}
self.parameters = {}
# Parameters are provided for unversioned services
all_modules = available_modules() | _tempest_internal_modules()
unversioned_services = set([x.split('.')[0] for x in all_modules])
for service in unversioned_services:
self.parameters[service] = self._setup_parameters(
client_parameters.pop(service, {}))
# Check that no client parameters was supplied for unregistered clients
if client_parameters:
raise exceptions.UnknownServiceClient(
services=list(client_parameters.keys()))
# Register service clients from the registry (__tempest__ and plugins)
clients_registry = ClientsRegistry()
plugin_service_clients = clients_registry.get_service_clients()
registration_errors = []
for plugin in plugin_service_clients:
service_clients = plugin_service_clients[plugin]
# Each plugin returns a list of service client parameters
for service_client in service_clients:
# NOTE(andreaf) If a plugin cannot register, stop the
# registration process, log some details to help
# troubleshooting, and re-raise
try:
self.register_service_client_module(**service_client)
except Exception:
registration_errors.append(sys.exc_info())
LOG.exception(
'Failed to register service client from plugin %s '
'with parameters %s', plugin, service_client)
if registration_errors:
raise testtools.MultipleExceptions(*registration_errors)