def test_storing_null_domain_id_in_project_ref(self):
"""Test the special storage of domain_id=None in sql resource driver.
The resource driver uses a special value in place of None for domain_id
in the project record. This shouldn't escape the driver. Hence we test
the interface to ensure that you can store a domain_id of None, and
that any special value used inside the driver does not escape through
the interface.
"""
spoiler_project = unit.new_project_ref(
domain_id=CONF.identity.default_domain_id)
self.resource_api.create_project(spoiler_project['id'],
spoiler_project)
# First let's create a project with a None domain_id and make sure we
# can read it back.
project = unit.new_project_ref(domain_id=None, is_domain=True)
project = self.resource_api.create_project(project['id'], project)
ref = self.resource_api.get_project(project['id'])
self.assertDictEqual(project, ref)
# Can we get it by name?
ref = self.resource_api.get_project_by_name(project['name'], None)
self.assertDictEqual(project, ref)
# Can we filter for them - create a second domain to ensure we are
# testing the receipt of more than one.
project2 = unit.new_project_ref(domain_id=None, is_domain=True)
project2 = self.resource_api.create_project(project2['id'], project2)
hints = driver_hints.Hints()
hints.add_filter('domain_id', None)
refs = self.resource_api.list_projects(hints)
self.assertThat(refs, matchers.HasLength(2 + self.domain_count))
self.assertIn(project, refs)
self.assertIn(project2, refs)
# Can we update it?
project['name'] = uuid.uuid4().hex
self.resource_api.update_project(project['id'], project)
ref = self.resource_api.get_project(project['id'])
self.assertDictEqual(project, ref)
# Finally, make sure we can delete it
project['enabled'] = False
self.resource_api.update_project(project['id'], project)
self.resource_api.delete_project(project['id'])
self.assertRaises(exception.ProjectNotFound,
self.resource_api.get_project,
project['id'])
def validate_primary_key(self):
crypto, keys = credential_fernet.get_multi_fernet_keys()
primary_key_hash = credential_fernet.primary_key_hash(keys)
credentials = self.credential_api.driver.list_credentials(
driver_hints.Hints())
for credential in credentials:
if credential['key_hash'] != primary_key_hash:
msg = _('Unable to rotate credential keys because not all '
'credentials are encrypted with the primary key. '
'Please make sure all credentials have been encrypted '
'with the primary key using `keystone-manage '
'credential_migrate`.')
raise SystemExit(msg)
def list_users(self, domain_scope=None, hints=None):
driver = self._select_identity_driver(domain_scope)
hints = hints or driver_hints.Hints()
if driver.is_domain_aware():
# Force the domain_scope into the hint to ensure that we only get
# back domains for that scope.
self._ensure_domain_id_in_hints(hints, domain_scope)
else:
# We are effectively satisfying any domain_id filter by the above
# driver selection, so remove any such filter.
self._mark_domain_id_filter_satisfied(hints)
ref_list = driver.list_users(hints)
return self._set_domain_id_and_mapping(ref_list, domain_scope, driver,
mapping.EntityType.USER)
def pagination(context, hints=None):
"""Enhance Hints with SCIM pagination info (limit and offset)"""
q = context['query_string']
if hints is None:
hints = driver_hints.Hints()
try:
hints.scim_limit = q['count']
except KeyError:
pass
try:
hints.scim_offset = q['startIndex']
except KeyError:
pass
return hints
def _delete_access_rules_for_user(self, user_id, initiator=None):
"""Delete all access rules for a user.
:param str user_id: User ID
This is triggered when a user is deleted.
"""
access_rules = self.driver.list_access_rules_for_user(
user_id, driver_hints.Hints())
self.driver.delete_access_rules_for_user(user_id)
for rule in access_rules:
self.get_access_rule.invalidate(self, rule['id'])
notifications.Audit.deleted(self._ACCESS_RULE, rule['id'],
initiator)
def _normalize_role_list(self, trust_roles):
roles = [{'id': role['id']} for role in trust_roles if 'id' in role]
names = [role['name'] for role in trust_roles if 'id' not in role]
if len(names):
# Long way
for name in names:
hints = driver_hints.Hints()
hints.add_filter("name", name, case_sensitive=True)
found_roles = self.role_api.list_roles(hints)
if len(found_roles) == 1:
roles.append({'id': found_roles[0]['id']})
else:
raise exception.RoleNotFound(
_("role %s is not defined") % name)
return roles
def list_projects_for_user(self, user_id, hints=None):
# NOTE(henry-nash): In order to get a complete list of user projects,
# the driver will need to look at group assignments. To avoid cross
# calling between the assignment and identity driver we get the group
# list here and pass it in. The rest of the detailed logic of listing
# projects for a user is pushed down into the driver to enable
# optimization with the various backend technologies (SQL, LDAP etc.).
group_ids = self._get_group_ids_for_user_id(user_id)
project_ids = self.list_project_ids_for_user(
user_id, group_ids, hints or driver_hints.Hints())
if not CONF.os_inherit.enabled:
return self.resource_api.list_projects_from_ids(project_ids)
# Inherited roles are enabled, so check to see if this user has any
# inherited role (direct or group) on any parent project, in which
# case we must add in all the projects in that parent's subtree.
project_ids = set(project_ids)
project_ids_inherited = self.list_project_ids_for_user(
user_id, group_ids, hints or driver_hints.Hints(), inherited=True)
for proj_id in project_ids_inherited:
project_ids.update(
(x['id']
for x in self.resource_api.list_projects_in_subtree(proj_id)))
# Now do the same for any domain inherited roles
domain_ids = self.list_domain_ids_for_user(user_id,
group_ids,
hints
or driver_hints.Hints(),
inherited=True)
project_ids.update(
self.resource_api.list_project_ids_from_domain_ids(domain_ids))
return self.resource_api.list_projects_from_ids(list(project_ids))
def check_user_in_group(self, user_id, group_id):
user_refs = self.list_users_in_group(group_id, driver_hints.Hints())
for x in user_refs:
if x['id'] == user_id:
break
else:
# Try to fetch the user to see if it even exists. This
# will raise a more accurate exception.
self.get_user(user_id)
raise exception.NotFound(
_("User '%(user_id)s' not found in"
" group '%(group_id)s'") % {
'user_id': user_id,
'group_id': group_id
})
def _get_specified_limit_value(self, project_id, resource_name, service_id,
region_id, is_parent=True):
"""Get the specified limit value.
Try to give the resource limit first. If the specified limit is a
parent in a project tree and the resource limit value is None, get the
related registered limit value instead.
"""
hints = driver_hints.Hints()
hints.add_filter('project_id', project_id)
hints.add_filter('service_id', service_id)
hints.add_filter('resource_name', resource_name)
hints.add_filter('region_id', region_id)
limits = PROVIDERS.unified_limit_api.list_limits(hints)
limit_value = limits[0]['resource_limit'] if limits else None
if not limits and is_parent:
hints = driver_hints.Hints()
hints.add_filter('service_id', service_id)
hints.add_filter('resource_name', resource_name)
hints.add_filter('region_id', region_id)
limits = PROVIDERS.unified_limit_api.list_registered_limits(hints)
limit_value = limits[0]['default_limit'] if limits else None
return limit_value
def test_filter_with_both_query_and_hints_set(self):
hints = driver_hints.Hints()
# NOTE: doesn't have to be a real query, we just need to make sure the
# filter string is concatenated correctly
query = uuid.uuid4().hex
username = uuid.uuid4().hex
expected_result = '(&%(query)s(%(user_name_attr)s=%(username)s))' % (
{
'query': query,
'user_name_attr': self.filter_attribute_name,
'username': username
})
hints.add_filter(self.attribute_name, username)
self.assertEqual(expected_result,
self.base_ldap.filter_query(hints=hints, query=query))
def delete_credentials_for_project(self, project_id):
"""Delete all credentials for a project."""
hints = driver_hints.Hints()
hints.add_filter('project_id', project_id)
creds = self.driver.list_credentials(hints)
self.driver.delete_credentials_for_project(project_id)
for cred in creds:
self._get_credential.invalidate(self, cred['id'])
self._list_credentials_for_user.invalidate(self,
cred['user_id'],
cred['type'])
self._list_credentials_for_user.invalidate(self,
cred['user_id'],
None)
def _exercise_project_api(ref_id):
driver = self.resource_api.driver
self.assertRaises(exception.ProjectNotFound,
driver.get_project,
ref_id)
self.assertRaises(exception.ProjectNotFound,
driver.get_project_by_name,
resource.NULL_DOMAIN_ID,
ref_id)
project_ids = [x['id'] for x in
driver.list_projects(driver_hints.Hints())]
self.assertNotIn(ref_id, project_ids)
projects = driver.list_projects_from_ids([ref_id])
self.assertThat(projects, matchers.HasLength(0))
project_ids = [x for x in
driver.list_project_ids_from_domain_ids([ref_id])]
self.assertNotIn(ref_id, project_ids)
self.assertRaises(exception.DomainNotFound,
driver.list_projects_in_domain,
ref_id)
projects = driver.list_projects_in_subtree(ref_id)
self.assertThat(projects, matchers.HasLength(0))
self.assertRaises(exception.ProjectNotFound,
driver.list_project_parents,
ref_id)
# A non-existing project just returns True from the driver
self.assertTrue(driver.is_leaf_project(ref_id))
self.assertRaises(exception.ProjectNotFound,
driver.update_project,
ref_id,
{})
self.assertRaises(exception.ProjectNotFound,
driver.delete_project,
ref_id)
# Deleting list of projects that includes a non-existing project
# should be silent
driver.delete_projects_from_ids([ref_id])
def _delete_child_regions(self, region_id, root_region_id):
"""Delete all child regions.
Recursively delete any region that has the supplied region
as its parent.
"""
children = [
r for r in self.list_regions(driver_hints.Hints())
if r['parent_region_id'] == region_id
]
for child in children:
if child['id'] == root_region_id:
# Hit a circular region hierarchy
return
self._delete_child_regions(child['id'], root_region_id)
self._delete_region(child['id'])
def run(self):
# First off, let's just check we can talk to the domain database
try:
self.resource_manager.list_domains(driver_hints.Hints())
except Exception:
# It is likely that there is some SQL or other backend error
# related to set up
print(_('Unable to access the keystone database, please check it '
'is configured correctly.'))
raise
if not self.valid_options():
return 1
if not self.read_domain_configs_from_files():
return 1
def _ensure_role_exists(self, role_name):
# NOTE(morganfainberg): Do not create the role if it already exists.
try:
role_id = uuid.uuid4().hex
role = {'name': role_name, 'id': role_id}
role = PROVIDERS.role_api.create_role(role_id, role)
LOG.info('Created role %s', role_name)
return role
except exception.Conflict:
LOG.info('Role %s exists, skipping creation.', role_name)
# NOTE(davechen): There is no backend method to get the role
# by name, so build the hints to list the roles and filter by
# name instead.
hints = driver_hints.Hints()
hints.add_filter('name', role_name)
return PROVIDERS.role_api.list_roles(hints)[0]
def _cleanup_identity_provider(self, service, resource_type, operation,
payload):
domain_id = payload['resource_info']
hints = driver_hints.Hints()
hints.add_filter('domain_id', domain_id)
idps = self.driver.list_idps(hints=hints)
for idp in idps:
try:
self.delete_idp(idp['id'])
except exception.IdentityProviderNotFound:
LOG.debug(('Identity Provider %(idpid)s not found when '
'deleting domain contents for %(domainid)s, '
'continuing with cleanup.'), {
'idpid': idp['id'],
'domainid': domain_id
})
def check_default_roles_are_immutable(self):
hints = driver_hints.Hints()
hints.add_filter('domain_id', None) # Only check global roles
roles = PROVIDERS.role_api.list_roles(hints=hints)
default_roles = ('admin', 'member', 'reader',)
failed_roles = []
for role in [r for r in roles if r['name'] in default_roles]:
if not role.get('options', {}).get('immutable'):
failed_roles.append(role['name'])
if any(failed_roles):
return upgradecheck.Result(
upgradecheck.Code.FAILURE,
"Roles are not immutable: %s" % ", ".join(failed_roles)
)
return upgradecheck.Result(
upgradecheck.Code.SUCCESS, "Default roles are immutable.")
def _delete_application_credentials_for_user(self,
user_id,
initiator=None):
"""Delete all application credentials for a user.
:param str user_id: User ID
This is triggered when a user is deleted.
"""
app_creds = self.driver.list_application_credentials_for_user(
user_id, driver_hints.Hints())
self.driver.delete_application_credentials_for_user(user_id)
for app_cred in app_creds:
self.get_application_credential.invalidate(self, app_cred['id'])
notifications.Audit.deleted(self._APP_CRED, app_cred['id'],
initiator)
def test_create_iterate_satisfy(self):
hints = driver_hints.Hints()
hints.add_filter('t1', 'data1')
hints.add_filter('t2', 'data2')
self.assertEqual(2, len(hints.filters))
filter = hints.get_exact_filter_by_name('t1')
self.assertEqual('t1', filter['name'])
self.assertEqual('data1', filter['value'])
self.assertEqual('equals', filter['comparator'])
self.assertFalse(filter['case_sensitive'])
hints.filters.remove(filter)
filter_count = 0
for filter in hints.filters:
filter_count += 1
self.assertEqual('t2', filter['name'])
self.assertEqual(1, filter_count)
def _lookup_and_create_role(self, request, role_name, domain_id):
hints = driver_hints.Hints()
hints.add_filter("name", role_name, case_sensitive=True)
found_roles = self.role_api.list_roles(hints)
LOG.info("Found roles:", found_roles)
if not found_roles:
# Create the role
role_id = self.generate_consistent_id(role_name)
role_ref = {'id': role_id, 'name': role_name}
return self.role_api.create_role(role_ref['id'],
role_ref,
initiator=request.audit_initiator)
elif len(found_roles) == 1:
return self.role_api.get_role(found_roles[0]['id'])
else:
raise exception.AmbiguityError(resource='role', name=role_name)
def test_list_limit_by_multi_filter_with_project_id(self):
limit_1 = unit.new_limit_ref(
project_id=self.tenant_bar['id'],
service_id=self.service_one['id'],
region_id=self.region_one['id'],
resource_name='volume', resource_limit=10, id=uuid.uuid4().hex)
limit_2 = unit.new_limit_ref(
project_id=self.tenant_baz['id'],
service_id=self.service_one['id'],
region_id=self.region_two['id'],
resource_name='snapshot', resource_limit=10, id=uuid.uuid4().hex)
PROVIDERS.unified_limit_api.create_limits([limit_1, limit_2])
hints = driver_hints.Hints()
hints.add_filter('service_id', self.service_one['id'])
hints.add_filter('project_id', self.tenant_bar['id'])
res = PROVIDERS.unified_limit_api.list_limits(hints)
self.assertEqual(1, len(res))
def list_projects_for_groups(self, group_ids):
project_ids = (self.driver.list_project_ids_for_groups(
group_ids, driver_hints.Hints()))
if not CONF.os_inherit.enabled:
return self.resource_api.list_projects_from_ids(project_ids)
# Inherited roles are enabled, so check to see if these groups have any
# roles on any domain, in which case we must add in all the projects
# in that domain.
domain_ids = self.driver.list_domain_ids_for_groups(group_ids,
inherited=True)
project_ids_from_domains = (
self.resource_api.list_project_ids_from_domain_ids(domain_ids))
return self.resource_api.list_projects_from_ids(
list(set(project_ids + project_ids_from_domains)))
def _normalize_role_list(self, trust_roles):
roles = []
for role in trust_roles:
if role.get('id'):
roles.append({'id': role['id']})
else:
hints = driver_hints.Hints()
hints.add_filter("name", role['name'], case_sensitive=True)
found_roles = self.role_api.list_roles(hints)
if not found_roles:
raise exception.RoleNotFound(
_("Role %s is not defined") % role['name'])
elif len(found_roles) == 1:
roles.append({'id': found_roles[0]['id']})
else:
raise exception.AmbiguityError(resource='role',
name=role['name'])
return roles
def _delete_application_credentials_for_user_on_project(
self, user_id, project_id):
"""Delete all application credentials for a user on a given project.
:param str user_id: User ID
:param str project_id: Project ID
This is triggered when a user loses a role assignment on a project.
"""
hints = driver_hints.Hints()
hints.add_filter('project_id', project_id)
app_creds = self.driver.list_application_credentials_for_user(
user_id, hints)
self.driver.delete_application_credentials_for_user_on_project(
user_id, project_id)
for app_cred in app_creds:
self.get_application_credential.invalidate(self, app_cred['id'])
def run(self):
# First off, let's just check we can talk to the domain database
try:
self.resource_manager.list_domains(driver_hints.Hints())
except Exception:
# It is likely that there is some SQL or other backend error
# related to set up
print(_('Unable to access the keystone database, please check it '
'is configured correctly.'))
raise
try:
self.valid_options()
self.read_domain_configs_from_files()
except ValueError:
# We will already have printed out a nice message, so indicate
# to caller the non-success error code to be used.
return 1
def test_deleting_a_user_deletes_application_credentials(self):
app_cred_1 = self._new_app_cred_data(self.user_foo['id'],
project_id=self.tenant_bar['id'],
name='app1')
app_cred_2 = self._new_app_cred_data(self.user_foo['id'],
project_id=self.tenant_bar['id'],
name='app2')
self.app_cred_api.create_application_credential(app_cred_1)
self.app_cred_api.create_application_credential(app_cred_2)
self.assertIn(app_cred_1['id'], self._list_ids(self.user_foo))
self.assertIn(app_cred_2['id'], self._list_ids(self.user_foo))
# This should trigger a notification which should invoke a callback in
# the application credential Manager to cleanup user_foo's application
# credentials.
PROVIDERS.identity_api.delete_user(self.user_foo['id'])
hints = driver_hints.Hints()
self.assertListEqual(
[], self.app_cred_api.list_application_credentials(
self.user_foo['id'], hints))
def _check_and_fill_registered_limit_id(self, limit):
# Make sure there is a referenced registered limit first. Then add
# the registered limit id to the new created limit.
hints = driver_hints.Hints()
limit_copy = copy.deepcopy(limit)
hints.add_filter('service_id', limit_copy.pop('service_id'))
hints.add_filter('resource_name', limit_copy.pop('resource_name'))
hints.add_filter('region_id', limit_copy.pop('region_id', None))
with sql.session_for_read() as session:
registered_limits = session.query(RegisteredLimitModel)
registered_limits = sql.filter_limit_query(
RegisteredLimitModel, registered_limits, hints)
reg_limits = registered_limits.all()
if not reg_limits:
raise exception.NoLimitReference
limit_copy['registered_limit_id'] = reg_limits[0]['id']
return limit_copy
def migrate_credentials(self):
crypto, keys = credential_fernet.get_multi_fernet_keys()
primary_key_hash = credential_fernet.primary_key_hash(keys)
# FIXME(lbragstad): We *should* be able to use Hints() to ask only for
# credentials that have a key_hash equal to a secondary key hash or
# None, but Hints() doesn't seem to honor None values. See
# https://bugs.launchpad.net/keystone/+bug/1614154. As a workaround -
# we have to ask for *all* credentials and filter them ourselves.
credentials = self.credential_api.driver.list_credentials(
driver_hints.Hints())
for credential in credentials:
if credential['key_hash'] != primary_key_hash:
# If the key_hash isn't None but doesn't match the
# primary_key_hash, then we know the credential was encrypted
# with a secondary key. Let's decrypt it, and send it through
# the update path to re-encrypt it with the new primary key.
decrypted_blob = self.credential_provider_api.decrypt(
credential['encrypted_blob'])
cred = {'blob': decrypted_blob}
self.credential_api.update_credential(credential['id'], cred)
def test_list_application_credentials(self):
app_cred_1 = self._new_app_cred_data(self.user_foo['id'],
project_id=self.tenant_bar['id'],
name='app1')
app_cred_2 = self._new_app_cred_data(self.user_foo['id'],
project_id=self.tenant_bar['id'],
name='app2')
app_cred_3 = self._new_app_cred_data(self.user_two['id'],
project_id=self.tenant_baz['id'],
name='app3')
resp1 = self.app_cred_api.create_application_credential(app_cred_1)
resp2 = self.app_cred_api.create_application_credential(app_cred_2)
resp3 = self.app_cred_api.create_application_credential(app_cred_3)
hints = driver_hints.Hints()
resp = self.app_cred_api.list_application_credentials(
self.user_foo['id'], hints)
resp_ids = [ac['id'] for ac in resp]
self.assertIn(resp1['id'], resp_ids)
self.assertIn(resp2['id'], resp_ids)
self.assertNotIn(resp3['id'], resp_ids)
for ac in resp:
self.assertNotIn('secret_hash', ac)
def _check_unified_limit_unique(self, unified_limit,
is_registered_limit=True):
# Ensure the new created or updated unified limit won't break the
# current reference between registered limit and limit. i.e. We should
# ensure that there is no duplicate entry.
hints = driver_hints.Hints()
hints.add_filter('service_id', unified_limit['service_id'])
hints.add_filter('resource_name', unified_limit['resource_name'])
hints.add_filter('region_id', unified_limit.get('region_id'))
if is_registered_limit:
with sql.session_for_read() as session:
query = session.query(RegisteredLimitModel)
unified_limits = sql.filter_limit_query(RegisteredLimitModel,
query,
hints).all()
else:
hints.add_filter('project_id', unified_limit['project_id'])
with sql.session_for_read() as session:
query = session.query(LimitModel)
old_unified_limits = sql.filter_limit_query(LimitModel,
query,
hints).all()
query = session.query(
LimitModel).outerjoin(RegisteredLimitModel)
new_unified_limits = query.filter(
LimitModel.project_id ==
unified_limit['project_id'],
RegisteredLimitModel.service_id ==
unified_limit['service_id'],
RegisteredLimitModel.region_id ==
unified_limit.get('region_id'),
RegisteredLimitModel.resource_name ==
unified_limit['resource_name']).all()
unified_limits = old_unified_limits + new_unified_limits
if unified_limits:
msg = _('Duplicate entry')
limit_type = 'registered_limit' if is_registered_limit else 'limit'
raise exception.Conflict(type=limit_type, details=msg)