def _validate_token_data(openrc):
keystone_session = openstack_utils.get_keystone_session(openrc)
keystone_client = openstack_utils.get_keystone_session_client(
keystone_session)
token = keystone_session.get_token()
if (openstack_utils.get_os_release() <
openstack_utils.get_os_release('xenial_ocata')):
if len(token) != 32:
raise zaza_exceptions.KeystoneWrongTokenProvider(
'We expected a UUID token and got this: "{}"'.format(
token))
else:
if len(token) < 180:
raise zaza_exceptions.KeystoneWrongTokenProvider(
'We expected a Fernet token and got this: "{}"'.format(
token))
logging.info('token: "{}"'.format(pprint.pformat(token)))
if (openstack_utils.get_os_release() <
openstack_utils.get_os_release('trusty_mitaka')):
logging.info('skip: tokens.get_token_data() not allowed prior '
'to trusty_mitaka')
return
# get_token_data call also gets the service catalog
token_data = keystone_client.tokens.get_token_data(token)
if token_data.get('token', {}).get('catalog', None) is None:
raise zaza_exceptions.KeystoneAuthorizationStrict(
# NOTE(fnordahl) the above call will probably throw a
# http.Forbidden exception, but just in case
'Regular end user not allowed to retrieve the service '
'catalog. ("{}")'.format(pprint.pformat(token_data)))
logging.info('token_data: "{}"'.format(pprint.pformat(token_data)))
def get_expected_pools(radosgw=False):
"""Get expected ceph pools.
Return a list of expected ceph pools in a ceph + cinder + glance
test scenario, based on OpenStack release and whether ceph radosgw
is flagged as present or not.
:param radosgw: If radosgw is used or not
:type radosgw: boolean
:returns: List of pools that are expected
:rtype: list
"""
current_release = openstack_utils.get_os_release()
trusty_icehouse = openstack_utils.get_os_release('trusty_icehouse')
trusty_kilo = openstack_utils.get_os_release('trusty_kilo')
zesty_ocata = openstack_utils.get_os_release('zesty_ocata')
if current_release == trusty_icehouse:
# Icehouse
pools = ['data', 'metadata', 'rbd', 'cinder-ceph', 'glance']
elif (trusty_kilo <= current_release <= zesty_ocata):
# Kilo through Ocata
pools = ['rbd', 'cinder-ceph', 'glance']
else:
# Pike and later
pools = ['cinder-ceph', 'glance']
if radosgw:
pools.extend(
['.rgw.root', '.rgw.control', '.rgw', '.rgw.gc', '.users.uid'])
return pools
def setUpClass(cls):
"""Run class setup for running Keystone charm operation tests."""
super(BaseKeystoneTest, cls).setUpClass()
# Check if we are related to Vault TLS certificates
cls.tls_rid = zaza.model.get_relation_id(
'keystone', 'vault', remote_interface_name='certificates')
# Check for VIP
cls.vip = (zaza.model.get_application_config('keystone')
.get('vip').get('value'))
cls.keystone_ips = zaza.model.get_app_ips('keystone')
# If we have a VIP set and we are using TLS only check the VIP
# If you check the individual IP haproxy may send to a different
# back end which leads to mismatched certificates.
if cls.vip:
if cls.tls_rid:
cls.keystone_ips = [cls.vip]
else:
cls.keystone_ips.append(cls.vip)
if (openstack_utils.get_os_release() <
openstack_utils.get_os_release('xenial_queens')):
cls.default_api_version = '2'
else:
cls.default_api_version = '3'
cls.admin_keystone_session = (
openstack_utils.get_overcloud_keystone_session())
cls.admin_keystone_client = (
openstack_utils.get_keystone_session_client(
cls.admin_keystone_session,
client_api_version=cls.default_api_version))
def setUpClass(cls):
"""Run class setup for running Keystone charm operation tests."""
super(BaseKeystoneTest, cls).setUpClass()
cls.keystone_ips = zaza.model.get_app_ips('keystone')
if (openstack_utils.get_os_release() <
openstack_utils.get_os_release('xenial_queens')):
cls.default_api_version = '2'
else:
cls.default_api_version = '3'
def test_admin_project_scoped_access(self):
"""Verify cloud admin access using project scoped token.
`admin` user in `admin_domain` should be able to access API methods
guarded by `rule:cloud_admin` policy using a token scoped to `admin`
project in `admin_domain`.
We implement a policy that enables domain segregation and
administration delegation [0]. It is important to understand that this
differs from the default policy.
In the initial implementation it was necessary to switch between using
a `domain` scoped and `project` scoped token to successfully manage a
cloud, but since the introduction of `is_admin` functionality in
Keystone [1][2][3] and our subsequent adoption of it in Keystone charm
[4], this is no longer necessary.
This test here to validate this behaviour.
0: https://github.com/openstack/keystone/commit/c7a5c6c
1: https://github.com/openstack/keystone/commit/e702369
2: https://github.com/openstack/keystone/commit/e923a14
3: https://github.com/openstack/keystone/commit/9804081
4: https://github.com/openstack/charm-keystone/commit/10e3d84
"""
if (openstack_utils.get_os_release() <
openstack_utils.get_os_release('trusty_mitaka')):
logging.info('skipping test < trusty_mitaka')
return
with self.config_change(
{'preferred-api-version': self.default_api_version},
{'preferred-api-version': '3'},
application_name="keystone"):
for ip in self.keystone_ips:
try:
logging.info('keystone IP {}'.format(ip))
ks_session = openstack_utils.get_keystone_session(
openstack_utils.get_overcloud_auth(address=ip))
ks_client = openstack_utils.get_keystone_session_client(
ks_session)
result = ks_client.domains.list()
logging.info('.domains.list: "{}"'.format(
pprint.pformat(result)))
except keystoneauth1.exceptions.http.Forbidden as e:
raise zaza_exceptions.KeystoneAuthorizationStrict(
'Retrieve domain list as admin with project scoped '
'token FAILED. ({})'.format(e))
logging.info('OK')
def test_end_user_domain_admin_access(self):
"""Verify that end-user domain admin does not have elevated privileges.
In additon to validating that the `policy.json` is written and the
service is restarted on config-changed, the test validates that our
`policy.json` is correct.
Catch regressions like LP: #1651989
"""
if (openstack_utils.get_os_release() <
openstack_utils.get_os_release('xenial_ocata')):
logging.info('skipping test < xenial_ocata')
return
with self.config_change(
{'preferred-api-version': self.default_api_version},
{'preferred-api-version': '3'},
application_name="keystone"):
for ip in self.keystone_ips:
openrc = {
'API_VERSION': 3,
'OS_USERNAME': DEMO_ADMIN_USER,
'OS_PASSWORD': DEMO_ADMIN_USER_PASSWORD,
'OS_AUTH_URL': 'http://{}:5000/v3'.format(ip),
'OS_USER_DOMAIN_NAME': DEMO_DOMAIN,
'OS_DOMAIN_NAME': DEMO_DOMAIN,
}
if self.tls_rid:
openrc['OS_CACERT'] = openstack_utils.KEYSTONE_LOCAL_CACERT
openrc['OS_AUTH_URL'] = (openrc['OS_AUTH_URL'].replace(
'http', 'https'))
logging.info('keystone IP {}'.format(ip))
keystone_session = openstack_utils.get_keystone_session(
openrc, scope='DOMAIN')
keystone_client = openstack_utils.get_keystone_session_client(
keystone_session)
try:
# expect failure
keystone_client.domains.list()
except keystoneauth1.exceptions.http.Forbidden as e:
logging.debug('Retrieve domain list as end-user domain '
'admin NOT allowed...OK ({})'.format(e))
pass
else:
raise zaza_exceptions.KeystoneAuthorizationPermissive(
'Retrieve domain list as end-user domain admin '
'allowed when it should not be.')
logging.info('OK')
def test_get_openstack_release(self):
self.patch('zaza.utilities.openstack.get_current_os_release_pair',
new_callable=mock.MagicMock(),
name='_get_os_rel_pair')
# Bad release pair
release_pair = 'bad'
with self.assertRaises(exceptions.ReleasePairNotFound):
openstack_utils.get_os_release(release_pair)
# Normal scenario
expected = 4
result = openstack_utils.get_os_release('xenial_mitaka')
self.assertEqual(expected, result)
# Normal scenario with current release pair
self._get_os_rel_pair.return_value = 'xenial_mitaka'
expected = 4
result = openstack_utils.get_os_release()
self.assertEqual(expected, result)
# We can compare releases xenial_queens > xenial_mitaka
xenial_queens = openstack_utils.get_os_release('xenial_queens')
xenial_mitaka = openstack_utils.get_os_release('xenial_mitaka')
release_comp = xenial_queens > xenial_mitaka
self.assertTrue(release_comp)
def add_demo_user():
"""Add a demo user to the current deployment."""
def _v2():
keystone_session = openstack_utils.get_overcloud_keystone_session()
keystone_client = openstack_utils.get_keystone_session_client(
keystone_session, client_api_version=2)
tenant = keystone_client.tenants.create(tenant_name=DEMO_TENANT,
description='Demo Tenant',
enabled=True)
keystone_client.users.create(name=DEMO_USER,
password=DEMO_PASSWORD,
tenant_id=tenant.id)
def _v3():
keystone_session = openstack_utils.get_overcloud_keystone_session()
keystone_client = openstack_utils.get_keystone_session_client(
keystone_session)
domain = keystone_client.domains.create(
DEMO_DOMAIN,
description='Demo Domain',
enabled=True)
project = keystone_client.projects.create(
DEMO_PROJECT,
domain,
description='Demo Project',
enabled=True)
demo_user = keystone_client.users.create(
DEMO_USER,
domain=domain,
project=project,
password=DEMO_PASSWORD,
email='*****@*****.**',
description='Demo User',
enabled=True)
member_role = keystone_client.roles.find(name='Member')
keystone_client.roles.grant(
member_role,
user=demo_user,
project_domain=domain,
project=project)
demo_admin_user = keystone_client.users.create(
DEMO_ADMIN_USER,
domain=domain,
project=project,
password=DEMO_ADMIN_USER_PASSWORD,
email='*****@*****.**',
description='Demo Admin User',
enabled=True)
admin_role = keystone_client.roles.find(name='Admin')
keystone_client.roles.grant(
admin_role,
user=demo_admin_user,
domain=domain)
keystone_client.roles.grant(
member_role,
user=demo_admin_user,
project_domain=domain,
project=project)
keystone_client.roles.grant(
admin_role,
user=demo_admin_user,
project_domain=domain,
project=project)
if (openstack_utils.get_os_release() <
openstack_utils.get_os_release('trusty_mitaka')):
# create only V2 user
_v2()
return
if (openstack_utils.get_os_release() >=
openstack_utils.get_os_release('trusty_mitaka') and
openstack_utils.get_os_release() <
openstack_utils.get_os_release('xenial_queens')):
# create V2 and V3 user
_v2()
_singleton = BaseKeystoneTest()
_singleton.setUpClass()
# Explicitly set application name in case setup is called by a charm
# under test other than keystone.
with _singleton.config_change(
{'preferred-api-version': _singleton.default_api_version},
{'preferred-api-version': '3'}, application_name="keystone"):
_v3()
else:
# create only V3 user
_v3()
def test_key_distribution_and_rotation(self):
"""Verify key rotation.
Note that we make the assumption that test bundle configure
`token-expiration` to 60 and that it takes > 60s from deployment
completes until we get to this test.
"""
if (openstack_utils.get_os_release() <
openstack_utils.get_os_release('xenial_ocata')):
logging.info('skipping test < xenial_ocata')
return
with self.pause_resume(['apache2']):
KEY_KEY_REPOSITORY = 'key_repository'
CREDENTIAL_KEY_REPOSITORY = '/etc/keystone/credential-keys/'
FERNET_KEY_REPOSITORY = '/etc/keystone/fernet-keys/'
# get key repostiroy from leader storage
key_repository = json.loads(
juju_utils.leader_get(self.application_name,
KEY_KEY_REPOSITORY))
# sort keys so we can compare it to on-disk repositories
key_repository = json.loads(
json.dumps(key_repository, sort_keys=True),
object_pairs_hook=collections.OrderedDict)
logging.info('key_repository: "{}"'.format(
pprint.pformat(key_repository)))
for repo in [CREDENTIAL_KEY_REPOSITORY, FERNET_KEY_REPOSITORY]:
try:
for key_name, key in key_repository[repo].items():
if int(key_name) > 1:
# after initialization the repository contains the
# staging key (0) and the primary key (1). After
# rotation the repository contains at least one key
# with higher index.
break
else:
# NOTE the charm should only rotate the fernet key
# repostiory and not rotate the credential key
# repository.
if repo == FERNET_KEY_REPOSITORY:
raise zaza_exceptions.KeystoneKeyRepositoryError(
'Keys in Fernet key repository has not been '
'rotated.')
except KeyError:
raise zaza_exceptions.KeystoneKeyRepositoryError(
'Dict in leader setting "{}" does not contain key '
'repository "{}"'.format(KEY_KEY_REPOSITORY, repo))
# get on-disk key repository from all units
on_disk = {}
units = zaza.model.get_units(self.application_name)
for unit in units:
on_disk[unit.entity_id] = {}
for repo in [CREDENTIAL_KEY_REPOSITORY, FERNET_KEY_REPOSITORY]:
on_disk[unit.entity_id][repo] = {}
result = zaza.model.run_on_unit(
unit.entity_id, 'sudo ls -1 {}'.format(repo))
for key_name in result.get('Stdout').split():
result = zaza.model.run_on_unit(
unit.entity_id,
'sudo cat {}/{}'.format(repo, key_name))
on_disk[unit.entity_id][repo][key_name] = result.get(
'Stdout')
# sort keys so we can compare it to leader storage repositories
on_disk = json.loads(json.dumps(on_disk, sort_keys=True),
object_pairs_hook=collections.OrderedDict)
logging.info('on_disk: "{}"'.format(pprint.pformat(on_disk)))
for unit in units:
unit_repo = on_disk[unit.entity_id]
lead_repo = key_repository
if unit_repo != lead_repo:
raise zaza_exceptions.KeystoneKeyRepositoryError(
'expect: "{}" actual({}): "{}"'.format(
pprint.pformat(lead_repo), unit.entity_id,
pprint.pformat(unit_repo)))
logging.info('"{}" == "{}"'.format(pprint.pformat(unit_repo),
pprint.pformat(lead_repo)))
def test_end_user_acccess_and_token(self):
"""Verify regular end-user access resources and validate token data.
In effect this also validates user creation, presence of standard
roles (`_member_`, `Member`), effect of policy and configuration
of `token-provider`.
"""
def _validate_token_data(openrc):
keystone_session = openstack_utils.get_keystone_session(openrc)
keystone_client = openstack_utils.get_keystone_session_client(
keystone_session)
token = keystone_session.get_token()
if (openstack_utils.get_os_release() <
openstack_utils.get_os_release('xenial_ocata')):
if len(token) != 32:
raise zaza_exceptions.KeystoneWrongTokenProvider(
'We expected a UUID token and got this: "{}"'.format(
token))
else:
if len(token) < 180:
raise zaza_exceptions.KeystoneWrongTokenProvider(
'We expected a Fernet token and got this: "{}"'.format(
token))
logging.info('token: "{}"'.format(pprint.pformat(token)))
if (openstack_utils.get_os_release() <
openstack_utils.get_os_release('trusty_mitaka')):
logging.info('skip: tokens.get_token_data() not allowed prior '
'to trusty_mitaka')
return
# get_token_data call also gets the service catalog
token_data = keystone_client.tokens.get_token_data(token)
if token_data.get('token', {}).get('catalog', None) is None:
raise zaza_exceptions.KeystoneAuthorizationStrict(
# NOTE(fnordahl) the above call will probably throw a
# http.Forbidden exception, but just in case
'Regular end user not allowed to retrieve the service '
'catalog. ("{}")'.format(pprint.pformat(token_data)))
logging.info('token_data: "{}"'.format(pprint.pformat(token_data)))
if (openstack_utils.get_os_release() <
openstack_utils.get_os_release('xenial_queens')):
openrc = {
'API_VERSION': 2,
'OS_USERNAME': DEMO_USER,
'OS_PASSWORD': DEMO_PASSWORD,
'OS_TENANT_NAME': DEMO_TENANT,
}
for ip in self.keystone_ips:
openrc.update(
{'OS_AUTH_URL': 'http://{}:5000/v2.0'.format(ip)})
_validate_token_data(openrc)
if (openstack_utils.get_os_release() >=
openstack_utils.get_os_release('trusty_mitaka')):
openrc = {
'API_VERSION': 3,
'OS_REGION_NAME': 'RegionOne',
'OS_USER_DOMAIN_NAME': DEMO_DOMAIN,
'OS_USERNAME': DEMO_USER,
'OS_PASSWORD': DEMO_PASSWORD,
'OS_PROJECT_DOMAIN_NAME': DEMO_DOMAIN,
'OS_PROJECT_NAME': DEMO_PROJECT,
}
with self.config_change(
{'preferred-api-version': self.default_api_version},
{'preferred-api-version': '3'}):
for ip in self.keystone_ips:
openrc.update(
{'OS_AUTH_URL': 'http://{}:5000/v3'.format(ip)})
_validate_token_data(openrc)
def test_ceph_encryption(self):
"""Test Ceph encryption.
Verify that the new disk is added with encryption by checking for
Ceph's encryption keys directory.
"""
current_release = zaza_openstack.get_os_release()
trusty_mitaka = zaza_openstack.get_os_release('trusty_mitaka')
if current_release >= trusty_mitaka:
logging.warn("Skipping encryption test for Mitaka and higher")
return
unit_name = 'ceph-osd/0'
set_default = {
'osd-encrypt': 'False',
'osd-devices': '/dev/vdb /srv/ceph',
}
set_alternate = {
'osd-encrypt': 'True',
'osd-devices': '/dev/vdb /srv/ceph /srv/ceph_encrypted',
}
juju_service = 'ceph-osd'
logging.info('Making config change on {}...'.format(juju_service))
mtime = zaza_model.get_unit_time(unit_name)
file_mtime = None
folder_name = '/etc/ceph/dmcrypt-keys/'
with self.config_change(set_default, set_alternate):
with tempfile.TemporaryDirectory() as tempdir:
# Creating a temp dir to copy keys
temp_folder = '/tmp/dmcrypt-keys'
cmd = 'mkdir {}'.format(temp_folder)
ret = zaza_model.run_on_unit(unit_name, cmd)
logging.debug('Ret for cmd {} is {}'.format(cmd, ret))
# Copy keys from /etc to /tmp
cmd = 'sudo cp {}* {}'.format(folder_name, temp_folder)
ret = zaza_model.run_on_unit(unit_name, cmd)
logging.debug('Ret for cmd {} is {}'.format(cmd, ret))
# Changing permissions to be able to SCP the files
cmd = 'sudo chown -R ubuntu:ubuntu {}'.format(temp_folder)
ret = zaza_model.run_on_unit(unit_name, cmd)
logging.debug('Ret for cmd {} is {}'.format(cmd, ret))
# SCP to retrieve all files in folder
# -p: preserve timestamps
source = '/tmp/dmcrypt-keys/*'
zaza_model.scp_from_unit(unit_name=unit_name,
source=source,
destination=tempdir,
scp_opts='-p')
for elt in listdir(tempdir):
file_path = '/'.join([tempdir, elt])
if path.isfile(file_path):
file_mtime = path.getmtime(file_path)
if file_mtime:
break
if not file_mtime:
logging.warn('Could not determine mtime, assuming '
'folder does not exist')
raise FileNotFoundError('folder does not exist')
if file_mtime >= mtime:
logging.info('Folder mtime is newer than provided mtime '
'(%s >= %s) on %s (OK)' %
(file_mtime, mtime, unit_name))
else:
logging.warn('Folder mtime is older than provided mtime'
'(%s < on %s) on %s' % (file_mtime, mtime, unit_name))
raise Exception('Folder mtime is older than provided mtime')