def test_200_forced_cinder_failover(self):
"""Validate Ceph failover via Cinder when the primary site is down.
This test only makes sense if Cinder RBD mirroring mode is 'image'.
It will return early, if this is not the case.
This assumes that the primary site is already killed.
"""
cinder_rbd_mirroring_mode = get_cinder_rbd_mirroring_mode(
self.cinder_ceph_app_name)
if cinder_rbd_mirroring_mode != 'image':
logging.warning(
"Skipping 'test_200_cinder_failover_without_primary_site' "
"since Cinder RBD mirroring mode is {}.".format(
cinder_rbd_mirroring_mode))
return
# Make sure that the Cinder Ceph backend workaround is applied.
self.apply_cinder_ceph_workaround()
session = openstack.get_overcloud_keystone_session()
cinder = openstack.get_cinder_session_client(session, version=3)
openstack.failover_cinder_volume_host(
cinder=cinder,
backend_name=self.cinder_ceph_app_name,
target_backend_id='ceph',
target_status='disabled',
target_replication_status='failed-over')
# Check that the Cinder volumes are still available after forced
# failover.
for volume in cinder.volumes.list():
self.assertEqual(volume.status, 'available')
def setUpClass(cls):
"""Run class setup for running tests."""
super(CinderBackendTest, cls).setUpClass()
cls.keystone_session = openstack_utils.get_overcloud_keystone_session()
cls.model_name = zaza.model.get_juju_model()
cls.cinder_client = openstack_utils.get_cinder_session_client(
cls.keystone_session)
def setup_test_cinder_volume(self):
"""Set up the test Cinder volume into the Ceph RBD mirror environment.
If the volume already exists, then it's returned.
Also, if the Cinder RBD mirroring mode is 'image', the volume will
use an explicit volume type with the appropriate replication flags.
Otherwise, it is just a simple Cinder volume using the default backend.
:returns: Cinder volume
:rtype: :class:`Volume`.
"""
session = openstack.get_overcloud_keystone_session()
cinder = openstack.get_cinder_session_client(session, version=3)
try:
return cinder.volumes.find(name=self.test_cinder_volume_name)
except cinder_exceptions.NotFound:
logging.info("Test Cinder volume doesn't exist. Creating it")
glance = openstack.get_glance_session_client(session)
image = get_glance_image(glance)
kwargs = {
'cinder': cinder,
'name': self.test_cinder_volume_name,
'image_id': image.id,
}
if get_cinder_rbd_mirroring_mode(self.cinder_ceph_app_name) == 'image':
volume_type = setup_cinder_repl_volume_type(
cinder, backend_name=self.cinder_ceph_app_name)
kwargs['type_id'] = volume_type.id
return create_cinder_volume(**kwargs)
def setUpClass(cls):
"""Run class setup for running tests."""
super(CinderTests, cls).setUpClass()
cls.cinder_client = openstack_utils.get_cinder_session_client(
cls.keystone_session)
cls.nova_client = openstack_utils.get_nova_session_client(
cls.keystone_session)
def setUpClass(cls):
"""Run class setup for running glance tests with cinder backend."""
super(GlanceCinderBackendTest, cls).setUpClass()
cls.glance_client = openstack_utils.get_glance_session_client(
cls.keystone_session)
cls.cinder_client = openstack_utils.get_cinder_session_client(
cls.keystone_session)
def setUpClass(cls):
"""Run class setup for running tests."""
super(CinderLVMTest, cls).setUpClass(application_name='cinder-lvm')
cls.model_name = zaza.model.get_juju_model()
cls.cinder_client = openstack_utils.get_cinder_session_client(
cls.keystone_session)
cls.block_device = openstack_utils.get_application_config_option(
'cinder-lvm', 'block-device', model_name=cls.model_name)
def setUpClass(cls):
"""Run class setup for running tests."""
super().setUpClass(application_name=cls.application_name)
cls.cinder_client = openstack_utils.get_cinder_session_client(
cls.keystone_session)
cls.nova_client = openstack_utils.get_nova_session_client(
cls.keystone_session)
cls.keystone_client = openstack_utils.get_keystone_session_client(
cls.keystone_session)
def setUpClass(cls):
"""Run class setup for running tests."""
super(CinderTests, cls).setUpClass(application_name='cinder')
cls.application_name = 'cinder'
cls.lead_unit = zaza.model.get_lead_unit_name(
"cinder", model_name=cls.model_name)
cls.cinder_client = openstack_utils.get_cinder_session_client(
cls.keystone_session)
cls.nova_client = openstack_utils.get_nova_session_client(
cls.keystone_session)
def get_client_and_attempt_operation(self, ip):
"""Attempt to list the images as a policyd override.
This operation should pass normally for the demo_user, and fail when
the rule has been overriden (see the `rule` class variable.
:param ip: the IP address to get the session against.
:type ip: str
:raises: PolicydOperationFailedException if operation fails.
"""
cinder_client = openstack_utils.get_cinder_session_client(
self.get_keystone_session_admin_user(ip))
try:
cinder_client.volumes.list()
except cinderclient.exceptions.Forbidden:
raise PolicydOperationFailedException()
def test_100_cinder_failover(self):
"""Validate controlled failover via the Cinder API.
This test only makes sense if Cinder RBD mirroring mode is 'image'.
It will return early, if this is not the case.
"""
cinder_rbd_mirroring_mode = get_cinder_rbd_mirroring_mode(
self.cinder_ceph_app_name)
if cinder_rbd_mirroring_mode != 'image':
logging.warning(
"Skipping 'test_100_cinder_failover' since Cinder RBD "
"mirroring mode is {}.".format(cinder_rbd_mirroring_mode))
return
session = openstack.get_overcloud_keystone_session()
cinder = openstack.get_cinder_session_client(session, version=3)
# Check if the Cinder volume host is available with replication
# enabled.
host = 'cinder@{}'.format(self.cinder_ceph_app_name)
svc = cinder.services.list(host=host, binary='cinder-volume')[0]
self.assertEqual(svc.replication_status, 'enabled')
self.assertEqual(svc.status, 'enabled')
# Setup the test Cinder volume
volume = self.setup_test_cinder_volume()
# Check if the volume is properly mirrored
self.wait_for_mirror_state('up+replaying',
check_entries_behind_master=True,
application_name=self.application_name +
self.site_b_app_suffix,
model_name=self.site_b_model,
pools=[self.cinder_ceph_app_name])
# Execute the Cinder volume failover
openstack.failover_cinder_volume_host(
cinder=cinder,
backend_name=self.cinder_ceph_app_name,
target_backend_id='ceph',
target_status='disabled',
target_replication_status='failed-over')
# Check if the test volume is still available after failover
self.assertEqual(cinder.volumes.get(volume.id).status, 'available')
def test_101_cinder_failback(self):
"""Validate controlled failback via the Cinder API.
This test only makes sense if Cinder RBD mirroring mode is 'image'.
It will return early, if this is not the case.
The test needs to be executed when the Cinder volume host is already
failed-over with the test volume on it.
"""
cinder_rbd_mirroring_mode = get_cinder_rbd_mirroring_mode(
self.cinder_ceph_app_name)
if cinder_rbd_mirroring_mode != 'image':
logging.warning(
"Skipping 'test_101_cinder_failback' since Cinder RBD "
"mirroring mode is {}.".format(cinder_rbd_mirroring_mode))
return
session = openstack.get_overcloud_keystone_session()
cinder = openstack.get_cinder_session_client(session, version=3)
# Check if the Cinder volume host is already failed-over
host = 'cinder@{}'.format(self.cinder_ceph_app_name)
svc = cinder.services.list(host=host, binary='cinder-volume')[0]
self.assertEqual(svc.replication_status, 'failed-over')
self.assertEqual(svc.status, 'disabled')
# Check if the test Cinder volume is already present. The method
# 'cinder.volumes.find' raises 404 if the volume is not found.
volume = cinder.volumes.find(name=self.test_cinder_volume_name)
# Execute the Cinder volume failback
openstack.failover_cinder_volume_host(
cinder=cinder,
backend_name=self.cinder_ceph_app_name,
target_backend_id='default',
target_status='enabled',
target_replication_status='enabled')
# Check if the test volume is still available after failback
self.assertEqual(cinder.volumes.get(volume.id).status, 'available')
def test_cinder_volume_mirrored(self):
"""Validate that a volume created through Cinder is mirrored.
For RBD Mirroring to work clients must enable the correct set of
features when creating images.
The RBD image feature settings are announced by the ``ceph-mon`` charm
over the client relation when it has units related on its
``rbd-mirror`` endpoint.
By creating a volume through cinder on site A, checking for presence on
site B and subsequently comparing the contents we get a full end to end
test.
"""
session = openstack.get_overcloud_keystone_session()
glance = openstack.get_glance_session_client(session)
cinder = openstack.get_cinder_session_client(session)
image = next(glance.images.list(name=LTS_IMAGE_NAME))
# NOTE(fnordahl): for some reason create volume from image often fails
# when run just after deployment is finished. We should figure out
# why, resolve the underlying issue and then remove this.
#
# We do not use tenacity here as it will interfere with tenacity used
# in ``resource_reaches_status``
def create_volume_from_image(cinder, image, retry=20):
if retry < 1:
return
volume = cinder.volumes.create(8, name='zaza', imageRef=image.id)
try:
# Note(coreycb): stop_after_attempt is increased because using
# juju storage for ceph-osd backed by cinder on undercloud
# takes longer than the prior method of directory-backed OSD
# devices.
openstack.resource_reaches_status(cinder.volumes,
volume.id,
msg='volume',
stop_after_attempt=20)
return volume
except AssertionError:
logging.info('retrying')
volume.delete()
return create_volume_from_image(cinder, image, retry=retry - 1)
volume = create_volume_from_image(cinder, image)
site_a_hash = zaza.openstack.utilities.ceph.get_rbd_hash(
zaza.model.get_lead_unit_name('ceph-mon',
model_name=self.site_a_model),
'cinder-ceph',
'volume-{}'.format(volume.id),
model_name=self.site_a_model)
self.wait_for_mirror_state('up+replaying',
check_entries_behind_master=True,
application_name=self.application_name +
self.site_b_app_suffix,
model_name=self.site_b_model)
site_b_hash = zaza.openstack.utilities.ceph.get_rbd_hash(
zaza.model.get_lead_unit_name('ceph-mon' + self.site_b_app_suffix,
model_name=self.site_b_model),
'cinder-ceph',
'volume-{}'.format(volume.id),
model_name=self.site_b_model)
logging.info(site_a_hash)
logging.info(site_b_hash)
self.assertEqual(site_a_hash, site_b_hash)
def setUpClass(cls):
"""Run class setup for running cinder tests."""
super(CinderBackupSwiftTest, cls).setUpClass()
cls.cinder_client = openstack_utils.get_cinder_session_client(
cls.keystone_session)
