def test_contrail_services_status_after_restart_master_node(os_faults_steps):
"""Verify contrail services status after master node restart.
Steps:
#. Restart node with contrail-schema (active)
#. Wait some time
#. Check that contrail services statuses is correct
"""
services_statuses = contrail_status.get_services_statuses(os_faults_steps)
master_node_fqdn = None
for fqdn, services in services_statuses.items():
for service in services:
if (service['service'] == 'contrail-schema' and
service['status'] == contrail_status.STATUS_ACTIVE):
master_node_fqdn = fqdn
break
assert master_node_fqdn is not None, "Can't find master node"
master_node = os_faults_steps.get_node(fqdns=[master_node_fqdn])
os_faults_steps.reset_nodes(master_node)
waiter.wait(
contrail_status.check_services_statuses,
args=(os_faults_steps),
expected_exceptions=AssertionError,
timeout=settings.CONTRAIL_NODE_RESET_TIMEOUT)
def check_flavors_limited_in_launch_instance_form(self, image_name,
disk_size, ram_size):
"""Step to check flavors are limited in launch instance form."""
with self._page_images() as page:
page.table_images.row(
name=image_name).dropdown_menu.item_default.click()
with page.form_launch_instance as form:
form.item_flavor.click()
waiter.wait(lambda: (form.tab_flavor.table_available_flavors.
rows, "No available flavors"),
timeout_seconds=30,
sleep_seconds=0.1)
for row in form.tab_flavor.table_available_flavors.rows:
ram_cell = row.cell('ram')
if utils.get_size(ram_cell.value, to='mb') < ram_size:
assert_that(ram_cell.label_alert.is_present,
equal_to(True))
else:
assert_that(ram_cell.label_alert.is_present,
equal_to(False))
disk_cell = row.cell('root_disk')
if utils.get_size(disk_cell.value, to='gb') < disk_size:
assert_that(disk_cell.label_alert.is_present,
equal_to(True))
else:
assert_that(disk_cell.label_alert.is_present,
equal_to(False))
form.cancel()
def check_ironic_nodes_provision_state(self, nodes, state, node_timeout=0):
"""Check ironic node provision state was changed.
Args:
nodes (list): the list of ironic nodes.
state (str): the provision state mode.
node_timeout (int): seconds to wait a result of check.
Raises:
TimeoutExpired: if check failed after timeout.
"""
expected_provision_state = {node.uuid: state for node in nodes}
def _check_ironic_nodes_provision_state():
actual_provision_state = {}
for node in nodes:
try:
self._get_node(node)
actual_provision_state[node.uuid] = node.provision_state
except exceptions.NotFound:
actual_provision_state[node.uuid] = None
return waiter.expect_that(actual_provision_state,
equal_to(expected_provision_state))
timeout = len(nodes) * node_timeout
waiter.wait(_check_ironic_nodes_provision_state,
timeout_seconds=timeout)
def resize_instance(self, instance_name, flavor=None, check=True):
"""Step to resize instance."""
page_instances = self._page_instances()
flavor = flavor or config.FLAVOR_TINY
with page_instances.table_instances.row(
name=instance_name).dropdown_menu as menu:
menu.button_toggle.click()
menu.item_resize.click()
with page_instances.form_resize_instance as form:
form.wait_for_presence()
form.item_flavor.value = flavor
form.submit()
with page_instances.table_instances.row(name=instance_name) as row:
def _poll_instance_status():
return row.cell('status').value == config.CONFIM_RESIZE_STATUS
waiter.wait(lambda: _poll_instance_status(),
timeout_seconds=config.VERIFY_RESIZE_TIMEOUT,
sleep_seconds=config.VERIFY_RESIZE_SLEEP) is True
page_instances.table_instances.row(
name=instance_name).dropdown_menu.item_default.click()
if check:
row = page_instances.table_instances.row(name=instance_name)
assert_that(row.cell('size').value, contains_string(flavor))
def test_waiter_supports_expected_exceptions():
"""Verify that waiter supports expected exceptions."""
def predicate():
raise TypeError('error')
with pytest.raises(waiter.TimeoutExpired):
waiter.wait(predicate, expected_exceptions=(TypeError, ))
def revert_environment(destructor, snapshot_name):
"""Revert environment to original state."""
nodes = destructor.get_nodes()
# Sometimes revert fails with
# internal error: process exited while connecting to monitor
# so run several times (until it pass successful)
for _ in range(5):
try:
nodes.revert(snapshot_name)
break
except Exception:
time.sleep(5)
# Wait some time for preventing ansible freezes on time synchronization.
time.sleep(10)
waiter.wait(nodes.run_task,
args=({
'command': 'hwclock --hctosys'
}, ),
timeout_seconds=config.NODES_AVAILABILITY_TIMEOUT,
predicate_timeout=60,
expected_exceptions=(executor.AnsibleExecutionUnreachable,
executor.AnsibleExecutionException))
# Restart ceph services
nodes.run_task({'command': 'systemctl restart ceph-\*.service'},
raise_on_error=False)
nodes.run_task({'command': 'rm -rf {}'.format(config.RADOSGW_SOCK_FILE)},
raise_on_error=False)
nodes.run_task({'command': 'systemctl restart radosgw.service'},
raise_on_error=False)
def check_service_state(self,
service_name,
nodes,
must_run=True,
timeout=0):
"""Verify step to check that service is running or not on nodes.
Args:
service_name (str): name of service
nodes (obj): NodeCollection instance to check service state on it
must_run (bool): flag whether service should be run on nodes or not
timeout (int, optional): seconds to wait a result of check
Raises:
TimeoutExpired: if service state is wrong after timeout
"""
service = self._client.get_service(service_name)
def _check_service_state():
matcher = has_items(*nodes)
if not must_run:
matcher = is_not(matcher)
return expect_that(service.get_nodes(), matcher)
waiter.wait(_check_service_state, timeout_seconds=timeout)
def check_ports_presence(self, ports, must_present=True, port_timeout=0):
"""Step to check ports is present.
Args:
ports (list): list of ironic ports
must_present (bool): flag whether ports should be present or not
port_timeout (int): seconds to wait a result of check
Raises:
TimeoutExpired: if check failed after timeout
"""
expected_presence = {port.uuid: must_present for port in ports}
def _check_ports_presence():
actual_presence = {}
for port in ports:
try:
self._client.port.get(port.uuid)
actual_presence[port.uuid] = True
except exceptions.NotFound:
actual_presence[port.uuid] = False
return waiter.expect_that(actual_presence,
equal_to(expected_presence))
timeout = len(ports) * port_timeout
waiter.wait(_check_ports_presence, timeout_seconds=timeout)
def check_ironic_nodes_power_state(self, nodes, state, node_timeout=0):
"""Check ironic node power state was changed.
Args:
nodes (list): The list of ironic nodes.
state (str): the power state mode; `on` to put the node in power
state mode on; `off` to put the node in power state mode off;
`reboot` to reboot the node
node_timeout (int): seconds to wait a result of check
Raises:
TimeoutExpired: if check failed after timeout
"""
expected_power_state = {node.uuid: state for node in nodes}
def _check_ironic_nodes_power_state():
actual_power_state = {}
for node in nodes:
try:
self._client.node.get(node.uuid)
actual_power_state[node.uuid] = state
except exceptions.NotFound:
actual_power_state[node.uuid] = False
return expect_that(actual_power_state,
equal_to(expected_power_state))
timeout = len(nodes) * node_timeout
waiter.wait(_check_ironic_nodes_power_state, timeout_seconds=timeout)
def check_backup_status(self,
backup,
status,
transit_statuses=(config.STATUS_CREATING,),
timeout=0):
"""Check step volume backup status.
Args:
backup (object or str): volume backup object or its id
to check status
status (str): backup status name to check
transit_statuses (tuple): possible backup transitional statuses
timeout (int): seconds to wait a result of check
Raises:
TimeoutExpired: if check failed after timeout
"""
if not hasattr(backup, 'id'):
backup = self.get_backup_by_id(backup)
transit_matchers = [
equal_to_ignoring_case(st) for st in transit_statuses
]
def _check_backup_status():
backup.get()
return waiter.expect_that(backup.status,
is_not(any_of(*transit_matchers)))
waiter.wait(_check_backup_status, timeout_seconds=timeout)
err_msg = self._error_message(backup)
assert_that(backup.status, equal_to_ignoring_case(status), err_msg)
def check_output_show(self,
stack,
output_key,
expected_attr_values=None,
timeout=0):
"""Step to check stack attributes.
Args:
stack (obj): stack object
output_key (str): the name of a stack output
expected_attr_values (dict|None): expected attribute values
If None, only check that elements of output_show are not empty
timeout (int): seconds to wait a result of check
Raises:
TimeoutExpired: if check failed after timeout
"""
def _check_output_show():
output_show = self.get_stack_output_show(stack, output_key)
if expected_attr_values:
matcher = equal_to(expected_attr_values)
else:
matcher = only_contains(is_not(empty()))
return waiter.expect_that(output_show['output'], matcher)
waiter.wait(_check_output_show, timeout_seconds=timeout)
def check_service_instance_ready(contrail_api_client, service_instance,
server_steps):
"""Check that service instance creates nova server and it's booted."""
def _get_virtual_machine_uuid():
fresh_instance = contrail_api_client.service_instance_read(
id=service_instance.uuid)
refs = fresh_instance.get_virtual_machine_back_refs()
if refs:
return refs[0]['uuid']
server_uuid = waiter.wait(
_get_virtual_machine_uuid,
timeout_seconds=settings.SERVICE_INSTANCE_CREATE_TIMEOUT)
server = next(server for server in server_steps.get_servers()
if server.id == server_uuid)
server_steps.check_server_status(
server,
expected_statuses=[stepler_config.STATUS_ACTIVE],
transit_statuses=[stepler_config.STATUS_BUILD],
timeout=stepler_config.SERVER_ACTIVE_TIMEOUT)
def _check_record_in_log():
server.get()
console = server.get_console_output()
return re.search(settings.SERVICE_INSTANCE_BOOT_DONE_PATTERN, console)
waiter.wait(
_check_record_in_log,
timeout_seconds=settings.SERVICE_INSTANCE_BOOT_TIMEOUT)
def check_resource_presence(resource_id,
getter,
must_present=True,
timeout=0):
"""Step to check that resource is present.
Args:
resource_id (str): resource id
getter (obj): method to get resource
must_present (bool): flag whether resource should be present or not
timeout (int): seconds to wait a result of check
Raises:
TimeoutExpired: if check failed after timeout
exception
"""
def _check_resource_presence():
try:
resource = getter(resource_id)
except exceptions.DecapodAPIError:
is_present = False
else:
is_present = True if resource['time_deleted'] == 0 else False
return waiter.expect_that(is_present, equal_to(must_present))
waiter.wait(_check_resource_presence, timeout_seconds=timeout)
def check_aggregate_presence(self,
aggregate,
must_present=True,
timeout=0):
"""Verify step to check aggregate is present or not.
Args:
aggregate (object): nova aggregate to check presence status
must_present (bool): flag whether aggregate should present or not
timeout (int): seconds to wait a result of check
Raises:
TimeoutExpired: if check was triggered to False after timeout
"""
err_msg = "Invalid presence status of aggregate {}".format(
aggregate.id)
if timeout:
err_msg += " after {} second(s) of waiting".format(timeout)
def _check_aggregate_presence():
try:
# After deleting aggregate `get` method still returns object,
# so 'find' method is used
self._client.find(id=aggregate.id)
is_present = True
except exceptions.NotFound:
is_present = False
return waiter.expect_that(is_present, equal_to(must_present),
err_msg)
waiter.wait(_check_aggregate_presence, timeout_seconds=timeout)
def check_keypairs_presence(self,
keypairs,
must_present=True,
keypair_timeout=0):
"""Step to check keypairs presence status.
Args:
keypairs (list): Keypairs to check.
must_present (bool, optional): Flag whether keypairs must present
or not.
timeout (int, optional): Seconds to wait check result.
Raises:
TimeoutExpired: If check failed after timeout.
"""
expected_presence = {keypair.id: must_present for keypair in keypairs}
def _check_keypairs_presence():
actual_presence = {}
for keypair in keypairs:
try:
keypair.get()
actual_presence[keypair.id] = True
except nova_exceptions.NotFound:
actual_presence[keypair.id] = False
return expect_that(actual_presence, equal_to(expected_presence))
timeout = len(keypairs) * keypair_timeout
waiter.wait(_check_keypairs_presence, timeout_seconds=timeout)
def check_services_up(self, host_names=None, timeout=0):
"""Step to check that nova services are up.
If host_names is specified service states atr checked only on
these hosts.
Args:
host_names (list, optional): list of host names
timeout (int, optional): seconds to wait result of check
Raises:
TimeoutExpired: if check failed after timeout
"""
def _check_services_up():
services = self.get_services()
service_data = self._get_service_data(services)
expected_service_data = []
for data in service_data:
binary, host_name, status = data[0:3]
if host_names and host_name not in host_names:
expected_data = [binary, host_name, status, 'down']
else:
expected_data = [binary, host_name, 'enabled', 'up']
expected_service_data.append(expected_data)
return waiter.expect_that(service_data,
equal_to(expected_service_data))
waiter.wait(_check_services_up, timeout_seconds=timeout)
def check_ironic_chassis_presence(self,
chassis_list,
must_present=True,
chassis_timeout=0):
"""Verify step to check ironic chassis is present.
Args:
chassis_list (list): list of ironic chassis to check presence
status
must_present (bool): flag whether chassis should present or not
chassis_timeout (int): seconds to wait a result of check
Raises:
TimeoutExpired: if check failed after timeout
"""
expected_presence = {
chassis.uuid: must_present
for chassis in chassis_list
}
def _check_chassis_presence():
actual_presence = {}
for chassis in chassis_list:
try:
self._client.get(chassis.uuid)
actual_presence[chassis.uuid] = True
except exceptions.NotFound:
actual_presence[chassis.uuid] = False
return waiter.expect_that(actual_presence,
equal_to(expected_presence))
timeout = len(chassis_list) * chassis_timeout
waiter.wait(_check_chassis_presence, timeout_seconds=timeout)
def check_role_permission_presence(self,
role_id,
permission,
group_name=config.PERMISSIONS_GROUP_API,
must_present=True,
timeout=0):
"""Step to check that permission is present for role.
Args:
role_id (str): role id
permission (str): permission name
group_name (str): permissions group name
must_present (bool): flag whether permission should be
present or not
timeout (int): seconds to wait a result of check
Raises:
TimeoutExpired: if check failed after timeout
"""
def _check_role_permission_presence():
permissions = self.get_role_permissions_by_group(
role_id, group_name)
matcher = is_in(permissions)
if not must_present:
matcher = is_not(matcher)
return waiter.expect_that(permission, matcher)
waiter.wait(_check_role_permission_presence, timeout_seconds=timeout)
def check_port_interface_presence(self,
router,
port,
must_present=True,
timeout=0):
"""Verify step to check port is in router interfaces.
Args:
router (dict): router to check
port (dict): port to be found in router interfaces
must_present (bool): flag whether router should contain interface
to port or not
timeout (int): seconds to wait a result of check
Raises:
TimeoutExpired: if check failed after timeout
"""
def _check_port_interface_presence():
ports = self._client.get_interfaces_ports(router['id'])
matcher = is_in(ports)
if not must_present:
matcher = is_not(matcher)
return waiter.expect_that(port, matcher)
waiter.wait(_check_port_interface_presence, timeout_seconds=timeout)
def check_role_presence(self, role_id, must_present=True, timeout=0):
"""Check step that role is present.
Args:
role_id (str or obj): the role to be checked on the server
must_present (bool): flag whether role should present or no
timeout (int): seconds to wait a result of check
Raises:
TimeoutExpired/DecapodAPIError: if check failed after timeout or
API response exception
"""
def _check_role_presence():
try:
role = self._client.get_role(role_id)
if role['time_deleted'] == 0:
is_present = True
if role['time_deleted'] != 0:
is_present = False
except exceptions.DecapodAPIError:
is_present = False
return waiter.expect_that(is_present, equal_to(must_present))
waiter.wait(_check_role_presence, timeout_seconds=timeout)
def check_stack_status(self,
stack,
status,
transit_statuses=(),
timeout=0):
"""Verify step to check stack's `stack_status` property.
Args:
stack (obj): heat stack to check its status
status (str): expected stack status
transit_statuses (iterable): allowed transit statuses
timeout (int): seconds to wait a result of check
Raises:
TimeoutExpired: if check was failed after timeout
"""
def _check_stack_status():
stack.get()
value = self._get_property(stack,
'stack_status',
transit_values=transit_statuses,
timeout=timeout)
return waiter.expect_that(value.lower(), equal_to(status.lower()))
waiter.wait(_check_stack_status, timeout_seconds=timeout)
def check_flavor_presence(self, flavor, must_present=True, timeout=0):
"""Verify step to check flavor is present.
Args:
flavor (object): nova flavor to check presence status
must_present (bool): flag whether flavor should present or not
timeout (int): seconds to wait a result of check
Raises:
TimeoutExpired: if check was triggered to False after timeout
"""
err_msg = "Invalid presence status of flavor " \
"with ID {!r}".format(flavor.id)
if timeout:
err_msg += " after {!r} second(s) of waiting".format(timeout)
def _check_flavor_presence():
try:
# After deleting flavor `get` method still return object,
# so it was changed to find
self._client.find(id=flavor.id)
is_present = True
except exceptions.NotFound:
is_present = False
return waiter.expect_that(is_present, equal_to(must_present),
err_msg)
waiter.wait(_check_flavor_presence, timeout_seconds=timeout)
def check_admin_instances_pagination_filter(self, instance_names):
"""Step to check instances pagination with filtering as admin."""
instance_name = self._get_current_instance_name(instance_names)
page_instances = self._page_admin_instances()
page_instances.table_instances.link_next.wait_for_presence()
page_instances.table_instances.link_prev.wait_for_absence()
page_instances.table_instances.link_next.click()
page_instances.table_instances.link_next.wait_for_presence()
page_instances.table_instances.link_prev.wait_for_presence()
page_instances.item_instance_parameter(config.INSTANCE_NAME).click()
page_instances.field_filter_instances.value = instance_name
page_instances.button_filter_instances.click()
def check_rows():
is_present = False
for row in page_instances.table_instances.rows:
if not (row.is_present
and instance_name in row.link_instance.value):
break
is_present = True
return waiter.expect_that(is_present, equal_to(True))
waiter.wait(check_rows,
timeout_seconds=config.UI_TIMEOUT,
sleep_seconds=0.1)
def check_snapshots_presence(self,
snapshots,
must_present=True,
timeout=0):
"""Step to check snapshots presence status.
Args:
snapshots (list): cinder volume snapshots to check presence status
must_present (bool): flag whether snapshot should present
timeout (int): seconds to wait a result of check
Raises:
TimeoutExpired: if check failed after timeout
"""
snapshot_ids = [snapshot.id for snapshot in snapshots]
# Make a dict with desired presence values for each snapshot
expected_presence = dict.fromkeys(snapshot_ids, must_present)
def _check_snapshots_presence():
# Make a dict with actual presence values for each snapshot
actual_presence = dict.fromkeys(snapshot_ids, False)
for snapshot in self._client.list():
if snapshot.id in actual_presence:
actual_presence[snapshot.id] = True
return expect_that(actual_presence, equal_to(expected_presence))
waiter.wait(_check_snapshots_presence, timeout_seconds=timeout)
def test_waiter_raises_unexpected_exceptions():
"""Verify that waiter supports expected exceptions."""
def predicate():
raise LookupError('error')
with pytest.raises(LookupError):
waiter.wait(predicate, expected_exceptions=(TypeError, ))
def check_image_bind_status(self,
image,
project,
must_bound=True,
timeout=0):
"""Check step image binding status.
Args:
image (object): image bound/unbound with project
project (object): project bound/unbound with image
must_bound (bool): flag whether project and image should be bound
or unbound
timeout (int): seconds to wait a result of check
Raises:
TimeoutExpired: if check failed after timeout
"""
def _check_image_bind_status():
members = self._client.image_members.list(image.id)
member_ids = [member['member_id'] for member in members]
if must_bound:
assert_that(project.id, is_in(member_ids))
is_bound = True
else:
is_bound = False
return waiter.expect_that(is_bound, equal_to(must_bound))
waiter.wait(_check_image_bind_status, timeout_seconds=timeout)
def check_ironic_nodes_maintenance(self, nodes, state, node_timeout=0):
"""Check ironic node maintenance was changed.
Args:
nodes (list): The list of ironic nodes.
state (Bool): the maintenance mode; either a Boolean or a string
representation of a Boolean (eg, 'true', 'on', 'false',
'off'). True to put the node in maintenance mode; False
to take the node out of maintenance mode.
node_timeout (int): seconds to wait a result of check.
Raises:
TimeoutExpired: if check failed after timeout.
"""
expected_maintenance = {node.uuid: state for node in nodes}
def _check_ironic_node_maintenance():
actual_maintenance = {}
for node in nodes:
try:
self._get_node(node)
actual_maintenance[node.uuid] = node.maintenance
except exceptions.NotFound:
actual_maintenance[node.uuid] = None
return waiter.expect_that(actual_maintenance,
equal_to(expected_maintenance))
timeout = len(nodes) * node_timeout
waiter.wait(_check_ironic_node_maintenance, timeout_seconds=timeout)
def check_volume_status(self,
volume,
statuses,
transit_statuses=(),
timeout=0):
"""Check step volume status.
Args:
volume (object|str): cinder volume to check status or its id
statuses (list): list of statuses to check
transit_statuses (tuple): possible volume transitional statuses
timeout (int): seconds to wait a result of check
Raises:
TimeoutExpired|AssertionError: if check failed after timeout
"""
transit_matchers = [
equal_to_ignoring_case(status) for status in transit_statuses
]
if not hasattr(volume, 'id'):
volume = self.get_volume_by_id(volume)
def _check_volume_status():
volume.get()
return waiter.expect_that(volume.status,
is_not(any_of(*transit_matchers)))
waiter.wait(_check_volume_status, timeout_seconds=timeout)
matchers = [equal_to_ignoring_case(status) for status in statuses]
assert_that(volume.status, any_of(*matchers))
def check_ironic_nodes_presence(self,
nodes,
must_present=True,
node_timeout=0):
"""Verify step to check ironic node is present.
Args:
nodes (list): list of ironic nodes.
must_present (bool): flag whether node should present or not.
node_timeout (int): seconds to wait a result of check.
Raises:
TimeoutExpired: if check failed after timeout.
"""
expected_presence = {node.uuid: must_present for node in nodes}
def _check_ironic_nodes_presence():
actual_presence = {}
for node in nodes:
try:
self._client.node.get(node.uuid)
actual_presence[node.uuid] = True
except exceptions.NotFound:
actual_presence[node.uuid] = False
return waiter.expect_that(actual_presence,
equal_to(expected_presence))
timeout = len(nodes) * node_timeout
waiter.wait(_check_ironic_nodes_presence, timeout_seconds=timeout)
def detach_fixed_ip(self, server, fixed_ip=None, check=True):
"""Step to detach provided fixed IP from provided server.
Args:
server (object): nova instance to delete fixed ip
fixed_ip (str): Fixed IP address
check (bool): flag whether to check step or not
Raises:
TimeoutExpired: if check failed after timeout
"""
if not fixed_ip:
fixed_ip = self.get_ips(server, 'fixed').keys()[0]
self._client.remove_fixed_ip(server.id, fixed_ip)
if check:
def _check_detach_fixed_ip():
server.get()
fixed_ips = self.get_ips(server, 'fixed').keys()
return expect_that(fixed_ip, is_not(is_in(fixed_ips)))
waiter.wait(_check_detach_fixed_ip,
timeout_seconds=config.SERVER_UPDATE_TIMEOUT)
