def disrupt_all_controller_nodes(disrupt_method=sh.hard_reset_method,
sequentially=False,
exclude_list=None):
# reboot all controllers and wait for ssh Up on them
# method : method of disruptino to use : reset | network_disruption
# hard reset is simultaneous while soft is sequential
# exclude_list = list of nodes to NOT reset
controlplane_groups = ['controller', 'messaging', 'database', 'networker']
actual_controlplane_groups = tripleo_topology.actual_node_groups(
controlplane_groups)
nodes = topology.list_openstack_nodes(group=actual_controlplane_groups)
# remove excluded nodes from reset list
if exclude_list:
nodes = [node for node in nodes if node.name not in exclude_list]
for controller in nodes:
if isinstance(disrupt_method, sh.RebootHostMethod):
reboot_node(controller.name,
wait=sequentially,
reboot_method=disrupt_method)
else:
# using ssh_client.connect we use a fire and forget reboot method
controller.ssh_client.connect().exec_command(disrupt_method)
LOG.info('disrupt exec: {} on server: {}'.format(
disrupt_method, controller.name))
tobiko.cleanup_fixture(controller.ssh_client)
if sequentially:
check_overcloud_node_responsive(controller)
if not sequentially:
for controller in topology.list_openstack_nodes(group='controller'):
check_overcloud_node_responsive(controller)
def reboot_all_controller_nodes(reboot_method=sh.hard_reset_method,
sequentially=False,
exclude_list=None):
# reboot all controllers and wait for ssh Up on them
# method : method of disruptino to use : hard or soft reset
# hard reset is simultaneous while soft is sequential
# exclude_list = list of nodes to NOT reset
controlplane_groups = ['controller', 'messaging', 'database', 'networker']
actual_controlplane_groups = tripleo_topology.actual_node_groups(
controlplane_groups)
nodes = topology.list_openstack_nodes(group=actual_controlplane_groups)
# remove excluded nodes from reset list
if exclude_list:
nodes = [node for node in nodes if node.name not in exclude_list]
for controller in nodes:
sh.reboot_host(ssh_client=controller.ssh_client,
wait=sequentially,
method=reboot_method)
LOG.info('reboot exec: {} on server: {}'.format(
reboot_method, controller.name))
tobiko.cleanup_fixture(controller.ssh_client)
if not sequentially:
for controller in topology.list_openstack_nodes(group='controller'):
check_overcloud_node_responsive(controller)
def close(self):
"""Ensures it is disconnected from remote SSH server
"""
try:
tobiko.cleanup_fixture(self)
except Exception:
LOG.exception(f"Failed closing SSH connection to '{self.login}'")
def check_overcloud_node_responsive(node):
node_checked = sh.execute("hostname",
ssh_client=node.ssh_client,
expect_exit_status=None).stdout
LOG.info('{} is up '.format(node_checked))
tobiko.cleanup_fixture(node.ssh_client)
def run(cls, after: bool, **params):
fixture = tobiko.get_fixture(cls)
params.setdefault('passive_checks_only', False)
params.setdefault('skip_mac_table_size_test', True)
skips = frozenset(k for k, v in params.items() if v)
if after or skips < fixture.skips:
# Force re-check
tobiko.cleanup_fixture(fixture)
else:
LOG.info("Will skip Overcloud health checks if already "
f"executed: {params}")
fixture.skips = skips
tobiko.setup_fixture(fixture)
def cleanup_fixtures(self, stream=None):
stream = stream or sys.stdout
test_cases = self.discover_testcases()
success = True
for fixture_name in tobiko.list_required_fixtures(test_cases):
output = fixture_name + '\n'
if six.PY2:
output = output.decode()
stream.write(output)
try:
tobiko.cleanup_fixture(fixture_name)
except Exception:
LOG.exception('Fixture %r cleanup failed', fixture_name)
success = False
if not success:
sys.exit(1)
def stop(self) -> 'CurlProcessFixture':
try:
process = self._process
except RuntimeError:
pass # process not started
else:
process.kill(sudo=self.sudo)
return tobiko.cleanup_fixture(self)
def wait_for_operation(self, timeout: tobiko.Seconds = None):
if self.is_rebooted:
return
try:
for attempt in tobiko.retry(
timeout=tobiko.min_seconds(timeout, self.time_left),
default_timeout=self.default_wait_timeout,
default_count=self.default_wait_count,
default_interval=self.default_wait_interval):
# ensure SSH connection is closed before retrying connecting
tobiko.cleanup_fixture(self.ssh_client)
assert self.ssh_client.client is None
LOG.debug(f"Getting uptime after reboot '{self.hostname}' "
"after reboot... ")
try:
up_time = _uptime.get_uptime(ssh_client=self.ssh_client,
timeout=30.)
except Exception:
# if disconnected while getting up time we assume the VM is
# just rebooting. These are good news!
LOG.debug(
"Unable to get uptime from host "
f"'{self.hostname}'",
exc_info=1)
attempt.check_limits()
else:
# verify that reboot actually happened by comparing elapsed
# time with up_time
elapsed_time = self.elapsed_time
if up_time < elapsed_time:
assert self.ssh_client.client is not None
self.is_rebooted = True
LOG.debug(f"Host '{self.hostname}' restarted "
f"{elapsed_time} seconds after "
f"reboot operation (up_time={up_time})")
break
else:
LOG.debug(f"Host '{self.hostname}' still not "
f"restarted {elapsed_time} seconds after "
f"reboot operation (up_time={up_time!r})")
attempt.check_limits()
finally:
if not self.is_rebooted:
self.ssh_client.close()
def test_servers_creation(stack=TestServerCreationStack,
number_of_servers=2) -> \
tobiko.Selection[_nova.ServerStackFixture]:
initial_servers_ids = {server.id for server in nova.list_servers()}
pid = os.getpid()
fixture_obj = tobiko.get_fixture_class(stack)
# Get list of server stack instances
fixtures: tobiko.Selection[_nova.ServerStackFixture] = tobiko.select(
tobiko.get_fixture(fixture_obj, fixture_id=f'{pid}-{i}')
for i in range(number_of_servers or 1))
test_case = tobiko.get_test_case()
# Check fixtures types
for fixture in fixtures:
test_case.assertIsInstance(fixture, _nova.ServerStackFixture)
# Delete all servers stacks
for fixture in fixtures:
tobiko.cleanup_fixture(fixture)
# Create all servers stacks
for fixture in fixtures:
tobiko.use_fixture(fixture)
# Check every server ID is unique and new
server_ids = {fixture.server_id for fixture in fixtures}
test_case.assertEqual(number_of_servers or 1, len(server_ids))
test_case.assertFalse(server_ids & initial_servers_ids)
# sleep for 20 sec , ensure no race condition with ssh
time.sleep(20)
# Test SSH connectivity to floating IP address
for fixture in fixtures:
test_case.assertTrue(sh.get_hostname(ssh_client=fixture.ssh_client))
# Test pinging to floating IP address
ping.assert_reachable_hosts(fixture.floating_ip_address
for fixture in fixtures)
return fixtures
async def cleanup_actor(obj: ActorType,
fixture_id: typing.Optional[str] = None,
manager=None,
timeout: tobiko.Seconds = None) -> _actor.ActorRef[A]:
actor = tobiko.cleanup_fixture(obj,
fixture_id=fixture_id,
manager=manager)
await asyncio.wait_for(actor.cleanup_actor_future,
timeout=timeout)
return actor.ref
def reset_all_compute_nodes(hard_reset=False):
# reboot all computes and wait for ssh Up on them
# hard reset is simultaneous while soft is sequential
if hard_reset:
reset_method = sh.hard_reset_method
else:
reset_method = sh.soft_reset_method
for compute in topology.list_openstack_nodes(group='compute'):
# using ssh_client.connect we use a fire and forget reboot method
sh.reboot_host(ssh_client=compute.ssh_client,
wait=False,
method=reset_method)
LOG.info('reboot exec: {} on server: {}'.format(
reset_method, compute.name))
tobiko.cleanup_fixture(compute.ssh_client)
for compute in topology.list_openstack_nodes(group='compute'):
compute_checked = sh.execute("hostname",
ssh_client=compute.ssh_client,
expect_exit_status=None).stdout
LOG.info('{} is up '.format(compute_checked))
def test_1_create_server(self):
"""Test Nova server creation
"""
tobiko.cleanup_fixture(self.stack)
self.ensure_server()
self.stack.assert_is_reachable()
def tearDownClass(cls) -> None:
tobiko.cleanup_fixture(cls.stack.fixture)
async def stop_actor(obj: ActorType,
fixture_id: typing.Optional[str] = None,
manager=None) -> _actor.ActorRef[A]:
return tobiko.cleanup_fixture(obj,
fixture_id=fixture_id,
manager=manager).ref
def _test_cleanup_fixture(self, obj):
result = tobiko.cleanup_fixture(obj)
self.assertIs(tobiko.get_fixture(obj), result)
result.setup_fixture.assert_not_called()
result.cleanup_fixture.assert_called_once_with()
def cleanup_fixture(self):
tobiko.cleanup_fixture(self._connections)
self._names.clear()
self._groups.clear()
self._addresses.clear()
def setUp(self):
super(SSHClientFixtureTest, self).setUp()
tobiko.cleanup_fixture(self.fixture)
self.ssh_client = mock.MagicMock(specs=paramiko.SSHClient)
self.patch(paramiko, 'SSHClient', return_value=self.ssh_client)
def close(self) -> 'ShellConnection':
return tobiko.cleanup_fixture(self)