def negative_manual_cic_maintenance_mode(self):
"""Check negative scenario for manual maintenance mode
Scenario:
1. Revert snapshot
2. Disable UMM
3. Switch in maintenance mode
4. Check the controller not switching in maintenance mode
5. Check the controller become available
Duration 45m
"""
self.env.revert_snapshot('cic_maintenance_mode')
cluster_id = self.fuel_web.get_last_created_cluster()
# Select a non-primary controller
regular_ctrl = self.fuel_web.get_nailgun_node_by_name("slave-02")
dregular_ctrl = self.fuel_web.get_devops_node_by_nailgun_node(
regular_ctrl)
_ip = regular_ctrl['ip']
_id = regular_ctrl['id']
asserts.assert_true('True' in check_available_mode(_ip),
"Maintenance mode is not available")
self.ssh_manager.execute_on_remote(
ip=_ip,
cmd="umm disable")
asserts.assert_false('True' in check_available_mode(_ip),
"Maintenance mode should not be available")
logger.info('Try to execute maintenance mode '
'for node-{0}'.format(_id))
self.ssh_manager.execute_on_remote(
ip=_ip,
cmd="umm on",
assert_ec_equal=[1])
# If we don't disable maintenance mode,
# the node would have gone to reboot, so we just expect
time.sleep(30)
asserts.assert_true(
self.fuel_web.get_nailgun_node_by_devops_node(dregular_ctrl)
['online'],
'Node-{0} should be online after command "umm on"'.
format(_id))
try:
self.fuel_web.run_ostf(cluster_id, test_sets=['ha', 'smoke',
'sanity'])
except AssertionError:
logger.debug("Test failed from first probe,"
" we sleep 300 second try one more time"
" and if it fails again - test will fails ")
time.sleep(300)
self.fuel_web.run_ostf(cluster_id, test_sets=['ha', 'smoke',
'sanity'])
def negative_manual_cic_maintenance_mode(self):
"""Check negative scenario for manual maintenance mode
Scenario:
1. Revert snapshot
2. Disable UMM
3. Switch in maintenance mode
4. Check the controller not switching in maintenance mode
5. Check the controller become available
Duration 45m
"""
self.env.revert_snapshot('cic_maintenance_mode')
cluster_id = self.fuel_web.get_last_created_cluster()
# Select a non-primary controller
regular_ctrl = self.fuel_web.get_nailgun_node_by_name("slave-02")
dregular_ctrl = self.fuel_web.get_devops_node_by_nailgun_node(
regular_ctrl)
_ip = regular_ctrl['ip']
_id = regular_ctrl['id']
asserts.assert_true('True' in check_available_mode(_ip),
"Maintenance mode is not available")
self.ssh_manager.execute_on_remote(ip=_ip, cmd="umm disable")
asserts.assert_false('True' in check_available_mode(_ip),
"Maintenance mode should not be available")
logger.info('Try to execute maintenance mode '
'for node-{0}'.format(_id))
self.ssh_manager.execute_on_remote(ip=_ip,
cmd="umm on",
assert_ec_equal=[1])
# If we don't disable maintenance mode,
# the node would have gone to reboot, so we just expect
time.sleep(30)
asserts.assert_true(
self.fuel_web.get_nailgun_node_by_devops_node(
dregular_ctrl)['online'],
'Node-{0} should be online after command "umm on"'.format(_id))
try:
self.fuel_web.run_ostf(cluster_id,
test_sets=['ha', 'smoke', 'sanity'])
except AssertionError:
logger.debug("Test failed from first probe,"
" we sleep 300 second try one more time"
" and if it fails again - test will fails ")
time.sleep(300)
self.fuel_web.run_ostf(cluster_id,
test_sets=['ha', 'smoke', 'sanity'])
def manual_cic_maintenance_mode(self):
"""Check manual maintenance mode for controller
Scenario:
1. Revert snapshot
2. Switch in maintenance mode
3. Wait until controller is rebooting
4. Exit maintenance mode
5. Check the controller become available
Duration 155m
"""
self.env.revert_snapshot('cic_maintenance_mode')
cluster_id = self.fuel_web.get_last_created_cluster()
# Select a non-primary controller
regular_ctrl = self.fuel_web.get_nailgun_node_by_name("slave-02")
dregular_ctrl = self.fuel_web.get_devops_node_by_nailgun_node(
regular_ctrl)
_ip = regular_ctrl['ip']
_id = regular_ctrl['id']
logger.info('Maintenance mode for node-{0}'.format(_id))
asserts.assert_true('True' in check_available_mode(_ip),
"Maintenance mode is not available")
self.ssh_manager.check_call(ip=_ip, command="umm on", expected=[-1])
self.fuel_web.wait_node_is_offline(dregular_ctrl)
asserts.assert_true(
checkers.check_ping(self.env.get_admin_node_ip(),
_ip,
deadline=600),
"Host {0} is not reachable by ping during 600 sec".format(_ip))
asserts.assert_true('True' in check_auto_mode(_ip),
"Maintenance mode is not switched on")
self.ssh_manager.check_call(ip=_ip, command="umm off")
self.fuel_web.wait_node_is_online(dregular_ctrl)
# Wait until Cinder services UP on a controller
self.fuel_web.wait_cinder_is_up([dregular_ctrl.name])
# Wait until RabbitMQ cluster is UP
wait_pass(lambda: self.fuel_web.run_single_ostf_test(
cluster_id,
test_sets=['ha'],
test_name=ostf_test_mapping.OSTF_TEST_MAPPING.get(
'RabbitMQ availability')),
timeout=1500)
logger.info('RabbitMQ cluster is available')
wait_pass(lambda: self.fuel_web.run_single_ostf_test(
cluster_id,
test_sets=['sanity'],
test_name=ostf_test_mapping.OSTF_TEST_MAPPING.get(
'Check that required services are running')),
timeout=1500)
logger.info("Required services are running")
# TODO(astudenov): add timeout_msg
try:
self.fuel_web.run_ostf(cluster_id,
test_sets=['smoke', 'sanity', 'ha'])
except AssertionError:
logger.debug("Test failed from first probe,"
" we sleep 600 second try one more time"
" and if it fails again - test will fails ")
time.sleep(600)
self.fuel_web.run_ostf(cluster_id,
test_sets=['smoke', 'sanity', 'ha'])
def negative_auto_cic_maintenance_mode(self):
"""Check negative scenario for auto maintenance mode
Scenario:
1. Revert snapshot
2. Disable UMM
3. Change UMM.CONF
4. Unexpected reboot
5. Check the controller not switching in maintenance mode
6. Check the controller become available
Duration 85m
"""
self.env.revert_snapshot('cic_maintenance_mode')
cluster_id = self.fuel_web.get_last_created_cluster()
# Select a non-primary controller
regular_ctrl = self.fuel_web.get_nailgun_node_by_name("slave-02")
dregular_ctrl = self.fuel_web.get_devops_node_by_nailgun_node(
regular_ctrl)
_ip = regular_ctrl['ip']
_id = regular_ctrl['id']
asserts.assert_true('True' in check_available_mode(_ip),
"Maintenance mode is not available")
logger.info('Disable UMM on node-{0}'.format(_id))
change_config(_ip, umm=False, reboot_count=0)
asserts.assert_false('True' in check_available_mode(_ip),
"Maintenance mode should not be available")
logger.info('Unexpected reboot on node-{0}'.format(_id))
self.ssh_manager.check_call(ip=_ip, command='reboot >/dev/null & ')
wait(
lambda: not checkers.check_ping(self.env.get_admin_node_ip(), _ip),
timeout=60 * 10,
timeout_msg='Node {} still responds to ping'.format(
dregular_ctrl.name))
# Node don't have enough time for set offline status
# after reboot
# Just waiting
asserts.assert_true(
checkers.check_ping(self.env.get_admin_node_ip(),
_ip,
deadline=600),
"Host {0} is not reachable by ping during 600 sec".format(_ip))
self.fuel_web.wait_node_is_online(dregular_ctrl)
logger.info('Check that node-{0} not in maintenance mode after'
' unexpected reboot'.format(_id))
wait(lambda: tcp_ping(_ip, 22),
timeout=60 * 10,
timeout_msg='Node {} still is not available by SSH'.format(
dregular_ctrl.name))
asserts.assert_false('True' in check_auto_mode(_ip),
"Maintenance mode should not switched")
# Wait until MySQL Galera is UP on some controller
self.fuel_web.wait_mysql_galera_is_up([dregular_ctrl.name])
# Wait until Cinder services UP on a controller
self.fuel_web.wait_cinder_is_up([dregular_ctrl.name])
# Wait until RabbitMQ cluster is UP
wait_pass(lambda: self.fuel_web.run_single_ostf_test(
cluster_id,
test_sets=['ha'],
test_name=ostf_test_mapping.OSTF_TEST_MAPPING.get(
'RabbitMQ availability')),
timeout=1500)
logger.info('RabbitMQ cluster is available')
# TODO(astudenov): add timeout_msg
wait_pass(lambda: self.fuel_web.run_single_ostf_test(
cluster_id,
test_sets=['sanity'],
test_name=ostf_test_mapping.OSTF_TEST_MAPPING.get(
'Check that required services are running')),
timeout=1500)
logger.info("Required services are running")
try:
self.fuel_web.run_ostf(cluster_id,
test_sets=['smoke', 'sanity', 'ha'])
except AssertionError:
logger.debug("Test failed from first probe,"
" we sleep 600 second try one more time"
" and if it fails again - test will fails ")
time.sleep(600)
self.fuel_web.run_ostf(cluster_id,
test_sets=['smoke', 'sanity', 'ha'])
def auto_cic_maintenance_mode(self):
"""Check auto maintenance mode for controller
Scenario:
1. Revert snapshot
2. Unexpected reboot
3. Wait until controller is switching in maintenance mode
4. Exit maintenance mode
5. Check the controller become available
Duration 155m
"""
self.env.revert_snapshot('cic_maintenance_mode')
cluster_id = self.fuel_web.get_last_created_cluster()
# Select a non-primary controller
regular_ctrl = self.fuel_web.get_nailgun_node_by_name("slave-02")
dregular_ctrl = self.fuel_web.get_devops_node_by_nailgun_node(
regular_ctrl)
_ip = regular_ctrl['ip']
_id = regular_ctrl['id']
asserts.assert_true('True' in check_available_mode(_ip),
"Maintenance mode is not available")
change_config(_ip, reboot_count=0)
logger.info('Change UMM.CONF on node-{0}'.format(_id))
logger.info('Unexpected reboot on node-{0}'.format(_id))
command = 'reboot --force >/dev/null & '
self.ssh_manager.execute_on_remote(ip=_ip, cmd=command)
wait(
lambda: not checkers.check_ping(self.env.get_admin_node_ip(), _ip),
timeout=60 * 10,
timeout_msg='Node {} still responds to ping'.format(
dregular_ctrl.name))
self.fuel_web.wait_node_is_offline(dregular_ctrl)
logger.info('Check that node-{0} in maintenance mode after'
' unexpected reboot'.format(_id))
asserts.assert_true(
checkers.check_ping(self.env.get_admin_node_ip(),
_ip,
deadline=600),
"Host {0} is not reachable by ping during 600 sec".format(_ip))
asserts.assert_true('True' in check_auto_mode(_ip),
"Maintenance mode is not switched on")
logger.info('turn off Maintenance mode')
self.ssh_manager.execute_on_remote(ip=_ip, cmd="umm off")
time.sleep(30)
change_config(_ip)
self.fuel_web.wait_node_is_online(dregular_ctrl)
# Wait until MySQL Galera is UP on some controller
self.fuel_web.wait_mysql_galera_is_up([dregular_ctrl.name])
# Wait until Cinder services UP on a controller
self.fuel_web.wait_cinder_is_up([dregular_ctrl.name])
# Wait until RabbitMQ cluster is UP
wait_pass(lambda: self.fuel_web.run_single_ostf_test(
cluster_id,
test_sets=['ha'],
test_name=ostf_test_mapping.OSTF_TEST_MAPPING.get(
'RabbitMQ availability')),
timeout=1500)
logger.info('RabbitMQ cluster is available')
# Wait until all Openstack services are UP
wait_pass(lambda: self.fuel_web.run_single_ostf_test(
cluster_id,
test_sets=['sanity'],
test_name=ostf_test_mapping.OSTF_TEST_MAPPING.get(
'Check that required services are running')),
timeout=1500)
logger.info("Required services are running")
try:
self.fuel_web.run_ostf(cluster_id,
test_sets=['smoke', 'sanity', 'ha'])
except AssertionError:
logger.debug("Test failed from first probe,"
" we sleep 600 second try one more time"
" and if it fails again - test will fails ")
time.sleep(600)
self.fuel_web.run_ostf(cluster_id,
test_sets=['smoke', 'sanity', 'ha'])
def manual_cic_maintenance_mode(self):
"""Check manual maintenance mode for controller
Scenario:
1. Revert snapshot
2. Switch in maintenance mode
3. Wait until controller is rebooting
4. Exit maintenance mode
5. Check the controller become available
Duration 155m
"""
self.env.revert_snapshot('cic_maintenance_mode')
cluster_id = self.fuel_web.get_last_created_cluster()
# Select a non-primary controller
regular_ctrl = self.fuel_web.get_nailgun_node_by_name("slave-02")
dregular_ctrl = self.fuel_web.get_devops_node_by_nailgun_node(
regular_ctrl)
_ip = regular_ctrl['ip']
_id = regular_ctrl['id']
logger.info('Maintenance mode for node-{0}'.format(_id))
asserts.assert_true('True' in check_available_mode(_ip),
"Maintenance mode is not available")
self.ssh_manager.execute_on_remote(
ip=_ip,
cmd="umm on")
self.fuel_web.wait_node_is_offline(dregular_ctrl)
asserts.assert_true(
checkers.check_ping(self.env.get_admin_node_ip(),
_ip,
deadline=600),
"Host {0} is not reachable by ping during 600 sec"
.format(_ip))
asserts.assert_true('True' in check_auto_mode(_ip),
"Maintenance mode is not switched on")
self.ssh_manager.execute_on_remote(
ip=_ip,
cmd="umm off")
self.fuel_web.wait_node_is_online(dregular_ctrl)
# Wait until Cinder services UP on a controller
self.fuel_web.wait_cinder_is_up(
[dregular_ctrl.name])
# Wait until RabbitMQ cluster is UP
wait_pass(lambda:
self.fuel_web.run_single_ostf_test(
cluster_id, test_sets=['ha'],
test_name=ostf_test_mapping.OSTF_TEST_MAPPING.get(
'RabbitMQ availability')),
timeout=1500)
logger.info('RabbitMQ cluster is available')
wait_pass(lambda:
self.fuel_web.run_single_ostf_test(
cluster_id, test_sets=['sanity'],
test_name=ostf_test_mapping.OSTF_TEST_MAPPING.get(
'Check that required services are running')),
timeout=1500)
logger.info("Required services are running")
# TODO(astudenov): add timeout_msg
try:
self.fuel_web.run_ostf(cluster_id,
test_sets=['smoke', 'sanity', 'ha'])
except AssertionError:
logger.debug("Test failed from first probe,"
" we sleep 600 second try one more time"
" and if it fails again - test will fails ")
time.sleep(600)
self.fuel_web.run_ostf(cluster_id,
test_sets=['smoke', 'sanity', 'ha'])
def negative_auto_cic_maintenance_mode(self):
"""Check negative scenario for auto maintenance mode
Scenario:
1. Revert snapshot
2. Disable UMM
3. Change UMM.CONF
4. Unexpected reboot
5. Check the controller not switching in maintenance mode
6. Check the controller become available
Duration 85m
"""
self.env.revert_snapshot('cic_maintenance_mode')
cluster_id = self.fuel_web.get_last_created_cluster()
# Select a non-primary controller
regular_ctrl = self.fuel_web.get_nailgun_node_by_name("slave-02")
dregular_ctrl = self.fuel_web.get_devops_node_by_nailgun_node(
regular_ctrl)
_ip = regular_ctrl['ip']
_id = regular_ctrl['id']
asserts.assert_true('True' in check_available_mode(_ip),
"Maintenance mode is not available")
logger.info('Disable UMM on node-{0}'.format(_id))
change_config(_ip, umm=False, reboot_count=0)
asserts.assert_false('True' in check_available_mode(_ip),
"Maintenance mode should not be available")
command = 'reboot --force >/dev/null & '
logger.info('Unexpected reboot on node-{0}'
.format(_id))
self.ssh_manager.execute_on_remote(
ip=_ip,
cmd=command)
wait(lambda:
not checkers.check_ping(self.env.get_admin_node_ip(),
_ip),
timeout=60 * 10,
timeout_msg='Node {} still responds to ping'.format(
dregular_ctrl.name))
# Node don't have enough time for set offline status
# after reboot --force
# Just waiting
asserts.assert_true(
checkers.check_ping(self.env.get_admin_node_ip(),
_ip,
deadline=600),
"Host {0} is not reachable by ping during 600 sec"
.format(_ip))
self.fuel_web.wait_node_is_online(dregular_ctrl)
logger.info('Check that node-{0} not in maintenance mode after'
' unexpected reboot'.format(_id))
wait(lambda: tcp_ping(_ip, 22),
timeout=60 * 10,
timeout_msg='Node {} still is not available by SSH'.format(
dregular_ctrl.name))
asserts.assert_false('True' in check_auto_mode(_ip),
"Maintenance mode should not switched")
# Wait until MySQL Galera is UP on some controller
self.fuel_web.wait_mysql_galera_is_up(
[dregular_ctrl.name])
# Wait until Cinder services UP on a controller
self.fuel_web.wait_cinder_is_up(
[dregular_ctrl.name])
# Wait until RabbitMQ cluster is UP
wait_pass(lambda:
self.fuel_web.run_single_ostf_test(
cluster_id, test_sets=['ha'],
test_name=ostf_test_mapping.OSTF_TEST_MAPPING.get(
'RabbitMQ availability')),
timeout=1500)
logger.info('RabbitMQ cluster is available')
# TODO(astudenov): add timeout_msg
wait_pass(lambda:
self.fuel_web.run_single_ostf_test(
cluster_id, test_sets=['sanity'],
test_name=ostf_test_mapping.OSTF_TEST_MAPPING.get(
'Check that required services are running')),
timeout=1500)
logger.info("Required services are running")
try:
self.fuel_web.run_ostf(cluster_id,
test_sets=['smoke', 'sanity', 'ha'])
except AssertionError:
logger.debug("Test failed from first probe,"
" we sleep 600 second try one more time"
" and if it fails again - test will fails ")
time.sleep(600)
self.fuel_web.run_ostf(cluster_id,
test_sets=['smoke', 'sanity', 'ha'])
