def test_node_wait(self):
local = LocalNode()
self.assertTrue(
local.wait_for_port(host='127.0.0.1', attempts=2, timeout=5))
#self.assertFalse(
with self.assertRaises(Exception):
local.wait_for_port(host='192.0.2.1', attempts=2, timeout=5)
def wait_node_is_ready(node,
timeout=900,
conman_line_max_age=None,
max_cold_restart=3,
port_lookup=22,
port_lookup_timeout=None,
port_lookup_attempts=None):
""" Return true if node(and ssh) start in time
Raise exceptions in other cases
Parameters:
timeout: overall timeout for boot and start ssh
(ssh starts after end of kiwi provisioning)
conman_line_max_age: how long last conman line
coud be not changed (mean node stuck)
max_cold_restart: maximum nuber of cold restarts
"""
if port_lookup_timeout is None:
port_lookup_timeout = default_port_lookup_timeout
if port_lookup_attempts is None:
port_lookup_attempts = default_port_lookup_attempts
if conman_line_max_age is None:
conman_line_max_age = default_conman_line_max_age
starttime = time.time()
conman_line = 'start_line'
conman_line_time = time.time()
conmanfile = conman_log_prefix + node['node'].split('.')[0]
cold_restart_count = 0
while starttime + timeout > time.time():
# check last line in conman log
new_conman_line = last_nonempty_line(conmanfile)
if conman_line != new_conman_line:
# logging.debug("New log detected:"+new_conman_line)
conman_line = new_conman_line
conman_line_time = time.time()
if (time.time() - conman_line_time) > conman_line_max_age:
logging.info("Node boot failure detected, make cold restart")
exec_bmc_command(node, 'power off')
time.sleep(default_cold_restart_timeout)
exec_bmc_command(node, 'power on')
if cold_restart_count >= max_cold_restart:
logging.error('Achieved max cold restart couter ' +
'%s for node %s,throws exception' %
(max_cold_restart, node['node']))
raise CannotBootException(
"max cold restart couter (%s) for %s" %
(max_cold_restart, node['node']))
cold_restart_count += 1
conman_line_time = time.time()
next
# check port status
try:
logging.debug("timeout=" + str(port_lookup_timeout))
local = LocalNode()
local.wait_for_port(host=get_provision_ip(node),
port=port_lookup,
timeout=port_lookup_timeout,
attempts=port_lookup_attempts)
logging.debug("Connected to node %s " % node['node'])
return True
except StopThread as st:
raise StopThread
except Exception as es:
logging.debug("Node {} have not started in timeout {}".format(
get_provision_ip(node), port_lookup_timeout))
raise TimeoutException("{} have not started in timeout {}".format(
get_provision_ip(node), timeout))
