def node_wait_for_power_state(task, new_state, timeout=None):
"""Wait for node to be in new power state.
:param task: a TaskManager instance.
:param new_state: the desired new power state, one of the power states
in :mod:`ironic.common.states`.
:param timeout: number of seconds to wait before giving up. If not
specified, uses the conductor.power_state_change_timeout config value.
:raises: PowerStateFailure if timed out
"""
retry_timeout = (timeout or CONF.conductor.power_state_change_timeout)
def _wait():
status = task.driver.power.get_power_state(task)
if status == new_state:
raise loopingcall.LoopingCallDone(retvalue=status)
# NOTE(sambetts): Return False to trigger BackOffLoopingCall to start
# backing off.
return False
try:
timer = loopingcall.BackOffLoopingCall(_wait)
return timer.start(initial_delay=1, timeout=retry_timeout).wait()
except loopingcall.LoopingCallTimeOut:
LOG.error(
'Timed out after %(retry_timeout)s secs waiting for power '
'%(state)s on node %(node_id)s.', {
'retry_timeout': retry_timeout,
'state': new_state,
'node_id': task.node.uuid
})
raise exception.PowerStateFailure(pstate=new_state)
def _enable_console(mode):
"""Request ironicclient to enable/disable node console."""
try:
self.ironicclient.call('node.set_console_mode', node_uuid,
mode)
except (
ironic_exc.InternalServerError, # Validations
ironic_exc.BadRequest) as e: # Maintenance
LOG.error(
'Failed to set console mode to "%(mode)s" '
'for node %(node)s: %(reason)s', {
'mode': mode,
'node': node_uuid,
'reason': e
})
raise exception.ConsoleNotAvailable()
# Waiting for the console state to change (disabled/enabled)
try:
timer = loopingcall.BackOffLoopingCall(_wait_state, state=mode)
return timer.start(starting_interval=1, timeout=10,
jitter=0.5).wait()
except loopingcall.LoopingCallTimeOut:
LOG.error(
'Timeout while waiting for console mode to be '
'set to "%(mode)s" on node %(node)s', {
'mode': mode,
'node': node_uuid
})
raise exception.ConsoleNotAvailable()
def test_max_interval(self, sleep_mock, random_mock):
def false():
return False
random_mock.return_value = .8
self.assertRaises(
loopingcall.LoopingCallTimeOut,
loopingcall.BackOffLoopingCall(false).start(max_interval=60).wait)
expected_times = [
mock.call(1.6),
mock.call(2.4000000000000004),
mock.call(3.6),
mock.call(5.4),
mock.call(8.1),
mock.call(12.15),
mock.call(18.225),
mock.call(27.337500000000002),
mock.call(41.00625),
mock.call(60),
mock.call(60),
mock.call(60)
]
self.assertEqual(expected_times, sleep_mock.call_args_list)
def test_no_sleep(self, sleep_mock, random_mock):
# Any call that executes properly the first time shouldn't sleep
random_mock.return_value = 1
func = mock.Mock()
# func.side_effect
func.side_effect = loopingcall.LoopingCallDone(retvalue='return value')
retvalue = loopingcall.BackOffLoopingCall(func).start().wait()
self.assertFalse(sleep_mock.called)
self.assertTrue(retvalue, 'return value')
def test_no_backoff(self, sleep_mock, random_mock):
random_mock.return_value = 1
func = mock.Mock()
# func.side_effect
func.side_effect = [True, True, True, loopingcall.LoopingCallDone(
retvalue='return value')]
retvalue = loopingcall.BackOffLoopingCall(func).start().wait()
expected_times = [mock.call(1), mock.call(1), mock.call(1)]
self.assertEqual(expected_times, sleep_mock.call_args_list)
self.assertTrue(retvalue, 'return value')
def lookup_node(self, hardware_info, timeout, starting_interval,
node_uuid=None):
timer = loopingcall.BackOffLoopingCall(
self._do_lookup,
hardware_info=hardware_info,
node_uuid=node_uuid)
try:
node_content = timer.start(starting_interval=starting_interval,
timeout=timeout).wait()
except loopingcall.LoopingCallTimeOut:
raise errors.LookupNodeError('Could not look up node info. Check '
'logs for details.')
return node_content
def build_polling_task(retriever,
condition=lambda value: value,
sleep_time=1,
time_out=0):
def poll_and_check():
obj = retriever()
if condition(obj):
raise loopingcall.LoopingCallDone(retvalue=obj)
return loopingcall.BackOffLoopingCall(f=poll_and_check).start(
initial_delay=False,
starting_interval=sleep_time,
max_interval=30,
timeout=time_out)
def build_polling_task(retriever, condition=lambda value: value,
sleep_time=1, time_out=None):
start_time = time.time()
def poll_and_check():
obj = retriever()
if condition(obj):
raise loopingcall.LoopingCallDone(retvalue=obj)
if time_out is not None and time.time() - start_time > time_out:
raise exception.PollTimeOut
return loopingcall.BackOffLoopingCall(
f=poll_and_check).start(initial_delay=False,
starting_interval=sleep_time,
max_interval=30, timeout=time_out)
def build_polling_task(retriever, condition=lambda value: value,
sleep_time=1, time_out=0, initial_delay=0):
"""Run a function in a loop with backoff on error.
The condition function runs based on the retriever function result.
"""
def poll_and_check():
obj = retriever()
if condition(obj):
raise loopingcall.LoopingCallDone(retvalue=obj)
call = loopingcall.BackOffLoopingCall(f=poll_and_check)
return call.start(initial_delay=initial_delay,
starting_interval=sleep_time,
max_interval=30, timeout=time_out)
def test_exponential_backoff(self, sleep_mock, random_mock):
def false():
return False
random_mock.return_value = .8
self.assertRaises(loopingcall.LoopingCallTimeOut,
loopingcall.BackOffLoopingCall(false).start()
.wait)
expected_times = [mock.call(1.6000000000000001),
mock.call(2.5600000000000005),
mock.call(4.096000000000001),
mock.call(6.5536000000000021),
mock.call(10.485760000000004),
mock.call(16.777216000000006),
mock.call(26.843545600000013),
mock.call(42.949672960000022),
mock.call(68.719476736000033),
mock.call(109.95116277760006)]
self.assertEqual(expected_times, sleep_mock.call_args_list)
def node_wait_for_power_state(task, new_state, timeout=None):
retry_timeout = (timeout or CONF.conductor.power_state_change_timeout)
def _wait():
status = task.driver.power.get_power_state(task)
if status == new_state:
raise loopingcall.LoopingCallDone(retvalue=status)
# NOTE(sambetts): Return False to trigger BackOffLoopingCall to start
# backing off.
return False
try:
timer = loopingcall.BackOffLoopingCall(_wait)
return timer.start(initial_delay=1, timeout=retry_timeout).wait()
except loopingcall.LoopingCallTimeOut:
LOG.error(
'Timed out after %(retry_timeout) secs waiting for power '
'%(state)s on node %(node_id)s.', {
'state': new_state,
'node_id': task.node.uuid
})
raise exception.PowerStateFailure(pstate=new_state)
def test_exponential_backoff(self, sleep_mock, random_mock):
def false():
return False
random_mock.return_value = .8
self.assertRaises(loopingcall.LoopingCallTimeOut,
loopingcall.BackOffLoopingCall(false).start()
.wait)
expected_times = [mock.call(1.6),
mock.call(2.4000000000000004),
mock.call(3.6),
mock.call(5.4),
mock.call(8.1),
mock.call(12.15),
mock.call(18.225),
mock.call(27.337500000000002),
mock.call(41.00625),
mock.call(61.509375000000006),
mock.call(92.26406250000001)]
self.assertEqual(expected_times, sleep_mock.call_args_list)
def test_max_interval(self, sleep_mock, random_mock):
def false():
return False
random_mock.return_value = .8
self.assertRaises(loopingcall.LoopingCallTimeOut,
loopingcall.BackOffLoopingCall(false).start(
max_interval=60)
.wait)
expected_times = [mock.call(1.6000000000000001),
mock.call(2.5600000000000005),
mock.call(4.096000000000001),
mock.call(6.5536000000000021),
mock.call(10.485760000000004),
mock.call(16.777216000000006),
mock.call(26.843545600000013),
mock.call(42.949672960000022),
mock.call(60),
mock.call(60),
mock.call(60)]
self.assertEqual(expected_times, sleep_mock.call_args_list)