def node_wait_for_power_state(task, new_state, timeout=None): """Wait for node to be in new power state. :param task: a TaskManager instance. :param new_state: the desired new power state, one of the power states in :mod:`ironic.common.states`. :param timeout: number of seconds to wait before giving up. If not specified, uses the conductor.power_state_change_timeout config value. :raises: PowerStateFailure if timed out """ retry_timeout = (timeout or CONF.conductor.power_state_change_timeout) def _wait(): status = task.driver.power.get_power_state(task) if status == new_state: raise loopingcall.LoopingCallDone(retvalue=status) # NOTE(sambetts): Return False to trigger BackOffLoopingCall to start # backing off. return False try: timer = loopingcall.BackOffLoopingCall(_wait) return timer.start(initial_delay=1, timeout=retry_timeout).wait() except loopingcall.LoopingCallTimeOut: LOG.error( 'Timed out after %(retry_timeout)s secs waiting for power ' '%(state)s on node %(node_id)s.', { 'retry_timeout': retry_timeout, 'state': new_state, 'node_id': task.node.uuid }) raise exception.PowerStateFailure(pstate=new_state)
def _enable_console(mode): """Request ironicclient to enable/disable node console.""" try: self.ironicclient.call('node.set_console_mode', node_uuid, mode) except ( ironic_exc.InternalServerError, # Validations ironic_exc.BadRequest) as e: # Maintenance LOG.error( 'Failed to set console mode to "%(mode)s" ' 'for node %(node)s: %(reason)s', { 'mode': mode, 'node': node_uuid, 'reason': e }) raise exception.ConsoleNotAvailable() # Waiting for the console state to change (disabled/enabled) try: timer = loopingcall.BackOffLoopingCall(_wait_state, state=mode) return timer.start(starting_interval=1, timeout=10, jitter=0.5).wait() except loopingcall.LoopingCallTimeOut: LOG.error( 'Timeout while waiting for console mode to be ' 'set to "%(mode)s" on node %(node)s', { 'mode': mode, 'node': node_uuid }) raise exception.ConsoleNotAvailable()
def test_max_interval(self, sleep_mock, random_mock): def false(): return False random_mock.return_value = .8 self.assertRaises( loopingcall.LoopingCallTimeOut, loopingcall.BackOffLoopingCall(false).start(max_interval=60).wait) expected_times = [ mock.call(1.6), mock.call(2.4000000000000004), mock.call(3.6), mock.call(5.4), mock.call(8.1), mock.call(12.15), mock.call(18.225), mock.call(27.337500000000002), mock.call(41.00625), mock.call(60), mock.call(60), mock.call(60) ] self.assertEqual(expected_times, sleep_mock.call_args_list)
def test_no_sleep(self, sleep_mock, random_mock): # Any call that executes properly the first time shouldn't sleep random_mock.return_value = 1 func = mock.Mock() # func.side_effect func.side_effect = loopingcall.LoopingCallDone(retvalue='return value') retvalue = loopingcall.BackOffLoopingCall(func).start().wait() self.assertFalse(sleep_mock.called) self.assertTrue(retvalue, 'return value')
def test_no_backoff(self, sleep_mock, random_mock): random_mock.return_value = 1 func = mock.Mock() # func.side_effect func.side_effect = [True, True, True, loopingcall.LoopingCallDone( retvalue='return value')] retvalue = loopingcall.BackOffLoopingCall(func).start().wait() expected_times = [mock.call(1), mock.call(1), mock.call(1)] self.assertEqual(expected_times, sleep_mock.call_args_list) self.assertTrue(retvalue, 'return value')
def lookup_node(self, hardware_info, timeout, starting_interval, node_uuid=None): timer = loopingcall.BackOffLoopingCall( self._do_lookup, hardware_info=hardware_info, node_uuid=node_uuid) try: node_content = timer.start(starting_interval=starting_interval, timeout=timeout).wait() except loopingcall.LoopingCallTimeOut: raise errors.LookupNodeError('Could not look up node info. Check ' 'logs for details.') return node_content
def build_polling_task(retriever, condition=lambda value: value, sleep_time=1, time_out=0): def poll_and_check(): obj = retriever() if condition(obj): raise loopingcall.LoopingCallDone(retvalue=obj) return loopingcall.BackOffLoopingCall(f=poll_and_check).start( initial_delay=False, starting_interval=sleep_time, max_interval=30, timeout=time_out)
def build_polling_task(retriever, condition=lambda value: value, sleep_time=1, time_out=None): start_time = time.time() def poll_and_check(): obj = retriever() if condition(obj): raise loopingcall.LoopingCallDone(retvalue=obj) if time_out is not None and time.time() - start_time > time_out: raise exception.PollTimeOut return loopingcall.BackOffLoopingCall( f=poll_and_check).start(initial_delay=False, starting_interval=sleep_time, max_interval=30, timeout=time_out)
def build_polling_task(retriever, condition=lambda value: value, sleep_time=1, time_out=0, initial_delay=0): """Run a function in a loop with backoff on error. The condition function runs based on the retriever function result. """ def poll_and_check(): obj = retriever() if condition(obj): raise loopingcall.LoopingCallDone(retvalue=obj) call = loopingcall.BackOffLoopingCall(f=poll_and_check) return call.start(initial_delay=initial_delay, starting_interval=sleep_time, max_interval=30, timeout=time_out)
def test_exponential_backoff(self, sleep_mock, random_mock): def false(): return False random_mock.return_value = .8 self.assertRaises(loopingcall.LoopingCallTimeOut, loopingcall.BackOffLoopingCall(false).start() .wait) expected_times = [mock.call(1.6000000000000001), mock.call(2.5600000000000005), mock.call(4.096000000000001), mock.call(6.5536000000000021), mock.call(10.485760000000004), mock.call(16.777216000000006), mock.call(26.843545600000013), mock.call(42.949672960000022), mock.call(68.719476736000033), mock.call(109.95116277760006)] self.assertEqual(expected_times, sleep_mock.call_args_list)
def node_wait_for_power_state(task, new_state, timeout=None): retry_timeout = (timeout or CONF.conductor.power_state_change_timeout) def _wait(): status = task.driver.power.get_power_state(task) if status == new_state: raise loopingcall.LoopingCallDone(retvalue=status) # NOTE(sambetts): Return False to trigger BackOffLoopingCall to start # backing off. return False try: timer = loopingcall.BackOffLoopingCall(_wait) return timer.start(initial_delay=1, timeout=retry_timeout).wait() except loopingcall.LoopingCallTimeOut: LOG.error( 'Timed out after %(retry_timeout) secs waiting for power ' '%(state)s on node %(node_id)s.', { 'state': new_state, 'node_id': task.node.uuid }) raise exception.PowerStateFailure(pstate=new_state)
def test_exponential_backoff(self, sleep_mock, random_mock): def false(): return False random_mock.return_value = .8 self.assertRaises(loopingcall.LoopingCallTimeOut, loopingcall.BackOffLoopingCall(false).start() .wait) expected_times = [mock.call(1.6), mock.call(2.4000000000000004), mock.call(3.6), mock.call(5.4), mock.call(8.1), mock.call(12.15), mock.call(18.225), mock.call(27.337500000000002), mock.call(41.00625), mock.call(61.509375000000006), mock.call(92.26406250000001)] self.assertEqual(expected_times, sleep_mock.call_args_list)
def test_max_interval(self, sleep_mock, random_mock): def false(): return False random_mock.return_value = .8 self.assertRaises(loopingcall.LoopingCallTimeOut, loopingcall.BackOffLoopingCall(false).start( max_interval=60) .wait) expected_times = [mock.call(1.6000000000000001), mock.call(2.5600000000000005), mock.call(4.096000000000001), mock.call(6.5536000000000021), mock.call(10.485760000000004), mock.call(16.777216000000006), mock.call(26.843545600000013), mock.call(42.949672960000022), mock.call(60), mock.call(60), mock.call(60)] self.assertEqual(expected_times, sleep_mock.call_args_list)