def restart(self, message):
"""Reboots the machine.
If the reboot is successful, never returns: the process should just be
killed by OS.
If reboot fails, logs the error to the server and moves the bot to
quarantined mode.
"""
self.post_event('bot_rebooting', message)
self.cancel_all_timers()
if self._shutdown_hook:
try:
self._shutdown_hook(self)
except Exception as e:
logging.exception('shutdown hook failed: %s', e)
# os_utilities.restart should never return, unless restart is not happening.
# If restart is taking longer than N minutes, it probably not going to
# finish at all. Report this to the server.
try:
os_utilities.restart(message, timeout=15*60)
except LookupError:
# This is a special case where OSX is deeply hosed. In that case the disk
# is likely in read-only mode and there isn't much that can be done. This
# exception is deep inside pickle.py. So notify the server then hang in
# there.
self.post_error('This host partition is bad; please fix the host')
while True:
time.sleep(1)
self.post_error('Bot is stuck restarting for: %s' % message)
def restart(self, message):
"""Reboots the machine.
If the reboot is successful, never returns: the process should just be
killed by OS.
If reboot fails, logs the error to the server and moves the bot to
quarantined mode.
"""
self.post_event('bot_rebooting', message)
if self._shutdown_hook:
try:
self._shutdown_hook(self)
except Exception as e:
logging.exception('shutdown hook failed: %s', e)
# os_utilities.restart should never return, unless restart is not happening.
# If restart is taking longer than N minutes, it probably not going to
# finish at all. Report this to the server.
try:
os_utilities.restart(message, timeout=15 * 60)
except LookupError:
# This is a special case where OSX is deeply hosed. In that case the disk
# is likely in read-only mode and there isn't much that can be done. This
# exception is deep inside pickle.py. So notify the server then hang in
# there.
self.post_error('This host partition is bad; please fix the host')
while True:
time.sleep(1)
self.post_error('Bot is stuck restarting for: %s' % message)
def CMDrestart(_args):
"""Utility subcommand that hides the difference between each OS to reboot
the host."""
logging_utils.prepare_logging(None)
import os_utilities
# This function doesn't return.
os_utilities.restart()
# Should never reach here.
return 1
def CMDrestart(_args): """Utility subcommand that hides the difference between each OS to reboot the host.""" logging_utils.prepare_logging(None) import os_utilities # This function doesn't return. os_utilities.restart() # Should never reach here. return 1
def test_restart(self):
class Foo(Exception):
pass
def raise_exception(x):
raise x
self.mock(subprocess, 'check_call', lambda _: None)
self.mock(time, 'sleep', lambda _: raise_exception(Foo()))
self.mock(logging, 'error', lambda *_: None)
with self.assertRaises(Foo):
os_utilities.restart()
def test_restart(self):
class Foo(Exception):
pass
def raise_exception(x):
raise x
self.mock(subprocess, 'check_call', lambda _: None)
self.mock(time, 'sleep', lambda _: raise_exception(Foo()))
self.mock(logging, 'error', lambda *_: None)
with self.assertRaises(Foo):
os_utilities.restart()
def restart(self, message):
"""Reboots the machine.
If the reboot is successful, never returns: the process should just be
killed by OS.
If reboot fails, logs the error to the server and moves the bot to
quarantined mode.
"""
self.post_event('bot_rebooting', message)
if self._shutdown_hook:
try:
self._shutdown_hook(self)
except Exception as e:
logging.exception('shutdown hook failed: %s', e)
# os_utilities.restart should never return, unless restart is not happening.
# If restart is taking longer than N minutes, it probably not going to
# finish at all. Report this to the server.
os_utilities.restart(message, timeout=15*60)
self.post_error('Bot is stuck restarting for: %s' % message)
def restart(self, message):
"""Reboots the machine.
If the reboot is successful, never returns: the process should just be
killed by OS.
If reboot fails, logs the error to the server and moves the bot to
quarantined mode.
"""
self.post_event('bot_rebooting', message)
if self._shutdown_hook:
try:
self._shutdown_hook(self)
except Exception as e:
logging.exception('shutdown hook failed: %s', e)
# os_utilities.restart should never return, unless restart is not happening.
# If restart is taking longer than N minutes, it probably not going to
# finish at all. Report this to the server.
os_utilities.restart(message, timeout=15 * 60)
self.post_error('Bot is stuck restarting for: %s' % message)
def test_restart_with_timeout(self):
self.mock(subprocess, 'check_call', lambda _: None)
self.mock(logging, 'error', lambda *_: None)
now = [0]
def mock_sleep(dt):
now[0] += dt
self.mock(time, 'sleep', mock_sleep)
self.mock(time, 'time', lambda: now[0])
self.assertFalse(os_utilities.restart(timeout=60))
self.assertEqual(time.time(), 60)
def test_restart_with_timeout(self):
self.mock(subprocess, 'check_call', lambda _: None)
self.mock(logging, 'error', lambda *_: None)
now = [0]
def mock_sleep(dt):
now[0] += dt
self.mock(time, 'sleep', mock_sleep)
self.mock(time, 'time', lambda: now[0])
self.assertFalse(os_utilities.restart(timeout=60))
self.assertEqual(time.time(), 60)
