def RecoverDevice(device, blacklist, should_reboot=lambda device: True):
if device_status.IsBlacklisted(device.adb.GetDeviceSerial(),
blacklist):
logger.debug('%s is blacklisted, skipping recovery.', str(device))
return
if device.adb.GetState() == 'unauthorized' and TryAuth(device):
logger.info('Successfully authed device %s!', str(device))
return
if should_reboot(device):
try:
device.WaitUntilFullyBooted(retries=0)
except (device_errors.CommandTimeoutError,
device_errors.CommandFailedError,
device_errors.DeviceUnreachableError):
logger.exception('Failure while waiting for %s. '
'Attempting to recover.', str(device))
try:
try:
device.Reboot(block=False, timeout=5, retries=0)
except device_errors.CommandTimeoutError:
logger.warning('Timed out while attempting to reboot %s normally.'
'Attempting alternative reboot.', str(device))
# The device drops offline before we can grab the exit code, so
# we don't check for status.
try:
device.adb.Root()
finally:
# We are already in a failure mode, attempt to reboot regardless of
# what device.adb.Root() returns. If the sysrq reboot fails an
# exception willbe thrown at that level.
device.adb.Shell('echo b > /proc/sysrq-trigger', expect_status=None,
timeout=5, retries=0)
except (device_errors.CommandFailedError,
device_errors.DeviceUnreachableError):
logger.exception('Failed to reboot %s.', str(device))
if blacklist:
blacklist.Extend([device.adb.GetDeviceSerial()],
reason='reboot_failure')
except device_errors.CommandTimeoutError:
logger.exception('Timed out while rebooting %s.', str(device))
if blacklist:
blacklist.Extend([device.adb.GetDeviceSerial()],
reason='reboot_timeout')
try:
device.WaitUntilFullyBooted(
retries=0, timeout=device.REBOOT_DEFAULT_TIMEOUT)
except (device_errors.CommandFailedError,
device_errors.DeviceUnreachableError):
logger.exception('Failure while waiting for %s.', str(device))
if blacklist:
blacklist.Extend([device.adb.GetDeviceSerial()],
reason='reboot_failure')
except device_errors.CommandTimeoutError:
logger.exception('Timed out while waiting for %s.', str(device))
if blacklist:
blacklist.Extend([device.adb.GetDeviceSerial()],
reason='reboot_timeout')
def device_shard_helper(shard_id):
if device_status.IsBlacklisted(
str(self._devices[shard_id]), self._env.blacklist):
logging.warning('Device %s is not active. Will not create shard %s.',
str(self._devices[shard_id]), shard_id)
return None
s = DeviceTestShard(self._env, self._test_instance,
self._devices[shard_id], shard_id,
self._test_buckets[shard_id],
retries=self._env.max_tries, timeout=self._timeout)
return s.RunTestsOnShard()
def RecoverDevice(device, blacklist, should_reboot=lambda device: True):
if device_status.IsBlacklisted(device.adb.GetDeviceSerial(), blacklist):
logging.debug('%s is blacklisted, skipping recovery.', str(device))
return
if should_reboot(device):
try:
device.WaitUntilFullyBooted(retries=0)
except (device_errors.CommandTimeoutError,
device_errors.CommandFailedError):
logging.exception(
'Failure while waiting for %s. '
'Attempting to recover.', str(device))
try:
try:
device.Reboot(block=False, timeout=5, retries=0)
except device_errors.CommandTimeoutError:
logging.warning(
'Timed out while attempting to reboot %s normally.'
'Attempting alternative reboot.', str(device))
# The device drops offline before we can grab the exit code, so
# we don't check for status.
device.adb.Root()
device.adb.Shell('echo b > /proc/sysrq-trigger',
expect_status=None,
timeout=5,
retries=0)
except device_errors.CommandFailedError:
logging.exception('Failed to reboot %s.', str(device))
if blacklist:
blacklist.Extend([device.adb.GetDeviceSerial()],
reason='reboot_failure')
except device_errors.CommandTimeoutError:
logging.exception('Timed out while rebooting %s.', str(device))
if blacklist:
blacklist.Extend([device.adb.GetDeviceSerial()],
reason='reboot_timeout')
try:
device.WaitUntilFullyBooted(retries=0,
timeout=device.REBOOT_DEFAULT_TIMEOUT)
except device_errors.CommandFailedError:
logging.exception('Failure while waiting for %s.', str(device))
if blacklist:
blacklist.Extend([device.adb.GetDeviceSerial()],
reason='reboot_failure')
except device_errors.CommandTimeoutError:
logging.exception('Timed out while waiting for %s.', str(device))
if blacklist:
blacklist.Extend([device.adb.GetDeviceSerial()],
reason='reboot_timeout')
try:
logger.info('kill %d %d (%s [%s])', sig, p.pid, p.name,
' '.join(p.cmdline))
p.send_signal(sig)
except (psutil.NoSuchProcess, psutil.AccessDenied):
pass
for p in get_all_adb():
try:
logger.error('Unable to kill %d (%s [%s])', p.pid, p.name,
' '.join(p.cmdline))
except (psutil.NoSuchProcess, psutil.AccessDenied):
pass
def RecoverDevice(presentation.device, blacklist, should_reboot=lambda presentation.device: True):
if device_status.IsBlacklisted(presentation.device.adb.GetDeviceSerial(),
blacklist):
logger.debug('%s is blacklisted, skipping recovery.', str(presentation.device))
return
if should_reboot(presentation.device):
try:
presentation.device.WaitUntilFullyBooted(retries=0)
except (device_errors.CommandTimeoutError,
device_errors.CommandFailedError):
logger.exception('Failure while waiting for %s. '
'Attempting to recover.', str(presentation.device))
try:
try:
presentation.device.Reboot(block=False, timeout=5, retries=0)
except device_errors.CommandTimeoutError:
logger.warning('Timed out while attempting to reboot %s normally.'
