def RecoverDevices(devices, blacklist, enable_usb_reset=False):
"""Attempts to recover any inoperable devices in the provided list.
Args:
devices: The list of devices to attempt to recover.
blacklist: The current device blacklist, which will be used then
reset.
"""
statuses = device_status.DeviceStatus(devices, blacklist)
should_restart_usb = set(
status['serial'] for status in statuses
if (not status['usb_status']
or status['adb_status'] in ('offline', 'missing')))
should_restart_adb = should_restart_usb.union(set(
status['serial'] for status in statuses
if status['adb_status'] == 'unauthorized'))
should_reboot_device = should_restart_usb.union(set(
status['serial'] for status in statuses
if status['blacklisted']))
logger.debug('Should restart USB for:')
for d in should_restart_usb:
logger.debug(' %s', d)
logger.debug('Should restart ADB for:')
for d in should_restart_adb:
logger.debug(' %s', d)
logger.debug('Should reboot:')
for d in should_reboot_device:
logger.debug(' %s', d)
if blacklist:
blacklist.Reset()
if should_restart_adb:
KillAllAdb()
adb_wrapper.AdbWrapper.StartServer()
for serial in should_restart_usb:
try:
# TODO(crbug.com/642194): Resetting may be causing more harm
# (specifically, kernel panics) than it does good.
if enable_usb_reset:
reset_usb.reset_android_usb(serial)
else:
logger.warning('USB reset disabled for %s (crbug.com/642914)',
serial)
except IOError:
logger.exception('Unable to reset USB for %s.', serial)
if blacklist:
blacklist.Extend([serial], reason='USB failure')
except device_errors.DeviceUnreachableError:
logger.exception('Unable to reset USB for %s.', serial)
if blacklist:
blacklist.Extend([serial], reason='offline')
device_utils.DeviceUtils.parallel(devices).pMap(
RecoverDevice, blacklist,
should_reboot=lambda device: device.serial in should_reboot_device)
def RecoverDevices(devices, blacklist, enable_usb_reset=False):
"""Attempts to recover any inoperable devices in the provided list.
Args:
devices: The list of devices to attempt to recover.
blacklist: The current device blacklist, which will be used then
reset.
"""
statuses = device_status.DeviceStatus(devices, blacklist)
should_restart_usb = set(
status["serial"]
for status in statuses
if (not status["usb_status"] or status["adb_status"] in ("offline", "missing"))
)
should_restart_adb = should_restart_usb.union(
set(status["serial"] for status in statuses if status["adb_status"] == "unauthorized")
)
should_reboot_device = should_restart_adb.union(
set(status["serial"] for status in statuses if status["blacklisted"])
)
logging.debug("Should restart USB for:")
for d in should_restart_usb:
logging.debug(" %s", d)
logging.debug("Should restart ADB for:")
for d in should_restart_adb:
logging.debug(" %s", d)
logging.debug("Should reboot:")
for d in should_reboot_device:
logging.debug(" %s", d)
if blacklist:
blacklist.Reset()
if should_restart_adb:
KillAllAdb()
for serial in should_restart_usb:
try:
# TODO(crbug.com/642194): Resetting may be causing more harm
# (specifically, kernel panics) than it does good.
if enable_usb_reset:
reset_usb.reset_android_usb(serial)
else:
logging.warning("USB reset disabled for %s (crbug.com/642914)", serial)
except IOError:
logging.exception("Unable to reset USB for %s.", serial)
if blacklist:
blacklist.Extend([serial], reason="USB failure")
except device_errors.DeviceUnreachableError:
logging.exception("Unable to reset USB for %s.", serial)
if blacklist:
blacklist.Extend([serial], reason="offline")
device_utils.DeviceUtils.parallel(devices).pMap(
RecoverDevice, blacklist, should_reboot=lambda device: device in should_reboot_device
)
def RecoverDevices(devices, blacklist):
"""Attempts to recover any inoperable devices in the provided list.
Args:
devices: The list of devices to attempt to recover.
blacklist: The current device blacklist, which will be used then
reset.
"""
statuses = device_status.DeviceStatus(devices, blacklist)
should_restart_usb = set(
status["serial"]
for status in statuses
if (not status["usb_status"] or status["adb_status"] in ("offline", "missing"))
)
should_restart_adb = should_restart_usb.union(
set(status["serial"] for status in statuses if status["adb_status"] == "unauthorized")
)
should_reboot_device = should_restart_adb.union(
set(status["serial"] for status in statuses if status["blacklisted"])
)
logging.debug("Should restart USB for:")
for d in should_restart_usb:
logging.debug(" %s", d)
logging.debug("Should restart ADB for:")
for d in should_restart_adb:
logging.debug(" %s", d)
logging.debug("Should reboot:")
for d in should_reboot_device:
logging.debug(" %s", d)
if blacklist:
blacklist.Reset()
if should_restart_adb:
KillAllAdb()
for serial in should_restart_usb:
try:
reset_usb.reset_android_usb(serial)
except IOError:
logging.exception("Unable to reset USB for %s.", serial)
if blacklist:
blacklist.Extend([serial], reason="USB failure")
except device_errors.DeviceUnreachableError:
logging.exception("Unable to reset USB for %s.", serial)
if blacklist:
blacklist.Extend([serial], reason="offline")
device_utils.DeviceUtils.parallel(devices).pMap(
RecoverDevice, blacklist, should_reboot=lambda device: device in should_reboot_device
)
def RecoverDevices(devices, blacklist):
"""Attempts to recover any inoperable devices in the provided list.
Args:
devices: The list of devices to attempt to recover.
blacklist: The current device blacklist, which will be used then
reset.
"""
statuses = device_status.DeviceStatus(devices, blacklist)
should_restart_usb = set(
status['serial'] for status in statuses
if (not status['usb_status']
or status['adb_status'] in ('offline', 'missing')))
should_restart_adb = should_restart_usb.union(set(
status['serial'] for status in statuses
if status['adb_status'] == 'unauthorized'))
should_reboot_device = should_restart_adb.union(set(
status['serial'] for status in statuses
if status['blacklisted']))
logging.debug('Should restart USB for:')
for d in should_restart_usb:
logging.debug(' %s', d)
logging.debug('Should restart ADB for:')
for d in should_restart_adb:
logging.debug(' %s', d)
logging.debug('Should reboot:')
for d in should_reboot_device:
logging.debug(' %s', d)
if blacklist:
blacklist.Reset()
if should_restart_adb:
KillAllAdb()
for serial in should_restart_usb:
try:
reset_usb.reset_android_usb(serial)
except IOError:
logging.exception('Unable to reset USB for %s.', serial)
if blacklist:
blacklist.Extend([serial], reason='USB failure')
except device_errors.DeviceUnreachableError:
logging.exception('Unable to reset USB for %s.', serial)
if blacklist:
blacklist.Extend([serial], reason='offline')
device_utils.DeviceUtils.parallel(devices).pMap(
RecoverDevice, blacklist,
should_reboot=lambda device: device in should_reboot_device)
def RecoverDevices(devices, blacklist):
"""Attempts to recover any inoperable devices in the provided list.
Args:
devices: The list of devices to attempt to recover.
blacklist: The current device blacklist, which will be used then
reset.
"""
statuses = device_status.DeviceStatus(devices, blacklist)
should_restart_usb = set(
status['serial'] for status in statuses
if (not status['usb_status']
or status['adb_status'] in ('offline', 'missing')))
should_restart_adb = should_restart_usb.union(set(
status['serial'] for status in statuses
if status['adb_status'] == 'unauthorized'))
should_reboot_device = should_restart_adb.union(set(
status['serial'] for status in statuses
if status['blacklisted']))
logging.debug('Should restart USB for:')
for d in should_restart_usb:
logging.debug(' %s', d)
logging.debug('Should restart ADB for:')
for d in should_restart_adb:
logging.debug(' %s', d)
logging.debug('Should reboot:')
for d in should_reboot_device:
logging.debug(' %s', d)
if blacklist:
blacklist.Reset()
if should_restart_adb:
KillAllAdb()
for serial in should_restart_usb:
try:
reset_usb.reset_android_usb(serial)
except IOError:
logging.exception('Unable to reset USB for %s.', serial)
if blacklist:
blacklist.Extend([serial], reason='USB failure')
except device_errors.DeviceUnreachableError:
logging.exception('Unable to reset USB for %s.', serial)
if blacklist:
blacklist.Extend([serial], reason='offline')
device_utils.DeviceUtils.parallel(devices).pMap(
RecoverDevice, blacklist,
should_reboot=lambda device: device in should_reboot_device)
def RecoverDevices(devices, blacklist):
"""Attempts to recover any inoperable devices in the provided list.
Args:
devices: The list of devices to attempt to recover.
blacklist: The current device blacklist, which will be used then
reset.
Returns:
Nothing.
"""
statuses = DeviceStatus(devices, blacklist)
should_restart_usb = set(
status['serial'] for status in statuses
if (not status['usb_status'] or status['adb_status'] in ('offline',
'missing')))
should_restart_adb = should_restart_usb.union(
set(status['serial'] for status in statuses
if status['adb_status'] == 'unauthorized'))
should_reboot_device = should_restart_adb.union(
set(status['serial'] for status in statuses if status['blacklisted']))
logging.debug('Should restart USB for:')
for d in should_restart_usb:
logging.debug(' %s', d)
logging.debug('Should restart ADB for:')
for d in should_restart_adb:
logging.debug(' %s', d)
logging.debug('Should reboot:')
for d in should_reboot_device:
logging.debug(' %s', d)
if blacklist:
blacklist.Reset()
if should_restart_adb:
KillAllAdb()
for serial in should_restart_usb:
try:
reset_usb.reset_android_usb(serial)
except IOError:
logging.exception('Unable to reset USB for %s.', serial)
if blacklist:
blacklist.Extend([serial], reason='usb_failure')
except device_errors.DeviceUnreachableError:
logging.exception('Unable to reset USB for %s.', serial)
if blacklist:
blacklist.Extend([serial], reason='offline')
def blacklisting_recovery(device):
if _IsBlacklisted(device.adb.GetDeviceSerial(), blacklist):
logging.debug('%s is blacklisted, skipping recovery.', str(device))
return
if str(device) in should_reboot_device:
try:
device.WaitUntilFullyBooted(retries=0)
return
except (device_errors.CommandTimeoutError,
device_errors.CommandFailedError):
logging.exception(
'Failure while waiting for %s. '
'Attempting to recover.', str(device))
try:
try:
device.Reboot(block=False, timeout=5, retries=0)
except device_errors.CommandTimeoutError:
logging.warning(
'Timed out while attempting to reboot %s normally.'
'Attempting alternative reboot.', str(device))
# The device drops offline before we can grab the exit code, so
# we don't check for status.
device.adb.Root()
device.adb.Shell('echo b > /proc/sysrq-trigger',
expect_status=None,
timeout=5,
retries=0)
except device_errors.CommandFailedError:
logging.exception('Failed to reboot %s.', str(device))
if blacklist:
blacklist.Extend([device.adb.GetDeviceSerial()],
reason='reboot_failure')
except device_errors.CommandTimeoutError:
logging.exception('Timed out while rebooting %s.', str(device))
if blacklist:
blacklist.Extend([device.adb.GetDeviceSerial()],
reason='reboot_timeout')
try:
device.WaitUntilFullyBooted(retries=0)
except device_errors.CommandFailedError:
logging.exception('Failure while waiting for %s.', str(device))
if blacklist:
blacklist.Extend([device.adb.GetDeviceSerial()],
reason='reboot_failure')
except device_errors.CommandTimeoutError:
logging.exception('Timed out while waiting for %s.',
str(device))
if blacklist:
blacklist.Extend([device.adb.GetDeviceSerial()],
reason='reboot_timeout')
device_utils.DeviceUtils.parallel(devices).pMap(blacklisting_recovery)
def RecoverDevices(devices, blacklist):
"""Attempts to recover any inoperable devices in the provided list.
Args:
devices: The list of devices to attempt to recover.
blacklist: The current device blacklist, which will be used then
reset.
Returns:
Nothing.
"""
statuses = DeviceStatus(devices, blacklist)
should_restart_usb = set(
status['serial'] for status in statuses
if (not status['usb_status']
or status['adb_status'] == 'unknown'))
should_restart_adb = should_restart_usb.union(set(
status['serial'] for status in statuses
if status['adb_status'] in ('offline', 'unauthorized')))
should_reboot_device = should_restart_adb.union(set(
status['serial'] for status in statuses
if status['blacklisted']))
logging.debug('Should restart USB for:')
for d in should_restart_usb:
logging.debug(' %s', d)
logging.debug('Should restart ADB for:')
for d in should_restart_adb:
logging.debug(' %s', d)
logging.debug('Should reboot:')
for d in should_reboot_device:
logging.debug(' %s', d)
if blacklist:
blacklist.Reset()
if should_restart_adb:
KillAllAdb()
for serial in should_restart_usb:
try:
reset_usb.reset_android_usb(serial)
except (IOError, device_errors.DeviceUnreachableError):
logging.exception('Unable to reset USB for %s.', serial)
if blacklist:
blacklist.Extend([serial])
def blacklisting_recovery(device):
if _IsBlacklisted(device.adb.GetDeviceSerial(), blacklist):
logging.debug('%s is blacklisted, skipping recovery.', str(device))
return
if device in should_reboot_device:
try:
device.WaitUntilFullyBooted()
return
except (device_errors.CommandTimeoutError,
device_errors.CommandFailedError):
logging.exception('Failure while waiting for %s. '
'Attempting to recover.', str(device))
try:
device.Reboot()
device.WaitUntilFullyBooted()
except device_errors.CommandFailedError:
logging.exception('Failure while waiting for %s.', str(device))
if blacklist:
blacklist.Extend([device.adb.GetDeviceSerial()])
except device_errors.CommandTimeoutError:
logging.exception('Timed out while waiting for %s. ', str(device))
if blacklist:
blacklist.Extend([device.adb.GetDeviceSerial()])
device_utils.DeviceUtils.parallel(devices).pMap(blacklisting_recovery)
def RecoverDevices(devices, blacklist):
"""Attempts to recover any inoperable devices in the provided list.
Args:
devices: The list of devices to attempt to recover.
blacklist: The current device blacklist, which will be used then
reset.
Returns:
Nothing.
"""
statuses = DeviceStatus(devices, blacklist)
should_restart_usb = set(
status['serial'] for status in statuses
if (not status['usb_status']
or status['adb_status'] == 'unknown'))
should_restart_adb = should_restart_usb.union(set(
status['serial'] for status in statuses
if status['adb_status'] in ('offline', 'unauthorized')))
should_reboot_device = should_restart_adb.union(set(
status['serial'] for status in statuses
if status['blacklisted']))
logging.debug('Should restart USB for:')
for d in should_restart_usb:
logging.debug(' %s', d)
logging.debug('Should restart ADB for:')
for d in should_restart_adb:
logging.debug(' %s', d)
logging.debug('Should reboot:')
for d in should_reboot_device:
logging.debug(' %s', d)
if blacklist:
blacklist.Reset()
if should_restart_adb:
KillAllAdb()
for serial in should_restart_usb:
try:
reset_usb.reset_android_usb(serial)
except (IOError, device_errors.DeviceUnreachableError):
logging.exception('Unable to reset USB for %s.', serial)
if blacklist:
blacklist.Extend([serial], reason='usb_failure')
def blacklisting_recovery(device):
if _IsBlacklisted(device.adb.GetDeviceSerial(), blacklist):
logging.debug('%s is blacklisted, skipping recovery.', str(device))
return
if str(device) in should_reboot_device:
try:
device.WaitUntilFullyBooted(retries=0)
return
except (device_errors.CommandTimeoutError,
device_errors.CommandFailedError):
logging.exception('Failure while waiting for %s. '
'Attempting to recover.', str(device))
try:
try:
device.Reboot(block=False, timeout=5, retries=0)
except device_errors.CommandTimeoutError:
logging.warning('Timed out while attempting to reboot %s normally.'
'Attempting alternative reboot.', str(device))
# The device drops offline before we can grab the exit code, so
# we don't check for status.
device.adb.Root()
device.adb.Shell('echo b > /proc/sysrq-trigger', expect_status=None,
timeout=5, retries=0)
except device_errors.CommandFailedError:
logging.exception('Failed to reboot %s.', str(device))
if blacklist:
blacklist.Extend([device.adb.GetDeviceSerial()],
reason='reboot_failure')
except device_errors.CommandTimeoutError:
logging.exception('Timed out while rebooting %s.', str(device))
if blacklist:
blacklist.Extend([device.adb.GetDeviceSerial()],
reason='reboot_timeout')
try:
device.WaitUntilFullyBooted(retries=0)
except device_errors.CommandFailedError:
logging.exception('Failure while waiting for %s.', str(device))
if blacklist:
blacklist.Extend([device.adb.GetDeviceSerial()],
reason='reboot_failure')
except device_errors.CommandTimeoutError:
logging.exception('Timed out while waiting for %s.', str(device))
if blacklist:
blacklist.Extend([device.adb.GetDeviceSerial()],
reason='reboot_timeout')
device_utils.DeviceUtils.parallel(devices).pMap(blacklisting_recovery)