def _GetAllDevices():
devices_path = os.path.join(os.environ.get('CHROMIUM_OUT_DIR', 'out'),
device_list.LAST_DEVICES_FILENAME)
try:
devices = device_list.GetPersistentDeviceList(devices_path)
except IOError as e:
logging.error('Unable to find %s [%s]', devices_path, e)
devices = android_commands.GetAttachedDevices()
return sorted(devices)
def RecoverDevices(blacklist, output_directory):
# Remove the last build's "bad devices" before checking device statuses.
blacklist.Reset()
previous_devices = set(a.GetDeviceSerial()
for a in adb_wrapper.AdbWrapper.Devices())
KillAllAdb()
reset_usb.reset_all_android_devices()
try:
expected_devices = set(
device_list.GetPersistentDeviceList(
os.path.join(output_directory,
device_list.LAST_DEVICES_FILENAME)))
except IOError:
expected_devices = set()
all_devices = [
device_utils.DeviceUtils(d)
for d in previous_devices.union(expected_devices)
]
def blacklisting_recovery(device):
try:
device.WaitUntilFullyBooted()
except device_errors.CommandFailedError:
logging.exception(
'Failure while waiting for %s. Adding to blacklist.',
str(device))
blacklist.Extend([str(device)])
except device_errors.CommandTimeoutError:
logging.exception(
'Timed out while waiting for %s. Adding to blacklist.',
str(device))
blacklist.Extend([str(device)])
device_utils.DeviceUtils.parallel(all_devices).pMap(blacklisting_recovery)
devices = device_utils.DeviceUtils.HealthyDevices(blacklist)
device_serials = set(d.adb.GetDeviceSerial() for d in devices)
missing_devices = expected_devices.difference(device_serials)
new_devices = device_serials.difference(expected_devices)
if missing_devices or new_devices:
logging.warning('expected_devices:')
for d in sorted(expected_devices):
logging.warning(' %s', d)
logging.warning('devices:')
for d in sorted(device_serials):
logging.warning(' %s', d)
return devices
def _GetAllDevices(active_devices):
devices_path = os.path.join(os.environ.get('CHROMIUM_OUT_DIR', 'out'),
device_list.LAST_DEVICES_FILENAME)
try:
devices = [
device_utils.DeviceUtils(s)
for s in device_list.GetPersistentDeviceList(devices_path)
]
except IOError as e:
logging.error('Unable to find %s [%s]', devices_path, e)
devices = active_devices
return sorted(devices)
def main():
parser = optparse.OptionParser()
parser.add_option('',
'--out-dir',
help='Directory where the device path is stored',
default=os.path.join(constants.DIR_SOURCE_ROOT, 'out'))
parser.add_option(
'--no-provisioning-check',
action='store_true',
help='Will not check if devices are provisioned properly.')
parser.add_option('--device-status-dashboard',
action='store_true',
help='Output device status data for dashboard.')
parser.add_option('--restart-usb',
action='store_true',
help='Restart USB ports before running device check.')
parser.add_option('--json-output',
help='Output JSON information into a specified file.')
parser.add_option('-v',
'--verbose',
action='count',
default=1,
help='Log more information.')
options, args = parser.parse_args()
if args:
parser.error('Unknown options %s' % args)
run_tests_helper.SetLogLevel(options.verbose)
# Remove the last build's "bad devices" before checking device statuses.
device_blacklist.ResetBlacklist()
try:
expected_devices = device_list.GetPersistentDeviceList(
os.path.join(options.out_dir, device_list.LAST_DEVICES_FILENAME))
except IOError:
expected_devices = []
devices = device_utils.DeviceUtils.HealthyDevices()
device_serials = [d.adb.GetDeviceSerial() for d in devices]
# Only restart usb if devices are missing.
if set(expected_devices) != set(device_serials):
logging.warning('expected_devices: %s', expected_devices)
logging.warning('devices: %s', device_serials)
KillAllAdb()
retries = 5
usb_restarted = True
if options.restart_usb:
if not RestartUsb():
usb_restarted = False
bb_annotations.PrintWarning()
logging.error('USB reset stage failed, '
'wait for any device to come back.')
while retries:
logging.info('retry adb devices...')
time.sleep(1)
devices = device_utils.DeviceUtils.HealthyDevices()
device_serials = [d.adb.GetDeviceSerial() for d in devices]
if set(expected_devices) == set(device_serials):
# All devices are online, keep going.
break
if not usb_restarted and devices:
# The USB wasn't restarted, but there's at least one device online.
# No point in trying to wait for all devices.
break
retries -= 1
types, builds, batteries, errors, devices_ok, json_data = ([], [], [], [],
[], [])
if devices:
types, builds, batteries, errors, devices_ok, json_data = (zip(
*[DeviceInfo(dev, options) for dev in devices]))
# Write device info to file for buildbot info display.
if os.path.exists('/home/chrome-bot'):
with open('/home/chrome-bot/.adb_device_info', 'w') as f:
for device in json_data:
try:
f.write('%s %s %s %.1fC %s%%\n' %
(device['serial'], device['type'], device['build'],
float(device['battery']['temperature']) / 10,
device['battery']['level']))
except Exception:
pass
err_msg = CheckForMissingDevices(options, devices) or []
unique_types = list(set(types))
unique_builds = list(set(builds))
bb_annotations.PrintMsg('Online devices: %d. Device types %s, builds %s' %
(len(devices), unique_types, unique_builds))
for j in json_data:
logging.info('Device %s (%s)', j.get('serial'), j.get('type'))
logging.info(' Build: %s (%s)', j.get('build'), j.get('build_detail'))
logging.info(' Current Battery Service state:')
for k, v in j.get('battery', {}).iteritems():
logging.info(' %s: %s', k, v)
logging.info(' IMEI slice: %s', j.get('imei_slice'))
logging.info(' WiFi IP: %s', j.get('wifi_ip'))
for dev, dev_errors in zip(devices, errors):
if dev_errors:
err_msg += ['%s errors:' % str(dev)]
err_msg += [' %s' % error for error in dev_errors]
if err_msg:
bb_annotations.PrintWarning()
for e in err_msg:
logging.error(e)
from_address = '*****@*****.**'
to_addresses = ['*****@*****.**']
bot_name = os.environ.get('BUILDBOT_BUILDERNAME')
slave_name = os.environ.get('BUILDBOT_SLAVENAME')
subject = 'Device status check errors on %s, %s.' % (slave_name,
bot_name)
SendEmail(from_address, to_addresses, [], subject, '\n'.join(err_msg))
if options.device_status_dashboard:
offline_devices = [
device_utils.DeviceUtils(a)
for a in adb_wrapper.AdbWrapper.Devices(is_ready=False)
if a.GetState() == 'offline'
]
perf_tests_results_helper.PrintPerfResult('BotDevices',
'OnlineDevices',
[len(devices)], 'devices')
perf_tests_results_helper.PrintPerfResult('BotDevices',
'OfflineDevices',
[len(offline_devices)],
'devices', 'unimportant')
for dev, battery in zip(devices, batteries):
perf_tests_results_helper.PrintPerfResult('DeviceBattery',
str(dev), [battery], '%',
'unimportant')
if options.json_output:
with open(options.json_output, 'wb') as f:
f.write(json.dumps(json_data, indent=4))
num_failed_devs = 0
for device_ok, device in zip(devices_ok, devices):
if not device_ok:
logging.warning('Blacklisting %s', str(device))
device_blacklist.ExtendBlacklist([str(device)])
num_failed_devs += 1
if num_failed_devs == len(devices):
return 2
if not devices:
return 1
def CheckForMissingDevices(options, devices):
"""Uses file of previous online devices to detect broken phones.
Args:
options: out_dir parameter of options argument is used as the base
directory to load and update the cache file.
devices: A list of DeviceUtils instance for the currently visible and
online attached devices.
"""
out_dir = os.path.abspath(options.out_dir)
device_serials = set(d.adb.GetDeviceSerial() for d in devices)
# last_devices denotes all known devices prior to this run
last_devices_path = os.path.join(out_dir,
device_list.LAST_DEVICES_FILENAME)
last_missing_devices_path = os.path.join(
out_dir, device_list.LAST_MISSING_DEVICES_FILENAME)
try:
last_devices = device_list.GetPersistentDeviceList(last_devices_path)
except IOError:
# Ignore error, file might not exist
last_devices = []
try:
last_missing_devices = device_list.GetPersistentDeviceList(
last_missing_devices_path)
except IOError:
last_missing_devices = []
missing_devs = list(set(last_devices) - device_serials)
new_missing_devs = list(set(missing_devs) - set(last_missing_devices))
if new_missing_devs and os.environ.get('BUILDBOT_SLAVENAME'):
logging.info('new_missing_devs %s' % new_missing_devs)
devices_missing_msg = '%d devices not detected.' % len(missing_devs)
bb_annotations.PrintSummaryText(devices_missing_msg)
from_address = '*****@*****.**'
to_addresses = [
'*****@*****.**',
'*****@*****.**'
]
cc_addresses = ['*****@*****.**']
subject = 'Devices offline on %s, %s, %s' % (
os.environ.get('BUILDBOT_SLAVENAME'),
os.environ.get('BUILDBOT_BUILDERNAME'),
os.environ.get('BUILDBOT_BUILDNUMBER'))
msg = ('Please reboot the following devices:\n%s' %
'\n'.join(map(str, new_missing_devs)))
SendEmail(from_address, to_addresses, cc_addresses, subject, msg)
all_known_devices = list(device_serials | set(last_devices))
device_list.WritePersistentDeviceList(last_devices_path, all_known_devices)
device_list.WritePersistentDeviceList(last_missing_devices_path,
missing_devs)
if not all_known_devices:
# This can happen if for some reason the .last_devices file is not
# present or if it was empty.
return ['No online devices. Have any devices been plugged in?']
if missing_devs:
devices_missing_msg = '%d devices not detected.' % len(missing_devs)
bb_annotations.PrintSummaryText(devices_missing_msg)
return [
'Current online devices: %s' % ', '.join(d
for d in device_serials),
'%s are no longer visible. Were they removed?' % missing_devs
]
else:
new_devs = device_serials - set(last_devices)
if new_devs and os.path.exists(last_devices_path):
bb_annotations.PrintWarning()
bb_annotations.PrintSummaryText('%d new devices detected' %
len(new_devs))
logging.info('New devices detected:')
for d in new_devs:
logging.info(' %s', d)
def main():
parser = optparse.OptionParser()
parser.add_option('',
'--out-dir',
help='Directory where the device path is stored',
default=os.path.join(constants.DIR_SOURCE_ROOT, 'out'))
parser.add_option(
'--no-provisioning-check',
action='store_true',
help='Will not check if devices are provisioned properly.')
parser.add_option('--device-status-dashboard',
action='store_true',
help='Output device status data for dashboard.')
parser.add_option('--restart-usb',
action='store_true',
help='Restart USB ports before running device check.')
parser.add_option('--json-output',
help='Output JSON information into a specified file.')
options, args = parser.parse_args()
if args:
parser.error('Unknown options %s' % args)
# Remove the last build's "bad devices" before checking device statuses.
device_blacklist.ResetBlacklist()
try:
expected_devices = device_list.GetPersistentDeviceList(
os.path.join(options.out_dir, device_list.LAST_DEVICES_FILENAME))
except IOError:
expected_devices = []
devices = android_commands.GetAttachedDevices()
# Only restart usb if devices are missing.
if set(expected_devices) != set(devices):
print 'expected_devices: %s, devices: %s' % (expected_devices, devices)
KillAllAdb()
retries = 5
usb_restarted = True
if options.restart_usb:
if not RestartUsb():
usb_restarted = False
bb_annotations.PrintWarning()
print 'USB reset stage failed, wait for any device to come back.'
while retries:
print 'retry adb devices...'
time.sleep(1)
devices = android_commands.GetAttachedDevices()
if set(expected_devices) == set(devices):
# All devices are online, keep going.
break
if not usb_restarted and devices:
# The USB wasn't restarted, but there's at least one device online.
# No point in trying to wait for all devices.
break
retries -= 1
# TODO(navabi): Test to make sure this fails and then fix call
offline_devices = android_commands.GetAttachedDevices(hardware=False,
emulator=False,
offline=True)
types, builds, batteries, reports, errors = [], [], [], [], []
fail_step_lst = []
if devices:
types, builds, batteries, reports, errors, fail_step_lst = (zip(
*[DeviceInfo(dev, options) for dev in devices]))
err_msg = CheckForMissingDevices(options, devices) or []
unique_types = list(set(types))
unique_builds = list(set(builds))
bb_annotations.PrintMsg('Online devices: %d. Device types %s, builds %s' %
(len(devices), unique_types, unique_builds))
print '\n'.join(reports)
for serial, dev_errors in zip(devices, errors):
if dev_errors:
err_msg += ['%s errors:' % serial]
err_msg += [' %s' % error for error in dev_errors]
if err_msg:
bb_annotations.PrintWarning()
msg = '\n'.join(err_msg)
print msg
from_address = '*****@*****.**'
to_addresses = ['*****@*****.**']
bot_name = os.environ.get('BUILDBOT_BUILDERNAME')
slave_name = os.environ.get('BUILDBOT_SLAVENAME')
subject = 'Device status check errors on %s, %s.' % (slave_name,
bot_name)
SendEmail(from_address, to_addresses, [], subject, msg)
if options.device_status_dashboard:
perf_tests_results_helper.PrintPerfResult('BotDevices',
'OnlineDevices',
[len(devices)], 'devices')
perf_tests_results_helper.PrintPerfResult('BotDevices',
'OfflineDevices',
[len(offline_devices)],
'devices', 'unimportant')
for serial, battery in zip(devices, batteries):
perf_tests_results_helper.PrintPerfResult('DeviceBattery', serial,
[battery], '%',
'unimportant')
if options.json_output:
with open(options.json_output, 'wb') as f:
f.write(
json.dumps({
'online_devices': devices,
'offline_devices': offline_devices,
'expected_devices': expected_devices,
'unique_types': unique_types,
'unique_builds': unique_builds,
}))
if False in fail_step_lst:
# TODO(navabi): Build fails on device status check step if there exists any
# devices with critically low battery. Remove those devices from testing,
# allowing build to continue with good devices.
return 2
if not devices:
return 1
def CheckForMissingDevices(options, adb_online_devs):
"""Uses file of previous online devices to detect broken phones.
Args:
options: out_dir parameter of options argument is used as the base
directory to load and update the cache file.
adb_online_devs: A list of serial numbers of the currently visible
and online attached devices.
"""
# TODO(navabi): remove this once the bug that causes different number
# of devices to be detected between calls is fixed.
logger = logging.getLogger()
logger.setLevel(logging.INFO)
out_dir = os.path.abspath(options.out_dir)
# last_devices denotes all known devices prior to this run
last_devices_path = os.path.join(out_dir,
device_list.LAST_DEVICES_FILENAME)
last_missing_devices_path = os.path.join(
out_dir, device_list.LAST_MISSING_DEVICES_FILENAME)
try:
last_devices = device_list.GetPersistentDeviceList(last_devices_path)
except IOError:
# Ignore error, file might not exist
last_devices = []
try:
last_missing_devices = device_list.GetPersistentDeviceList(
last_missing_devices_path)
except IOError:
last_missing_devices = []
missing_devs = list(set(last_devices) - set(adb_online_devs))
new_missing_devs = list(set(missing_devs) - set(last_missing_devices))
if new_missing_devs and os.environ.get('BUILDBOT_SLAVENAME'):
logging.info('new_missing_devs %s' % new_missing_devs)
devices_missing_msg = '%d devices not detected.' % len(missing_devs)
bb_annotations.PrintSummaryText(devices_missing_msg)
from_address = '*****@*****.**'
to_addresses = [
'*****@*****.**',
'*****@*****.**'
]
cc_addresses = ['*****@*****.**']
subject = 'Devices offline on %s, %s, %s' % (
os.environ.get('BUILDBOT_SLAVENAME'),
os.environ.get('BUILDBOT_BUILDERNAME'),
os.environ.get('BUILDBOT_BUILDNUMBER'))
msg = ('Please reboot the following devices:\n%s' %
'\n'.join(map(str, new_missing_devs)))
SendEmail(from_address, to_addresses, cc_addresses, subject, msg)
all_known_devices = list(set(adb_online_devs) | set(last_devices))
device_list.WritePersistentDeviceList(last_devices_path, all_known_devices)
device_list.WritePersistentDeviceList(last_missing_devices_path,
missing_devs)
if not all_known_devices:
# This can happen if for some reason the .last_devices file is not
# present or if it was empty.
return ['No online devices. Have any devices been plugged in?']
if missing_devs:
devices_missing_msg = '%d devices not detected.' % len(missing_devs)
bb_annotations.PrintSummaryText(devices_missing_msg)
# TODO(navabi): Debug by printing both output from GetCmdOutput and
# GetAttachedDevices to compare results.
crbug_link = (
'https://code.google.com/p/chromium/issues/entry?summary='
'%s&comment=%s&labels=Restrict-View-Google,OS-Android,Infra' %
(urllib.quote('Device Offline'),
urllib.quote('Buildbot: %s %s\n'
'Build: %s\n'
'(please don\'t change any labels)' %
(os.environ.get('BUILDBOT_BUILDERNAME'),
os.environ.get('BUILDBOT_SLAVENAME'),
os.environ.get('BUILDBOT_BUILDNUMBER')))))
return [
'Current online devices: %s' % adb_online_devs,
'%s are no longer visible. Were they removed?\n' % missing_devs,
'SHERIFF:\n',
'@@@STEP_LINK@Click here to file a bug@%s@@@\n' % crbug_link,
'Cache file: %s\n\n' % last_devices_path,
'adb devices: %s' % GetCmdOutput(['adb', 'devices']),
'adb devices(GetAttachedDevices): %s' % adb_online_devs
]
else:
new_devs = set(adb_online_devs) - set(last_devices)
if new_devs and os.path.exists(last_devices_path):
bb_annotations.PrintWarning()
bb_annotations.PrintSummaryText('%d new devices detected' %
len(new_devs))
print(
'New devices detected %s. And now back to your '
'regularly scheduled program.' % list(new_devs))
