def RunCmd(command, flunk_on_failure=True, halt_on_failure=False,
warning_code=constants.WARNING_EXIT_CODE, stdout=None,
cwd=CHROME_SRC):
"""Run a command relative to the chrome source root."""
code = SpawnCmd(command, stdout, cwd).wait()
print '<', CommandToString(command)
if code != 0:
print 'ERROR: process exited with code %d' % code
if code != warning_code and flunk_on_failure:
bb_annotations.PrintError()
else:
bb_annotations.PrintWarning()
# Allow steps to have both halting (i.e. 1) and non-halting exit codes.
if code != warning_code and halt_on_failure:
print 'FATAL %d != %d' % (code, warning_code)
sys.exit(1)
return code
def RebootDevices():
"""Reboot all attached and online devices."""
# Early return here to avoid presubmit dependence on adb,
# which might not exist in this checkout.
if bb_utils.TESTING:
return
devices = android_commands.GetAttachedDevices(emulator=False)
print 'Rebooting: %s' % devices
if devices:
pool = multiprocessing.Pool(len(devices))
results = pool.map_async(RebootDeviceSafe, devices).get(99999)
for device, result in zip(devices, results):
if result:
print '%s failed to startup.' % device
if any(results):
bb_annotations.PrintWarning()
else:
print 'Reboots complete.'
def ForceBatteryStatuses():
"""Apply battery status resets in case no battery is connected."""
# Early return here to avoid presubmit dependence on adb,
# which might not exist in this checkout.
if bb_utils.TESTING:
return
devices = android_commands.GetAttachedDevices(emulator=False)
print 'Forcing battery statuses: %s' % devices
if devices:
pool = multiprocessing.Pool(len(devices))
results = pool.map_async(ForceBatteryStatusSafe, devices).get(99999)
for device, result in zip(devices, results):
if result:
print 'failed to force battery status for %s.' % device
if any(results):
bb_annotations.PrintWarning()
else:
print 'Battery status forcing complete.'
def RunWebkitLayoutTests(options):
"""Run layout tests on an actual device."""
bb_annotations.PrintNamedStep('webkit_tests')
cmd_args = [
'--no-show-results',
'--no-new-test-results',
'--full-results-html',
'--clobber-old-results',
'--exit-after-n-failures',
'5000',
'--exit-after-n-crashes-or-timeouts',
'100',
'--debug-rwt-logging',
'--results-directory',
'../layout-test-results',
'--target',
options.target,
'--builder-name',
options.build_properties.get('buildername', ''),
'--build-number',
str(options.build_properties.get('buildnumber', '')),
'--master-name',
'ChromiumWebkit', # TODO: Get this from the cfg.
'--build-name',
options.build_properties.get('buildername', ''),
'--platform=android'
]
for flag in 'test_results_server', 'driver_name', 'additional_drt_flag':
if flag in options.factory_properties:
cmd_args.extend([
'--%s' % flag.replace('_', '-'),
options.factory_properties.get(flag)
])
for f in options.factory_properties.get('additional_expectations', []):
cmd_args.extend([
'--additional-expectations=%s' % os.path.join(CHROME_SRC_DIR, *f)
])
# TODO(dpranke): Remove this block after
# https://codereview.chromium.org/12927002/ lands.
for f in options.factory_properties.get('additional_expectations_files',
[]):
cmd_args.extend([
'--additional-expectations=%s' % os.path.join(CHROME_SRC_DIR, *f)
])
exit_code = RunCmd(
[SrcPath(os.path.join(BLINK_SCRIPTS_DIR, 'run-webkit-tests'))] +
cmd_args)
if exit_code == 255: # test_run_results.UNEXPECTED_ERROR_EXIT_STATUS
bb_annotations.PrintMsg('?? (crashed or hung)')
elif exit_code == 254: # test_run_results.NO_DEVICES_EXIT_STATUS
bb_annotations.PrintMsg('?? (no devices found)')
elif exit_code == 253: # test_run_results.NO_TESTS_EXIT_STATUS
bb_annotations.PrintMsg('?? (no tests found)')
else:
full_results_path = os.path.join('..', 'layout-test-results',
'full_results.json')
if os.path.exists(full_results_path):
full_results = json.load(open(full_results_path))
unexpected_passes, unexpected_failures, unexpected_flakes = (
_ParseLayoutTestResults(full_results))
if unexpected_failures:
_PrintDashboardLink('failed',
unexpected_failures,
max_tests=25)
elif unexpected_passes:
_PrintDashboardLink('unexpected passes',
unexpected_passes,
max_tests=10)
if unexpected_flakes:
_PrintDashboardLink('unexpected flakes',
unexpected_flakes,
max_tests=10)
if exit_code == 0 and (unexpected_passes or unexpected_flakes):
# If exit_code != 0, RunCmd() will have already printed an error.
bb_annotations.PrintWarning()
else:
bb_annotations.PrintError()
bb_annotations.PrintMsg('?? (results missing)')
if options.factory_properties.get('archive_webkit_results', False):
bb_annotations.PrintNamedStep('archive_webkit_results')
base = 'https://storage.googleapis.com/chromium-layout-test-archives'
builder_name = options.build_properties.get('buildername', '')
build_number = str(options.build_properties.get('buildnumber', ''))
results_link = '%s/%s/%s/layout-test-results/results.html' % (
base, EscapeBuilderName(builder_name), build_number)
bb_annotations.PrintLink('results', results_link)
bb_annotations.PrintLink(
'(zip)', '%s/%s/%s/layout-test-results.zip' %
(base, EscapeBuilderName(builder_name), build_number))
gs_bucket = 'gs://chromium-layout-test-archives'
RunCmd([
os.path.join(SLAVE_SCRIPTS_DIR, 'chromium',
'archive_layout_test_results.py'), '--results-dir',
'../../layout-test-results', '--build-number', build_number,
'--builder-name', builder_name, '--gs-bucket', gs_bucket
],
cwd=DIR_BUILD_ROOT)
def main():
parser = optparse.OptionParser()
parser.add_option('',
'--out-dir',
help='Directory where the device path is stored',
default=os.path.join(constants.DIR_SOURCE_ROOT, 'out'))
parser.add_option(
'--no-provisioning-check',
action='store_true',
help='Will not check if devices are provisioned properly.')
parser.add_option('--device-status-dashboard',
action='store_true',
help='Output device status data for dashboard.')
parser.add_option('--restart-usb',
action='store_true',
help='Restart USB ports before running device check.')
parser.add_option('--json-output',
help='Output JSON information into a specified file.')
parser.add_option('-v',
'--verbose',
action='count',
default=1,
help='Log more information.')
options, args = parser.parse_args()
if args:
parser.error('Unknown options %s' % args)
run_tests_helper.SetLogLevel(options.verbose)
# Remove the last build's "bad devices" before checking device statuses.
device_blacklist.ResetBlacklist()
try:
expected_devices = device_list.GetPersistentDeviceList(
os.path.join(options.out_dir, device_list.LAST_DEVICES_FILENAME))
except IOError:
expected_devices = []
devices = device_utils.DeviceUtils.HealthyDevices()
device_serials = [d.adb.GetDeviceSerial() for d in devices]
# Only restart usb if devices are missing.
if set(expected_devices) != set(device_serials):
logging.warning('expected_devices: %s', expected_devices)
logging.warning('devices: %s', device_serials)
KillAllAdb()
retries = 5
usb_restarted = True
if options.restart_usb:
if not RestartUsb():
usb_restarted = False
bb_annotations.PrintWarning()
logging.error('USB reset stage failed, '
'wait for any device to come back.')
while retries:
logging.info('retry adb devices...')
time.sleep(1)
devices = device_utils.DeviceUtils.HealthyDevices()
device_serials = [d.adb.GetDeviceSerial() for d in devices]
if set(expected_devices) == set(device_serials):
# All devices are online, keep going.
break
if not usb_restarted and devices:
# The USB wasn't restarted, but there's at least one device online.
# No point in trying to wait for all devices.
break
retries -= 1
types, builds, batteries, errors, devices_ok, json_data = ([], [], [], [],
[], [])
if devices:
types, builds, batteries, errors, devices_ok, json_data = (zip(
*[DeviceInfo(dev, options) for dev in devices]))
# Write device info to file for buildbot info display.
if os.path.exists('/home/chrome-bot'):
with open('/home/chrome-bot/.adb_device_info', 'w') as f:
for device in json_data:
try:
f.write('%s %s %s %.1fC %s%%\n' %
(device['serial'], device['type'], device['build'],
float(device['battery']['temperature']) / 10,
device['battery']['level']))
except Exception:
pass
err_msg = CheckForMissingDevices(options, devices) or []
unique_types = list(set(types))
unique_builds = list(set(builds))
bb_annotations.PrintMsg('Online devices: %d. Device types %s, builds %s' %
(len(devices), unique_types, unique_builds))
for j in json_data:
logging.info('Device %s (%s)', j.get('serial'), j.get('type'))
logging.info(' Build: %s (%s)', j.get('build'), j.get('build_detail'))
logging.info(' Current Battery Service state:')
for k, v in j.get('battery', {}).iteritems():
logging.info(' %s: %s', k, v)
logging.info(' IMEI slice: %s', j.get('imei_slice'))
logging.info(' WiFi IP: %s', j.get('wifi_ip'))
for dev, dev_errors in zip(devices, errors):
if dev_errors:
err_msg += ['%s errors:' % str(dev)]
err_msg += [' %s' % error for error in dev_errors]
if err_msg:
bb_annotations.PrintWarning()
for e in err_msg:
logging.error(e)
from_address = '*****@*****.**'
to_addresses = ['*****@*****.**']
bot_name = os.environ.get('BUILDBOT_BUILDERNAME')
slave_name = os.environ.get('BUILDBOT_SLAVENAME')
subject = 'Device status check errors on %s, %s.' % (slave_name,
bot_name)
SendEmail(from_address, to_addresses, [], subject, '\n'.join(err_msg))
if options.device_status_dashboard:
offline_devices = [
device_utils.DeviceUtils(a)
for a in adb_wrapper.AdbWrapper.Devices(is_ready=False)
if a.GetState() == 'offline'
]
perf_tests_results_helper.PrintPerfResult('BotDevices',
'OnlineDevices',
[len(devices)], 'devices')
perf_tests_results_helper.PrintPerfResult('BotDevices',
'OfflineDevices',
[len(offline_devices)],
'devices', 'unimportant')
for dev, battery in zip(devices, batteries):
perf_tests_results_helper.PrintPerfResult('DeviceBattery',
str(dev), [battery], '%',
'unimportant')
if options.json_output:
with open(options.json_output, 'wb') as f:
f.write(json.dumps(json_data, indent=4))
num_failed_devs = 0
for device_ok, device in zip(devices_ok, devices):
if not device_ok:
logging.warning('Blacklisting %s', str(device))
device_blacklist.ExtendBlacklist([str(device)])
num_failed_devs += 1
if num_failed_devs == len(devices):
return 2
if not devices:
return 1
def CheckForMissingDevices(options, devices):
"""Uses file of previous online devices to detect broken phones.
Args:
options: out_dir parameter of options argument is used as the base
directory to load and update the cache file.
devices: A list of DeviceUtils instance for the currently visible and
online attached devices.
"""
out_dir = os.path.abspath(options.out_dir)
device_serials = set(d.adb.GetDeviceSerial() for d in devices)
# last_devices denotes all known devices prior to this run
last_devices_path = os.path.join(out_dir,
device_list.LAST_DEVICES_FILENAME)
last_missing_devices_path = os.path.join(
out_dir, device_list.LAST_MISSING_DEVICES_FILENAME)
try:
last_devices = device_list.GetPersistentDeviceList(last_devices_path)
except IOError:
# Ignore error, file might not exist
last_devices = []
try:
last_missing_devices = device_list.GetPersistentDeviceList(
last_missing_devices_path)
except IOError:
last_missing_devices = []
missing_devs = list(set(last_devices) - device_serials)
new_missing_devs = list(set(missing_devs) - set(last_missing_devices))
if new_missing_devs and os.environ.get('BUILDBOT_SLAVENAME'):
logging.info('new_missing_devs %s' % new_missing_devs)
devices_missing_msg = '%d devices not detected.' % len(missing_devs)
bb_annotations.PrintSummaryText(devices_missing_msg)
from_address = '*****@*****.**'
to_addresses = [
'*****@*****.**',
'*****@*****.**'
]
cc_addresses = ['*****@*****.**']
subject = 'Devices offline on %s, %s, %s' % (
os.environ.get('BUILDBOT_SLAVENAME'),
os.environ.get('BUILDBOT_BUILDERNAME'),
os.environ.get('BUILDBOT_BUILDNUMBER'))
msg = ('Please reboot the following devices:\n%s' %
'\n'.join(map(str, new_missing_devs)))
SendEmail(from_address, to_addresses, cc_addresses, subject, msg)
all_known_devices = list(device_serials | set(last_devices))
device_list.WritePersistentDeviceList(last_devices_path, all_known_devices)
device_list.WritePersistentDeviceList(last_missing_devices_path,
missing_devs)
if not all_known_devices:
# This can happen if for some reason the .last_devices file is not
# present or if it was empty.
return ['No online devices. Have any devices been plugged in?']
if missing_devs:
devices_missing_msg = '%d devices not detected.' % len(missing_devs)
bb_annotations.PrintSummaryText(devices_missing_msg)
return [
'Current online devices: %s' % ', '.join(d
for d in device_serials),
'%s are no longer visible. Were they removed?' % missing_devs
]
else:
new_devs = device_serials - set(last_devices)
if new_devs and os.path.exists(last_devices_path):
bb_annotations.PrintWarning()
bb_annotations.PrintSummaryText('%d new devices detected' %
len(new_devs))
logging.info('New devices detected:')
for d in new_devs:
logging.info(' %s', d)
def main():
parser = optparse.OptionParser()
parser.add_option('',
'--out-dir',
help='Directory where the device path is stored',
default=os.path.join(constants.DIR_SOURCE_ROOT, 'out'))
parser.add_option(
'--no-provisioning-check',
action='store_true',
help='Will not check if devices are provisioned properly.')
parser.add_option('--device-status-dashboard',
action='store_true',
help='Output device status data for dashboard.')
parser.add_option('--restart-usb',
action='store_true',
help='Restart USB ports before running device check.')
parser.add_option('--json-output',
help='Output JSON information into a specified file.')
options, args = parser.parse_args()
if args:
parser.error('Unknown options %s' % args)
# Remove the last build's "bad devices" before checking device statuses.
device_blacklist.ResetBlacklist()
try:
expected_devices = device_list.GetPersistentDeviceList(
os.path.join(options.out_dir, device_list.LAST_DEVICES_FILENAME))
except IOError:
expected_devices = []
devices = android_commands.GetAttachedDevices()
# Only restart usb if devices are missing.
if set(expected_devices) != set(devices):
print 'expected_devices: %s, devices: %s' % (expected_devices, devices)
KillAllAdb()
retries = 5
usb_restarted = True
if options.restart_usb:
if not RestartUsb():
usb_restarted = False
bb_annotations.PrintWarning()
print 'USB reset stage failed, wait for any device to come back.'
while retries:
print 'retry adb devices...'
time.sleep(1)
devices = android_commands.GetAttachedDevices()
if set(expected_devices) == set(devices):
# All devices are online, keep going.
break
if not usb_restarted and devices:
# The USB wasn't restarted, but there's at least one device online.
# No point in trying to wait for all devices.
break
retries -= 1
# TODO(navabi): Test to make sure this fails and then fix call
offline_devices = android_commands.GetAttachedDevices(hardware=False,
emulator=False,
offline=True)
types, builds, batteries, reports, errors = [], [], [], [], []
fail_step_lst = []
if devices:
types, builds, batteries, reports, errors, fail_step_lst = (zip(
*[DeviceInfo(dev, options) for dev in devices]))
err_msg = CheckForMissingDevices(options, devices) or []
unique_types = list(set(types))
unique_builds = list(set(builds))
bb_annotations.PrintMsg('Online devices: %d. Device types %s, builds %s' %
(len(devices), unique_types, unique_builds))
print '\n'.join(reports)
for serial, dev_errors in zip(devices, errors):
if dev_errors:
err_msg += ['%s errors:' % serial]
err_msg += [' %s' % error for error in dev_errors]
if err_msg:
bb_annotations.PrintWarning()
msg = '\n'.join(err_msg)
print msg
from_address = '*****@*****.**'
to_addresses = ['*****@*****.**']
bot_name = os.environ.get('BUILDBOT_BUILDERNAME')
slave_name = os.environ.get('BUILDBOT_SLAVENAME')
subject = 'Device status check errors on %s, %s.' % (slave_name,
bot_name)
SendEmail(from_address, to_addresses, [], subject, msg)
if options.device_status_dashboard:
perf_tests_results_helper.PrintPerfResult('BotDevices',
'OnlineDevices',
[len(devices)], 'devices')
perf_tests_results_helper.PrintPerfResult('BotDevices',
'OfflineDevices',
[len(offline_devices)],
'devices', 'unimportant')
for serial, battery in zip(devices, batteries):
perf_tests_results_helper.PrintPerfResult('DeviceBattery', serial,
[battery], '%',
'unimportant')
if options.json_output:
with open(options.json_output, 'wb') as f:
f.write(
json.dumps({
'online_devices': devices,
'offline_devices': offline_devices,
'expected_devices': expected_devices,
'unique_types': unique_types,
'unique_builds': unique_builds,
}))
if False in fail_step_lst:
# TODO(navabi): Build fails on device status check step if there exists any
# devices with critically low battery. Remove those devices from testing,
# allowing build to continue with good devices.
return 2
if not devices:
return 1
def CheckForMissingDevices(options, adb_online_devs):
"""Uses file of previous online devices to detect broken phones.
Args:
options: out_dir parameter of options argument is used as the base
directory to load and update the cache file.
adb_online_devs: A list of serial numbers of the currently visible
and online attached devices.
"""
# TODO(navabi): remove this once the bug that causes different number
# of devices to be detected between calls is fixed.
logger = logging.getLogger()
logger.setLevel(logging.INFO)
out_dir = os.path.abspath(options.out_dir)
# last_devices denotes all known devices prior to this run
last_devices_path = os.path.join(out_dir,
device_list.LAST_DEVICES_FILENAME)
last_missing_devices_path = os.path.join(
out_dir, device_list.LAST_MISSING_DEVICES_FILENAME)
try:
last_devices = device_list.GetPersistentDeviceList(last_devices_path)
except IOError:
# Ignore error, file might not exist
last_devices = []
try:
last_missing_devices = device_list.GetPersistentDeviceList(
last_missing_devices_path)
except IOError:
last_missing_devices = []
missing_devs = list(set(last_devices) - set(adb_online_devs))
new_missing_devs = list(set(missing_devs) - set(last_missing_devices))
if new_missing_devs and os.environ.get('BUILDBOT_SLAVENAME'):
logging.info('new_missing_devs %s' % new_missing_devs)
devices_missing_msg = '%d devices not detected.' % len(missing_devs)
bb_annotations.PrintSummaryText(devices_missing_msg)
from_address = '*****@*****.**'
to_addresses = [
'*****@*****.**',
'*****@*****.**'
]
cc_addresses = ['*****@*****.**']
subject = 'Devices offline on %s, %s, %s' % (
os.environ.get('BUILDBOT_SLAVENAME'),
os.environ.get('BUILDBOT_BUILDERNAME'),
os.environ.get('BUILDBOT_BUILDNUMBER'))
msg = ('Please reboot the following devices:\n%s' %
'\n'.join(map(str, new_missing_devs)))
SendEmail(from_address, to_addresses, cc_addresses, subject, msg)
all_known_devices = list(set(adb_online_devs) | set(last_devices))
device_list.WritePersistentDeviceList(last_devices_path, all_known_devices)
device_list.WritePersistentDeviceList(last_missing_devices_path,
missing_devs)
if not all_known_devices:
# This can happen if for some reason the .last_devices file is not
# present or if it was empty.
return ['No online devices. Have any devices been plugged in?']
if missing_devs:
devices_missing_msg = '%d devices not detected.' % len(missing_devs)
bb_annotations.PrintSummaryText(devices_missing_msg)
# TODO(navabi): Debug by printing both output from GetCmdOutput and
# GetAttachedDevices to compare results.
crbug_link = (
'https://code.google.com/p/chromium/issues/entry?summary='
'%s&comment=%s&labels=Restrict-View-Google,OS-Android,Infra' %
(urllib.quote('Device Offline'),
urllib.quote('Buildbot: %s %s\n'
'Build: %s\n'
'(please don\'t change any labels)' %
(os.environ.get('BUILDBOT_BUILDERNAME'),
os.environ.get('BUILDBOT_SLAVENAME'),
os.environ.get('BUILDBOT_BUILDNUMBER')))))
return [
'Current online devices: %s' % adb_online_devs,
'%s are no longer visible. Were they removed?\n' % missing_devs,
'SHERIFF:\n',
'@@@STEP_LINK@Click here to file a bug@%s@@@\n' % crbug_link,
'Cache file: %s\n\n' % last_devices_path,
'adb devices: %s' % GetCmdOutput(['adb', 'devices']),
'adb devices(GetAttachedDevices): %s' % adb_online_devs
]
else:
new_devs = set(adb_online_devs) - set(last_devices)
if new_devs and os.path.exists(last_devices_path):
bb_annotations.PrintWarning()
bb_annotations.PrintSummaryText('%d new devices detected' %
len(new_devs))
print(
'New devices detected %s. And now back to your '
'regularly scheduled program.' % list(new_devs))
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--out-dir',
help='Directory where the device path is stored',
default=os.path.join(constants.DIR_SOURCE_ROOT, 'out'))
parser.add_argument('--no-provisioning-check',
action='store_true',
help='Will not check if devices are provisioned '
'properly.')
parser.add_argument('--device-status-dashboard',
action='store_true',
help='Output device status data for dashboard.')
parser.add_argument('--restart-usb',
action='store_true',
help='DEPRECATED. '
'This script now always tries to reset USB.')
parser.add_argument('--json-output',
help='Output JSON information into a specified file.')
parser.add_argument('--blacklist-file', help='Device blacklist JSON file.')
parser.add_argument('-v',
'--verbose',
action='count',
default=1,
help='Log more information.')
args = parser.parse_args()
run_tests_helper.SetLogLevel(args.verbose)
if args.blacklist_file:
blacklist = device_blacklist.Blacklist(args.blacklist_file)
else:
# TODO(jbudorick): Remove this once bots pass the blacklist file.
blacklist = device_blacklist.Blacklist(device_blacklist.BLACKLIST_JSON)
devices = RecoverDevices(blacklist, args.out_dir)
types, builds, batteries, errors, devices_ok, json_data = ([], [], [], [],
[], [])
if devices:
types, builds, batteries, errors, devices_ok, json_data = (zip(
*[DeviceInfo(dev, args) for dev in devices]))
# Write device info to file for buildbot info display.
if os.path.exists('/home/chrome-bot'):
with open('/home/chrome-bot/.adb_device_info', 'w') as f:
for device in json_data:
try:
f.write('%s %s %s %.1fC %s%%\n' %
(device['serial'], device['type'], device['build'],
float(device['battery']['temperature']) / 10,
device['battery']['level']))
except Exception:
pass
err_msg = CheckForMissingDevices(args, devices) or []
unique_types = list(set(types))
unique_builds = list(set(builds))
bb_annotations.PrintMsg('Online devices: %d. Device types %s, builds %s' %
(len(devices), unique_types, unique_builds))
for j in json_data:
logging.info('Device %s (%s)', j.get('serial'), j.get('type'))
logging.info(' Build: %s (%s)', j.get('build'), j.get('build_detail'))
logging.info(' Current Battery Service state:')
for k, v in j.get('battery', {}).iteritems():
logging.info(' %s: %s', k, v)
logging.info(' IMEI slice: %s', j.get('imei_slice'))
logging.info(' WiFi IP: %s', j.get('wifi_ip'))
for dev, dev_errors in zip(devices, errors):
if dev_errors:
err_msg += ['%s errors:' % str(dev)]
err_msg += [' %s' % error for error in dev_errors]
if err_msg:
bb_annotations.PrintWarning()
for e in err_msg:
logging.error(e)
from_address = '*****@*****.**'
to_addresses = ['*****@*****.**']
bot_name = os.environ.get('BUILDBOT_BUILDERNAME')
slave_name = os.environ.get('BUILDBOT_SLAVENAME')
subject = 'Device status check errors on %s, %s.' % (slave_name,
bot_name)
SendEmail(from_address, to_addresses, [], subject, '\n'.join(err_msg))
if args.device_status_dashboard:
offline_devices = [
device_utils.DeviceUtils(a)
for a in adb_wrapper.AdbWrapper.Devices(desired_state=None)
if a.GetState() == 'offline'
]
perf_tests_results_helper.PrintPerfResult('BotDevices',
'OnlineDevices',
[len(devices)], 'devices')
perf_tests_results_helper.PrintPerfResult('BotDevices',
'OfflineDevices',
[len(offline_devices)],
'devices', 'unimportant')
for dev, battery in zip(devices, batteries):
perf_tests_results_helper.PrintPerfResult('DeviceBattery',
str(dev), [battery], '%',
'unimportant')
if args.json_output:
with open(args.json_output, 'wb') as f:
f.write(json.dumps(json_data, indent=4))
num_failed_devs = 0
for device_ok, device in zip(devices_ok, devices):
if not device_ok:
logging.warning('Blacklisting %s', str(device))
blacklist.Extend([str(device)])
num_failed_devs += 1
if num_failed_devs == len(devices):
return 2
if not devices:
return 1
def CheckForMissingDevices(options, adb_online_devs):
"""Uses file of previous online devices to detect broken phones.
Args:
options: out_dir parameter of options argument is used as the base
directory to load and update the cache file.
adb_online_devs: A list of serial numbers of the currently visible
and online attached devices.
"""
# TODO(navabi): remove this once the bug that causes different number
# of devices to be detected between calls is fixed.
logger = logging.getLogger()
logger.setLevel(logging.INFO)
out_dir = os.path.abspath(options.out_dir)
def ReadDeviceList(file_name):
devices_path = os.path.join(out_dir, file_name)
devices = []
try:
with open(devices_path) as f:
devices = f.read().splitlines()
except IOError:
# Ignore error, file might not exist
pass
return devices
def WriteDeviceList(file_name, device_list):
path = os.path.join(out_dir, file_name)
if not os.path.exists(out_dir):
os.makedirs(out_dir)
with open(path, 'w') as f:
# Write devices currently visible plus devices previously seen.
f.write('\n'.join(set(device_list)))
last_devices_path = os.path.join(out_dir, '.last_devices')
last_devices = ReadDeviceList('.last_devices')
missing_devs = list(set(last_devices) - set(adb_online_devs))
all_known_devices = list(set(adb_online_devs) | set(last_devices))
WriteDeviceList('.last_devices', all_known_devices)
WriteDeviceList('.last_missing', missing_devs)
if not all_known_devices:
# This can happen if for some reason the .last_devices file is not
# present or if it was empty.
return ['No online devices. Have any devices been plugged in?']
if missing_devs:
devices_missing_msg = '%d devices not detected.' % len(missing_devs)
bb_annotations.PrintSummaryText(devices_missing_msg)
# TODO(navabi): Debug by printing both output from GetCmdOutput and
# GetAttachedDevices to compare results.
return [
'Current online devices: %s' % adb_online_devs,
'%s are no longer visible. Were they removed?\n' % missing_devs,
'SHERIFF: See go/chrome_device_monitor',
'Cache file: %s\n\n' % last_devices_path,
'adb devices: %s' % GetCmdOutput(['adb', 'devices']),
'adb devices(GetAttachedDevices): %s' %
android_commands.GetAttachedDevices()
]
else:
new_devs = set(adb_online_devs) - set(last_devices)
if new_devs and os.path.exists(last_devices_path):
bb_annotations.PrintWarning()
bb_annotations.PrintSummaryText('%d new devices detected' %
len(new_devs))
print(
'New devices detected %s. And now back to your '
'regularly scheduled program.' % list(new_devs))
def main():
parser = optparse.OptionParser()
parser.add_option('',
'--out-dir',
help='Directory where the device path is stored',
default=os.path.join(constants.DIR_SOURCE_ROOT, 'out'))
parser.add_option(
'--no-provisioning-check',
help='Will not check if devices are provisioned properly.')
parser.add_option('--device-status-dashboard',
help='Output device status data for dashboard.')
options, args = parser.parse_args()
if args:
parser.error('Unknown options %s' % args)
devices = android_commands.GetAttachedDevices()
# TODO(navabi): Test to make sure this fails and then fix call
offline_devices = android_commands.GetAttachedDevices(hardware=False,
emulator=False,
offline=True)
types, builds, batteries, reports, errors = [], [], [], [], []
fail_step_lst = []
if devices:
types, builds, batteries, reports, errors, fail_step_lst = (zip(
*[DeviceInfo(dev, options) for dev in devices]))
err_msg = CheckForMissingDevices(options, devices) or []
unique_types = list(set(types))
unique_builds = list(set(builds))
bb_annotations.PrintMsg('Online devices: %d. Device types %s, builds %s' %
(len(devices), unique_types, unique_builds))
print '\n'.join(reports)
for serial, dev_errors in zip(devices, errors):
if dev_errors:
err_msg += ['%s errors:' % serial]
err_msg += [' %s' % error for error in dev_errors]
if err_msg:
bb_annotations.PrintWarning()
msg = '\n'.join(err_msg)
print msg
SendDeviceStatusAlert(msg)
if options.device_status_dashboard:
perf_tests_helper.PrintPerfResult('BotDevices', 'OnlineDevices',
[len(devices)], 'devices')
perf_tests_helper.PrintPerfResult('BotDevices', 'OfflineDevices',
[len(offline_devices)], 'devices',
'unimportant')
for serial, battery in zip(devices, batteries):
perf_tests_helper.PrintPerfResult('DeviceBattery', serial,
[battery], '%', 'unimportant')
if False in fail_step_lst:
# TODO(navabi): Build fails on device status check step if there exists any
# devices with critically low battery or install speed. Remove those devices
# from testing, allowing build to continue with good devices.
return 1
if not devices:
return 1
def main():
parser = optparse.OptionParser()
parser.add_option('',
'--out-dir',
help='Directory where the device path is stored',
default=os.path.join(constants.DIR_SOURCE_ROOT, 'out'))
parser.add_option(
'--no-provisioning-check',
action='store_true',
help='Will not check if devices are provisioned properly.')
parser.add_option('--device-status-dashboard',
action='store_true',
help='Output device status data for dashboard.')
parser.add_option('--restart-usb',
action='store_true',
help='Restart USB ports before running device check.')
options, args = parser.parse_args()
if args:
parser.error('Unknown options %s' % args)
# Remove the last build's "bad devices" before checking device statuses.
device_blacklist.ResetBlacklist()
if options.restart_usb:
expected_devices = GetLastDevices(os.path.abspath(options.out_dir))
devices = android_commands.GetAttachedDevices()
# Only restart usb if devices are missing.
if set(expected_devices) != set(devices):
KillAllAdb()
retries = 5
usb_restarted = True
if not RestartUsb():
usb_restarted = False
bb_annotations.PrintWarning()
print 'USB reset stage failed, wait for any device to come back.'
while retries:
time.sleep(1)
devices = android_commands.GetAttachedDevices()
if set(expected_devices) == set(devices):
# All devices are online, keep going.
break
if not usb_restarted and devices:
# The USB wasn't restarted, but there's at least one device online.
# No point in trying to wait for all devices.
break
retries -= 1
devices = android_commands.GetAttachedDevices()
# TODO(navabi): Test to make sure this fails and then fix call
offline_devices = android_commands.GetAttachedDevices(hardware=False,
emulator=False,
offline=True)
types, builds, batteries, reports, errors = [], [], [], [], []
fail_step_lst = []
if devices:
types, builds, batteries, reports, errors, fail_step_lst = (zip(
*[DeviceInfo(dev, options) for dev in devices]))
err_msg = CheckForMissingDevices(options, devices) or []
unique_types = list(set(types))
unique_builds = list(set(builds))
bb_annotations.PrintMsg('Online devices: %d. Device types %s, builds %s' %
(len(devices), unique_types, unique_builds))
print '\n'.join(reports)
for serial, dev_errors in zip(devices, errors):
if dev_errors:
err_msg += ['%s errors:' % serial]
err_msg += [' %s' % error for error in dev_errors]
if err_msg:
bb_annotations.PrintWarning()
msg = '\n'.join(err_msg)
print msg
SendDeviceStatusAlert(msg)
if options.device_status_dashboard:
perf_tests_results_helper.PrintPerfResult('BotDevices',
'OnlineDevices',
[len(devices)], 'devices')
perf_tests_results_helper.PrintPerfResult('BotDevices',
'OfflineDevices',
[len(offline_devices)],
'devices', 'unimportant')
for serial, battery in zip(devices, batteries):
perf_tests_results_helper.PrintPerfResult('DeviceBattery', serial,
[battery], '%',
'unimportant')
if False in fail_step_lst:
# TODO(navabi): Build fails on device status check step if there exists any
# devices with critically low battery. Remove those devices from testing,
# allowing build to continue with good devices.
return 1
if not devices:
return 1
def CheckForMissingDevices(options, adb_online_devs):
"""Uses file of previous online devices to detect broken phones.
Args:
options: out_dir parameter of options argument is used as the base
directory to load and update the cache file.
adb_online_devs: A list of serial numbers of the currently visible
and online attached devices.
"""
# TODO(navabi): remove this once the bug that causes different number
# of devices to be detected between calls is fixed.
logger = logging.getLogger()
logger.setLevel(logging.INFO)
out_dir = os.path.abspath(options.out_dir)
def WriteDeviceList(file_name, device_list):
path = os.path.join(out_dir, file_name)
if not os.path.exists(out_dir):
os.makedirs(out_dir)
with open(path, 'w') as f:
# Write devices currently visible plus devices previously seen.
f.write('\n'.join(set(device_list)))
last_devices_path = os.path.join(out_dir, '.last_devices')
last_devices = GetLastDevices(out_dir)
missing_devs = list(set(last_devices) - set(adb_online_devs))
all_known_devices = list(set(adb_online_devs) | set(last_devices))
WriteDeviceList('.last_devices', all_known_devices)
WriteDeviceList('.last_missing', missing_devs)
if not all_known_devices:
# This can happen if for some reason the .last_devices file is not
# present or if it was empty.
return ['No online devices. Have any devices been plugged in?']
if missing_devs:
devices_missing_msg = '%d devices not detected.' % len(missing_devs)
bb_annotations.PrintSummaryText(devices_missing_msg)
# TODO(navabi): Debug by printing both output from GetCmdOutput and
# GetAttachedDevices to compare results.
crbug_link = (
'https://code.google.com/p/chromium/issues/entry?summary='
'%s&comment=%s&labels=Restrict-View-Google,OS-Android,Infra' %
(urllib.quote('Device Offline'),
urllib.quote('Buildbot: %s %s\n'
'Build: %s\n'
'(please don\'t change any labels)' %
(os.environ.get('BUILDBOT_BUILDERNAME'),
os.environ.get('BUILDBOT_SLAVENAME'),
os.environ.get('BUILDBOT_BUILDNUMBER')))))
return [
'Current online devices: %s' % adb_online_devs,
'%s are no longer visible. Were they removed?\n' % missing_devs,
'SHERIFF:\n',
'@@@STEP_LINK@Click here to file a bug@%s@@@\n' % crbug_link,
'Cache file: %s\n\n' % last_devices_path,
'adb devices: %s' % GetCmdOutput(['adb', 'devices']),
'adb devices(GetAttachedDevices): %s' %
android_commands.GetAttachedDevices()
]
else:
new_devs = set(adb_online_devs) - set(last_devices)
if new_devs and os.path.exists(last_devices_path):
bb_annotations.PrintWarning()
bb_annotations.PrintSummaryText('%d new devices detected' %
len(new_devs))
print(
'New devices detected %s. And now back to your '
'regularly scheduled program.' % list(new_devs))