def RunWebkitLayoutTests(options):
"""Run layout tests on an actual device."""
bb_annotations.PrintNamedStep('webkit_tests')
cmd_args = [
'--no-show-results',
'--no-new-test-results',
'--full-results-html',
'--clobber-old-results',
'--exit-after-n-failures',
'5000',
'--exit-after-n-crashes-or-timeouts',
'100',
'--debug-rwt-logging',
'--results-directory',
'../layout-test-results',
'--target',
options.target,
'--builder-name',
options.build_properties.get('buildername', ''),
'--build-number',
str(options.build_properties.get('buildnumber', '')),
'--master-name',
'ChromiumWebkit', # TODO: Get this from the cfg.
'--build-name',
options.build_properties.get('buildername', ''),
'--platform=android'
]
for flag in 'test_results_server', 'driver_name', 'additional_drt_flag':
if flag in options.factory_properties:
cmd_args.extend([
'--%s' % flag.replace('_', '-'),
options.factory_properties.get(flag)
])
for f in options.factory_properties.get('additional_expectations', []):
cmd_args.extend([
'--additional-expectations=%s' % os.path.join(CHROME_SRC_DIR, *f)
])
# TODO(dpranke): Remove this block after
# https://codereview.chromium.org/12927002/ lands.
for f in options.factory_properties.get('additional_expectations_files',
[]):
cmd_args.extend([
'--additional-expectations=%s' % os.path.join(CHROME_SRC_DIR, *f)
])
exit_code = RunCmd(
[SrcPath(os.path.join(BLINK_SCRIPTS_DIR, 'run-webkit-tests'))] +
cmd_args)
if exit_code == 255: # test_run_results.UNEXPECTED_ERROR_EXIT_STATUS
bb_annotations.PrintMsg('?? (crashed or hung)')
elif exit_code == 254: # test_run_results.NO_DEVICES_EXIT_STATUS
bb_annotations.PrintMsg('?? (no devices found)')
elif exit_code == 253: # test_run_results.NO_TESTS_EXIT_STATUS
bb_annotations.PrintMsg('?? (no tests found)')
else:
full_results_path = os.path.join('..', 'layout-test-results',
'full_results.json')
if os.path.exists(full_results_path):
full_results = json.load(open(full_results_path))
unexpected_passes, unexpected_failures, unexpected_flakes = (
_ParseLayoutTestResults(full_results))
if unexpected_failures:
_PrintDashboardLink('failed',
unexpected_failures,
max_tests=25)
elif unexpected_passes:
_PrintDashboardLink('unexpected passes',
unexpected_passes,
max_tests=10)
if unexpected_flakes:
_PrintDashboardLink('unexpected flakes',
unexpected_flakes,
max_tests=10)
if exit_code == 0 and (unexpected_passes or unexpected_flakes):
# If exit_code != 0, RunCmd() will have already printed an error.
bb_annotations.PrintWarning()
else:
bb_annotations.PrintError()
bb_annotations.PrintMsg('?? (results missing)')
if options.factory_properties.get('archive_webkit_results', False):
bb_annotations.PrintNamedStep('archive_webkit_results')
base = 'https://storage.googleapis.com/chromium-layout-test-archives'
builder_name = options.build_properties.get('buildername', '')
build_number = str(options.build_properties.get('buildnumber', ''))
results_link = '%s/%s/%s/layout-test-results/results.html' % (
base, EscapeBuilderName(builder_name), build_number)
bb_annotations.PrintLink('results', results_link)
bb_annotations.PrintLink(
'(zip)', '%s/%s/%s/layout-test-results.zip' %
(base, EscapeBuilderName(builder_name), build_number))
gs_bucket = 'gs://chromium-layout-test-archives'
RunCmd([
os.path.join(SLAVE_SCRIPTS_DIR, 'chromium',
'archive_layout_test_results.py'), '--results-dir',
'../../layout-test-results', '--build-number', build_number,
'--builder-name', builder_name, '--gs-bucket', gs_bucket
],
cwd=DIR_BUILD_ROOT)
def main():
parser = optparse.OptionParser()
parser.add_option('',
'--out-dir',
help='Directory where the device path is stored',
default=os.path.join(constants.DIR_SOURCE_ROOT, 'out'))
parser.add_option(
'--no-provisioning-check',
action='store_true',
help='Will not check if devices are provisioned properly.')
parser.add_option('--device-status-dashboard',
action='store_true',
help='Output device status data for dashboard.')
parser.add_option('--restart-usb',
action='store_true',
help='Restart USB ports before running device check.')
parser.add_option('--json-output',
help='Output JSON information into a specified file.')
parser.add_option('-v',
'--verbose',
action='count',
default=1,
help='Log more information.')
options, args = parser.parse_args()
if args:
parser.error('Unknown options %s' % args)
run_tests_helper.SetLogLevel(options.verbose)
# Remove the last build's "bad devices" before checking device statuses.
device_blacklist.ResetBlacklist()
try:
expected_devices = device_list.GetPersistentDeviceList(
os.path.join(options.out_dir, device_list.LAST_DEVICES_FILENAME))
except IOError:
expected_devices = []
devices = device_utils.DeviceUtils.HealthyDevices()
device_serials = [d.adb.GetDeviceSerial() for d in devices]
# Only restart usb if devices are missing.
if set(expected_devices) != set(device_serials):
logging.warning('expected_devices: %s', expected_devices)
logging.warning('devices: %s', device_serials)
KillAllAdb()
retries = 5
usb_restarted = True
if options.restart_usb:
if not RestartUsb():
usb_restarted = False
bb_annotations.PrintWarning()
logging.error('USB reset stage failed, '
'wait for any device to come back.')
while retries:
logging.info('retry adb devices...')
time.sleep(1)
devices = device_utils.DeviceUtils.HealthyDevices()
device_serials = [d.adb.GetDeviceSerial() for d in devices]
if set(expected_devices) == set(device_serials):
# All devices are online, keep going.
break
if not usb_restarted and devices:
# The USB wasn't restarted, but there's at least one device online.
# No point in trying to wait for all devices.
break
retries -= 1
types, builds, batteries, errors, devices_ok, json_data = ([], [], [], [],
[], [])
if devices:
types, builds, batteries, errors, devices_ok, json_data = (zip(
*[DeviceInfo(dev, options) for dev in devices]))
# Write device info to file for buildbot info display.
if os.path.exists('/home/chrome-bot'):
with open('/home/chrome-bot/.adb_device_info', 'w') as f:
for device in json_data:
try:
f.write('%s %s %s %.1fC %s%%\n' %
(device['serial'], device['type'], device['build'],
float(device['battery']['temperature']) / 10,
device['battery']['level']))
except Exception:
pass
err_msg = CheckForMissingDevices(options, devices) or []
unique_types = list(set(types))
unique_builds = list(set(builds))
bb_annotations.PrintMsg('Online devices: %d. Device types %s, builds %s' %
(len(devices), unique_types, unique_builds))
for j in json_data:
logging.info('Device %s (%s)', j.get('serial'), j.get('type'))
logging.info(' Build: %s (%s)', j.get('build'), j.get('build_detail'))
logging.info(' Current Battery Service state:')
for k, v in j.get('battery', {}).iteritems():
logging.info(' %s: %s', k, v)
logging.info(' IMEI slice: %s', j.get('imei_slice'))
logging.info(' WiFi IP: %s', j.get('wifi_ip'))
for dev, dev_errors in zip(devices, errors):
if dev_errors:
err_msg += ['%s errors:' % str(dev)]
err_msg += [' %s' % error for error in dev_errors]
if err_msg:
bb_annotations.PrintWarning()
for e in err_msg:
logging.error(e)
from_address = '*****@*****.**'
to_addresses = ['*****@*****.**']
bot_name = os.environ.get('BUILDBOT_BUILDERNAME')
slave_name = os.environ.get('BUILDBOT_SLAVENAME')
subject = 'Device status check errors on %s, %s.' % (slave_name,
bot_name)
SendEmail(from_address, to_addresses, [], subject, '\n'.join(err_msg))
if options.device_status_dashboard:
offline_devices = [
device_utils.DeviceUtils(a)
for a in adb_wrapper.AdbWrapper.Devices(is_ready=False)
if a.GetState() == 'offline'
]
perf_tests_results_helper.PrintPerfResult('BotDevices',
'OnlineDevices',
[len(devices)], 'devices')
perf_tests_results_helper.PrintPerfResult('BotDevices',
'OfflineDevices',
[len(offline_devices)],
'devices', 'unimportant')
for dev, battery in zip(devices, batteries):
perf_tests_results_helper.PrintPerfResult('DeviceBattery',
str(dev), [battery], '%',
'unimportant')
if options.json_output:
with open(options.json_output, 'wb') as f:
f.write(json.dumps(json_data, indent=4))
num_failed_devs = 0
for device_ok, device in zip(devices_ok, devices):
if not device_ok:
logging.warning('Blacklisting %s', str(device))
device_blacklist.ExtendBlacklist([str(device)])
num_failed_devs += 1
if num_failed_devs == len(devices):
return 2
if not devices:
return 1
def main():
parser = optparse.OptionParser()
parser.add_option('',
'--out-dir',
help='Directory where the device path is stored',
default=os.path.join(constants.DIR_SOURCE_ROOT, 'out'))
parser.add_option(
'--no-provisioning-check',
action='store_true',
help='Will not check if devices are provisioned properly.')
parser.add_option('--device-status-dashboard',
action='store_true',
help='Output device status data for dashboard.')
parser.add_option('--restart-usb',
action='store_true',
help='Restart USB ports before running device check.')
parser.add_option('--json-output',
help='Output JSON information into a specified file.')
options, args = parser.parse_args()
if args:
parser.error('Unknown options %s' % args)
# Remove the last build's "bad devices" before checking device statuses.
device_blacklist.ResetBlacklist()
try:
expected_devices = device_list.GetPersistentDeviceList(
os.path.join(options.out_dir, device_list.LAST_DEVICES_FILENAME))
except IOError:
expected_devices = []
devices = android_commands.GetAttachedDevices()
# Only restart usb if devices are missing.
if set(expected_devices) != set(devices):
print 'expected_devices: %s, devices: %s' % (expected_devices, devices)
KillAllAdb()
retries = 5
usb_restarted = True
if options.restart_usb:
if not RestartUsb():
usb_restarted = False
bb_annotations.PrintWarning()
print 'USB reset stage failed, wait for any device to come back.'
while retries:
print 'retry adb devices...'
time.sleep(1)
devices = android_commands.GetAttachedDevices()
if set(expected_devices) == set(devices):
# All devices are online, keep going.
break
if not usb_restarted and devices:
# The USB wasn't restarted, but there's at least one device online.
# No point in trying to wait for all devices.
break
retries -= 1
# TODO(navabi): Test to make sure this fails and then fix call
offline_devices = android_commands.GetAttachedDevices(hardware=False,
emulator=False,
offline=True)
types, builds, batteries, reports, errors = [], [], [], [], []
fail_step_lst = []
if devices:
types, builds, batteries, reports, errors, fail_step_lst = (zip(
*[DeviceInfo(dev, options) for dev in devices]))
err_msg = CheckForMissingDevices(options, devices) or []
unique_types = list(set(types))
unique_builds = list(set(builds))
bb_annotations.PrintMsg('Online devices: %d. Device types %s, builds %s' %
(len(devices), unique_types, unique_builds))
print '\n'.join(reports)
for serial, dev_errors in zip(devices, errors):
if dev_errors:
err_msg += ['%s errors:' % serial]
err_msg += [' %s' % error for error in dev_errors]
if err_msg:
bb_annotations.PrintWarning()
msg = '\n'.join(err_msg)
print msg
from_address = '*****@*****.**'
to_addresses = ['*****@*****.**']
bot_name = os.environ.get('BUILDBOT_BUILDERNAME')
slave_name = os.environ.get('BUILDBOT_SLAVENAME')
subject = 'Device status check errors on %s, %s.' % (slave_name,
bot_name)
SendEmail(from_address, to_addresses, [], subject, msg)
if options.device_status_dashboard:
perf_tests_results_helper.PrintPerfResult('BotDevices',
'OnlineDevices',
[len(devices)], 'devices')
perf_tests_results_helper.PrintPerfResult('BotDevices',
'OfflineDevices',
[len(offline_devices)],
'devices', 'unimportant')
for serial, battery in zip(devices, batteries):
perf_tests_results_helper.PrintPerfResult('DeviceBattery', serial,
[battery], '%',
'unimportant')
if options.json_output:
with open(options.json_output, 'wb') as f:
f.write(
json.dumps({
'online_devices': devices,
'offline_devices': offline_devices,
'expected_devices': expected_devices,
'unique_types': unique_types,
'unique_builds': unique_builds,
}))
if False in fail_step_lst:
# TODO(navabi): Build fails on device status check step if there exists any
# devices with critically low battery. Remove those devices from testing,
# allowing build to continue with good devices.
return 2
if not devices:
return 1
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--out-dir',
help='Directory where the device path is stored',
default=os.path.join(constants.DIR_SOURCE_ROOT, 'out'))
parser.add_argument('--no-provisioning-check',
action='store_true',
help='Will not check if devices are provisioned '
'properly.')
parser.add_argument('--device-status-dashboard',
action='store_true',
help='Output device status data for dashboard.')
parser.add_argument('--restart-usb',
action='store_true',
help='DEPRECATED. '
'This script now always tries to reset USB.')
parser.add_argument('--json-output',
help='Output JSON information into a specified file.')
parser.add_argument('--blacklist-file', help='Device blacklist JSON file.')
parser.add_argument('-v',
'--verbose',
action='count',
default=1,
help='Log more information.')
args = parser.parse_args()
run_tests_helper.SetLogLevel(args.verbose)
if args.blacklist_file:
blacklist = device_blacklist.Blacklist(args.blacklist_file)
else:
# TODO(jbudorick): Remove this once bots pass the blacklist file.
blacklist = device_blacklist.Blacklist(device_blacklist.BLACKLIST_JSON)
devices = RecoverDevices(blacklist, args.out_dir)
types, builds, batteries, errors, devices_ok, json_data = ([], [], [], [],
[], [])
if devices:
types, builds, batteries, errors, devices_ok, json_data = (zip(
*[DeviceInfo(dev, args) for dev in devices]))
# Write device info to file for buildbot info display.
if os.path.exists('/home/chrome-bot'):
with open('/home/chrome-bot/.adb_device_info', 'w') as f:
for device in json_data:
try:
f.write('%s %s %s %.1fC %s%%\n' %
(device['serial'], device['type'], device['build'],
float(device['battery']['temperature']) / 10,
device['battery']['level']))
except Exception:
pass
err_msg = CheckForMissingDevices(args, devices) or []
unique_types = list(set(types))
unique_builds = list(set(builds))
bb_annotations.PrintMsg('Online devices: %d. Device types %s, builds %s' %
(len(devices), unique_types, unique_builds))
for j in json_data:
logging.info('Device %s (%s)', j.get('serial'), j.get('type'))
logging.info(' Build: %s (%s)', j.get('build'), j.get('build_detail'))
logging.info(' Current Battery Service state:')
for k, v in j.get('battery', {}).iteritems():
logging.info(' %s: %s', k, v)
logging.info(' IMEI slice: %s', j.get('imei_slice'))
logging.info(' WiFi IP: %s', j.get('wifi_ip'))
for dev, dev_errors in zip(devices, errors):
if dev_errors:
err_msg += ['%s errors:' % str(dev)]
err_msg += [' %s' % error for error in dev_errors]
if err_msg:
bb_annotations.PrintWarning()
for e in err_msg:
logging.error(e)
from_address = '*****@*****.**'
to_addresses = ['*****@*****.**']
bot_name = os.environ.get('BUILDBOT_BUILDERNAME')
slave_name = os.environ.get('BUILDBOT_SLAVENAME')
subject = 'Device status check errors on %s, %s.' % (slave_name,
bot_name)
SendEmail(from_address, to_addresses, [], subject, '\n'.join(err_msg))
if args.device_status_dashboard:
offline_devices = [
device_utils.DeviceUtils(a)
for a in adb_wrapper.AdbWrapper.Devices(desired_state=None)
if a.GetState() == 'offline'
]
perf_tests_results_helper.PrintPerfResult('BotDevices',
'OnlineDevices',
[len(devices)], 'devices')
perf_tests_results_helper.PrintPerfResult('BotDevices',
'OfflineDevices',
[len(offline_devices)],
'devices', 'unimportant')
for dev, battery in zip(devices, batteries):
perf_tests_results_helper.PrintPerfResult('DeviceBattery',
str(dev), [battery], '%',
'unimportant')
if args.json_output:
with open(args.json_output, 'wb') as f:
f.write(json.dumps(json_data, indent=4))
num_failed_devs = 0
for device_ok, device in zip(devices_ok, devices):
if not device_ok:
logging.warning('Blacklisting %s', str(device))
blacklist.Extend([str(device)])
num_failed_devs += 1
if num_failed_devs == len(devices):
return 2
if not devices:
return 1
def main():
parser = optparse.OptionParser()
parser.add_option('',
'--out-dir',
help='Directory where the device path is stored',
default=os.path.join(constants.DIR_SOURCE_ROOT, 'out'))
parser.add_option(
'--no-provisioning-check',
help='Will not check if devices are provisioned properly.')
parser.add_option('--device-status-dashboard',
help='Output device status data for dashboard.')
options, args = parser.parse_args()
if args:
parser.error('Unknown options %s' % args)
devices = android_commands.GetAttachedDevices()
# TODO(navabi): Test to make sure this fails and then fix call
offline_devices = android_commands.GetAttachedDevices(hardware=False,
emulator=False,
offline=True)
types, builds, batteries, reports, errors = [], [], [], [], []
fail_step_lst = []
if devices:
types, builds, batteries, reports, errors, fail_step_lst = (zip(
*[DeviceInfo(dev, options) for dev in devices]))
err_msg = CheckForMissingDevices(options, devices) or []
unique_types = list(set(types))
unique_builds = list(set(builds))
bb_annotations.PrintMsg('Online devices: %d. Device types %s, builds %s' %
(len(devices), unique_types, unique_builds))
print '\n'.join(reports)
for serial, dev_errors in zip(devices, errors):
if dev_errors:
err_msg += ['%s errors:' % serial]
err_msg += [' %s' % error for error in dev_errors]
if err_msg:
bb_annotations.PrintWarning()
msg = '\n'.join(err_msg)
print msg
SendDeviceStatusAlert(msg)
if options.device_status_dashboard:
perf_tests_helper.PrintPerfResult('BotDevices', 'OnlineDevices',
[len(devices)], 'devices')
perf_tests_helper.PrintPerfResult('BotDevices', 'OfflineDevices',
[len(offline_devices)], 'devices',
'unimportant')
for serial, battery in zip(devices, batteries):
perf_tests_helper.PrintPerfResult('DeviceBattery', serial,
[battery], '%', 'unimportant')
if False in fail_step_lst:
# TODO(navabi): Build fails on device status check step if there exists any
# devices with critically low battery or install speed. Remove those devices
# from testing, allowing build to continue with good devices.
return 1
if not devices:
return 1
def main():
parser = optparse.OptionParser()
parser.add_option('',
'--out-dir',
help='Directory where the device path is stored',
default=os.path.join(constants.DIR_SOURCE_ROOT, 'out'))
parser.add_option(
'--no-provisioning-check',
action='store_true',
help='Will not check if devices are provisioned properly.')
parser.add_option('--device-status-dashboard',
action='store_true',
help='Output device status data for dashboard.')
parser.add_option('--restart-usb',
action='store_true',
help='Restart USB ports before running device check.')
options, args = parser.parse_args()
if args:
parser.error('Unknown options %s' % args)
# Remove the last build's "bad devices" before checking device statuses.
device_blacklist.ResetBlacklist()
if options.restart_usb:
expected_devices = GetLastDevices(os.path.abspath(options.out_dir))
devices = android_commands.GetAttachedDevices()
# Only restart usb if devices are missing.
if set(expected_devices) != set(devices):
KillAllAdb()
retries = 5
usb_restarted = True
if not RestartUsb():
usb_restarted = False
bb_annotations.PrintWarning()
print 'USB reset stage failed, wait for any device to come back.'
while retries:
time.sleep(1)
devices = android_commands.GetAttachedDevices()
if set(expected_devices) == set(devices):
# All devices are online, keep going.
break
if not usb_restarted and devices:
# The USB wasn't restarted, but there's at least one device online.
# No point in trying to wait for all devices.
break
retries -= 1
devices = android_commands.GetAttachedDevices()
# TODO(navabi): Test to make sure this fails and then fix call
offline_devices = android_commands.GetAttachedDevices(hardware=False,
emulator=False,
offline=True)
types, builds, batteries, reports, errors = [], [], [], [], []
fail_step_lst = []
if devices:
types, builds, batteries, reports, errors, fail_step_lst = (zip(
*[DeviceInfo(dev, options) for dev in devices]))
err_msg = CheckForMissingDevices(options, devices) or []
unique_types = list(set(types))
unique_builds = list(set(builds))
bb_annotations.PrintMsg('Online devices: %d. Device types %s, builds %s' %
(len(devices), unique_types, unique_builds))
print '\n'.join(reports)
for serial, dev_errors in zip(devices, errors):
if dev_errors:
err_msg += ['%s errors:' % serial]
err_msg += [' %s' % error for error in dev_errors]
if err_msg:
bb_annotations.PrintWarning()
msg = '\n'.join(err_msg)
print msg
SendDeviceStatusAlert(msg)
if options.device_status_dashboard:
perf_tests_results_helper.PrintPerfResult('BotDevices',
'OnlineDevices',
[len(devices)], 'devices')
perf_tests_results_helper.PrintPerfResult('BotDevices',
'OfflineDevices',
[len(offline_devices)],
'devices', 'unimportant')
for serial, battery in zip(devices, batteries):
perf_tests_results_helper.PrintPerfResult('DeviceBattery', serial,
[battery], '%',
'unimportant')
if False in fail_step_lst:
# TODO(navabi): Build fails on device status check step if there exists any
# devices with critically low battery. Remove those devices from testing,
# allowing build to continue with good devices.
return 1
if not devices:
return 1
