def ProvisionDevices(options):
is_perf = 'perf' in os.environ.get('BUILDBOT_BUILDERNAME', '').lower()
# TODO(jbudorick): Parallelize provisioning of all attached devices after
# switching from AndroidCommands.
if options.device is not None:
devices = [options.device]
else:
devices = android_commands.GetAttachedDevices()
# Wipe devices (unless --skip-wipe was specified)
if not options.skip_wipe:
WipeDevicesIfPossible(devices)
bad_devices = []
# Provision devices
for device_serial in devices:
try:
ProvisionDevice(device_serial, is_perf, options.disable_location)
except errors.WaitForResponseTimedOutError:
logging.info(
'Timed out waiting for device %s. Adding to blacklist.',
device_serial)
bad_devices.append(device_serial)
# Device black list is reset by bb_device_status_check.py per build.
device_blacklist.ExtendBlacklist([device_serial])
devices = [device for device in devices if device not in bad_devices]
# If there are no good devices
if not devices:
raise device_errors.NoDevicesError
try:
device_utils.DeviceUtils.parallel(devices).Reboot(True)
except errors.DeviceUnresponsiveError:
pass
bad_devices = []
for device_serial in devices:
device = device_utils.DeviceUtils(device_serial)
try:
device.WaitUntilFullyBooted(timeout=90)
(_, prop
) = device.old_interface.GetShellCommandStatusAndOutput('getprop')
for p in prop:
print p
except errors.WaitForResponseTimedOutError:
logging.info(
'Timed out waiting for device %s. Adding to blacklist.',
device_serial)
bad_devices.append(device_serial)
# Device black list is reset by bb_device_status_check.py per build.
device_blacklist.ExtendBlacklist([device_serial])
devices = [device for device in devices if device not in bad_devices]
# If there are no good devices
if not devices:
raise device_errors.NoDevicesError
if options.auto_reconnect:
PushAndLaunchAdbReboot(devices, options.target)
def ProvisionDevice(device, options):
try:
if not options.skip_wipe:
WipeDeviceIfPossible(device, options.reboot_timeout)
try:
device.EnableRoot()
except device_errors.CommandFailedError as e:
logging.warning(str(e))
_ConfigureLocalProperties(device, options.enable_java_debug)
device_settings.ConfigureContentSettings(
device, device_settings.DETERMINISTIC_DEVICE_SETTINGS)
if options.disable_location:
device_settings.ConfigureContentSettings(
device, device_settings.DISABLE_LOCATION_SETTINGS)
else:
device_settings.ConfigureContentSettings(
device, device_settings.ENABLE_LOCATION_SETTINGS)
device_settings.SetLockScreenSettings(device)
if options.disable_network:
device_settings.ConfigureContentSettings(
device, device_settings.NETWORK_DISABLED_SETTINGS)
if options.wait_for_battery:
try:
battery_info = device.old_interface.GetBatteryInfo()
except Exception as e:
battery_info = {}
logging.error('Unable to obtain battery info for %s, %s',
str(device), e)
while int(battery_info.get('level', 100)) < options.min_battery_level:
if not device.old_interface.IsDeviceCharging():
if device.old_interface.CanControlUsbCharging():
device.old_interface.EnableUsbCharging()
else:
logging.error('Device is not charging')
break
logging.info('Waiting for device to charge. Current level=%s',
battery_info.get('level', 0))
time.sleep(60)
battery_info = device.old_interface.GetBatteryInfo()
if not options.skip_wipe:
device.Reboot(True, timeout=options.reboot_timeout, retries=0)
device.RunShellCommand('date -s %s' % time.strftime('%Y%m%d.%H%M%S',
time.gmtime()),
as_root=True)
props = device.RunShellCommand('getprop')
for prop in props:
logging.info(' %s' % prop)
if options.auto_reconnect:
PushAndLaunchAdbReboot(device, options.target)
except (errors.WaitForResponseTimedOutError,
device_errors.CommandTimeoutError):
logging.info('Timed out waiting for device %s. Adding to blacklist.',
str(device))
# Device black list is reset by bb_device_status_check.py per build.
device_blacklist.ExtendBlacklist([str(device)])
except device_errors.CommandFailedError:
logging.exception('Failed to provision device %s. Adding to blacklist.',
str(device))
device_blacklist.ExtendBlacklist([str(device)])
def ProvisionDevice(device, options):
if options.reboot_timeout:
reboot_timeout = options.reboot_timeout
elif (device.build_version_sdk >=
constants.ANDROID_SDK_VERSION_CODES.LOLLIPOP):
reboot_timeout = _DEFAULT_TIMEOUTS.LOLLIPOP
else:
reboot_timeout = _DEFAULT_TIMEOUTS.PRE_LOLLIPOP
try:
if not options.skip_wipe:
WipeDeviceIfPossible(device, reboot_timeout, options)
try:
device.EnableRoot()
except device_errors.CommandFailedError as e:
logging.warning(str(e))
_ConfigureLocalProperties(device, options.enable_java_debug)
device_settings.ConfigureContentSettings(
device, device_settings.DETERMINISTIC_DEVICE_SETTINGS)
if options.disable_location:
device_settings.ConfigureContentSettings(
device, device_settings.DISABLE_LOCATION_SETTINGS)
else:
device_settings.ConfigureContentSettings(
device, device_settings.ENABLE_LOCATION_SETTINGS)
device_settings.SetLockScreenSettings(device)
if options.disable_network:
device_settings.ConfigureContentSettings(
device, device_settings.NETWORK_DISABLED_SETTINGS)
if options.min_battery_level is not None:
try:
battery = battery_utils.BatteryUtils(device)
battery.ChargeDeviceToLevel(options.min_battery_level)
except device_errors.CommandFailedError as e:
logging.exception(
'Unable to charge device to specified level.')
if not options.skip_wipe:
device.Reboot(True, timeout=reboot_timeout, retries=0)
device.RunShellCommand('date -s %s' %
time.strftime('%Y%m%d.%H%M%S', time.gmtime()),
as_root=True)
props = device.RunShellCommand('getprop')
for prop in props:
logging.info(' %s' % prop)
if options.auto_reconnect:
PushAndLaunchAdbReboot(device, options.target)
except (errors.WaitForResponseTimedOutError,
device_errors.CommandTimeoutError):
logging.info('Timed out waiting for device %s. Adding to blacklist.',
str(device))
# Device black list is reset by bb_device_status_check.py per build.
device_blacklist.ExtendBlacklist([str(device)])
except device_errors.CommandFailedError:
logging.exception(
'Failed to provision device %s. Adding to blacklist.', str(device))
device_blacklist.ExtendBlacklist([str(device)])
def blacklisting_install(device):
try:
device.Install(apk, reinstall=args.keep_data)
except device_errors.CommandFailedError:
logging.exception('Failed to install %s', args.apk)
device_blacklist.ExtendBlacklist([str(device)])
logging.warning('Blacklisting %s', str(device))
except device_errors.CommandTimeoutError:
logging.exception('Timed out while installing %s', args.apk)
device_blacklist.ExtendBlacklist([str(device)])
logging.warning('Blacklisting %s', str(device))
def ProvisionDevice(device, options):
if options.reboot_timeout:
reboot_timeout = options.reboot_timeout
elif (device.build_version_sdk >=
constants.ANDROID_SDK_VERSION_CODES.LOLLIPOP):
reboot_timeout = _DEFAULT_TIMEOUTS.LOLLIPOP
else:
reboot_timeout = _DEFAULT_TIMEOUTS.PRE_LOLLIPOP
def should_run_phase(phase_name):
return not options.phases or phase_name in options.phases
def run_phase(phase_func, reboot=True):
try:
device.WaitUntilFullyBooted(timeout=reboot_timeout, retries=0)
except device_errors.CommandTimeoutError:
logging.error('Device did not finish booting. Will try to reboot.')
device.Reboot(timeout=reboot_timeout)
phase_func(device, options)
if reboot:
device.Reboot(False, retries=0)
device.adb.WaitForDevice()
try:
if should_run_phase(_PHASES.WIPE):
run_phase(WipeDevice)
if should_run_phase(_PHASES.PROPERTIES):
run_phase(SetProperties)
if should_run_phase(_PHASES.FINISH):
run_phase(FinishProvisioning, reboot=False)
except (errors.WaitForResponseTimedOutError,
device_errors.CommandTimeoutError):
logging.exception(
'Timed out waiting for device %s. Adding to blacklist.',
str(device))
device_blacklist.ExtendBlacklist([str(device)])
except device_errors.CommandFailedError:
logging.exception(
'Failed to provision device %s. Adding to blacklist.', str(device))
device_blacklist.ExtendBlacklist([str(device)])
def main():
parser = optparse.OptionParser()
parser.add_option('',
'--out-dir',
help='Directory where the device path is stored',
default=os.path.join(constants.DIR_SOURCE_ROOT, 'out'))
parser.add_option(
'--no-provisioning-check',
action='store_true',
help='Will not check if devices are provisioned properly.')
parser.add_option('--device-status-dashboard',
action='store_true',
help='Output device status data for dashboard.')
parser.add_option('--restart-usb',
action='store_true',
help='Restart USB ports before running device check.')
parser.add_option('--json-output',
help='Output JSON information into a specified file.')
parser.add_option('-v',
'--verbose',
action='count',
default=1,
help='Log more information.')
options, args = parser.parse_args()
if args:
parser.error('Unknown options %s' % args)
run_tests_helper.SetLogLevel(options.verbose)
# Remove the last build's "bad devices" before checking device statuses.
device_blacklist.ResetBlacklist()
try:
expected_devices = device_list.GetPersistentDeviceList(
os.path.join(options.out_dir, device_list.LAST_DEVICES_FILENAME))
except IOError:
expected_devices = []
devices = device_utils.DeviceUtils.HealthyDevices()
device_serials = [d.adb.GetDeviceSerial() for d in devices]
# Only restart usb if devices are missing.
if set(expected_devices) != set(device_serials):
logging.warning('expected_devices: %s', expected_devices)
logging.warning('devices: %s', device_serials)
KillAllAdb()
retries = 5
usb_restarted = True
if options.restart_usb:
if not RestartUsb():
usb_restarted = False
bb_annotations.PrintWarning()
logging.error('USB reset stage failed, '
'wait for any device to come back.')
while retries:
logging.info('retry adb devices...')
time.sleep(1)
devices = device_utils.DeviceUtils.HealthyDevices()
device_serials = [d.adb.GetDeviceSerial() for d in devices]
if set(expected_devices) == set(device_serials):
# All devices are online, keep going.
break
if not usb_restarted and devices:
# The USB wasn't restarted, but there's at least one device online.
# No point in trying to wait for all devices.
break
retries -= 1
types, builds, batteries, errors, devices_ok, json_data = ([], [], [], [],
[], [])
if devices:
types, builds, batteries, errors, devices_ok, json_data = (zip(
*[DeviceInfo(dev, options) for dev in devices]))
# Write device info to file for buildbot info display.
if os.path.exists('/home/chrome-bot'):
with open('/home/chrome-bot/.adb_device_info', 'w') as f:
for device in json_data:
try:
f.write('%s %s %s %.1fC %s%%\n' %
(device['serial'], device['type'], device['build'],
float(device['battery']['temperature']) / 10,
device['battery']['level']))
except Exception:
pass
err_msg = CheckForMissingDevices(options, devices) or []
unique_types = list(set(types))
unique_builds = list(set(builds))
bb_annotations.PrintMsg('Online devices: %d. Device types %s, builds %s' %
(len(devices), unique_types, unique_builds))
for j in json_data:
logging.info('Device %s (%s)', j.get('serial'), j.get('type'))
logging.info(' Build: %s (%s)', j.get('build'), j.get('build_detail'))
logging.info(' Current Battery Service state:')
for k, v in j.get('battery', {}).iteritems():
logging.info(' %s: %s', k, v)
logging.info(' IMEI slice: %s', j.get('imei_slice'))
logging.info(' WiFi IP: %s', j.get('wifi_ip'))
for dev, dev_errors in zip(devices, errors):
if dev_errors:
err_msg += ['%s errors:' % str(dev)]
err_msg += [' %s' % error for error in dev_errors]
if err_msg:
bb_annotations.PrintWarning()
for e in err_msg:
logging.error(e)
from_address = '*****@*****.**'
to_addresses = ['*****@*****.**']
bot_name = os.environ.get('BUILDBOT_BUILDERNAME')
slave_name = os.environ.get('BUILDBOT_SLAVENAME')
subject = 'Device status check errors on %s, %s.' % (slave_name,
bot_name)
SendEmail(from_address, to_addresses, [], subject, '\n'.join(err_msg))
if options.device_status_dashboard:
offline_devices = [
device_utils.DeviceUtils(a)
for a in adb_wrapper.AdbWrapper.Devices(is_ready=False)
if a.GetState() == 'offline'
]
perf_tests_results_helper.PrintPerfResult('BotDevices',
'OnlineDevices',
[len(devices)], 'devices')
perf_tests_results_helper.PrintPerfResult('BotDevices',
'OfflineDevices',
[len(offline_devices)],
'devices', 'unimportant')
for dev, battery in zip(devices, batteries):
perf_tests_results_helper.PrintPerfResult('DeviceBattery',
str(dev), [battery], '%',
'unimportant')
if options.json_output:
with open(options.json_output, 'wb') as f:
f.write(json.dumps(json_data, indent=4))
num_failed_devs = 0
for device_ok, device in zip(devices_ok, devices):
if not device_ok:
logging.warning('Blacklisting %s', str(device))
device_blacklist.ExtendBlacklist([str(device)])
num_failed_devs += 1
if num_failed_devs == len(devices):
return 2
if not devices:
return 1
def ProvisionDevice(device, options, is_perf):
try:
if not options.skip_wipe:
WipeDeviceIfPossible(device)
try:
device.EnableRoot()
except device_errors.CommandFailedError as e:
logging.warning(str(e))
_ConfigureLocalProperties(device, is_perf)
device_settings.ConfigureContentSettings(
device, device_settings.DETERMINISTIC_DEVICE_SETTINGS)
if options.disable_location:
device_settings.ConfigureContentSettings(
device, device_settings.DISABLE_LOCATION_SETTINGS)
else:
device_settings.ConfigureContentSettings(
device, device_settings.ENABLE_LOCATION_SETTINGS)
device_settings.SetLockScreenSettings(device)
if is_perf:
# TODO(tonyg): We eventually want network on. However, currently radios
# can cause perfbots to drain faster than they charge.
device_settings.ConfigureContentSettings(
device, device_settings.NETWORK_DISABLED_SETTINGS)
# Some perf bots run benchmarks with USB charging disabled which leads
# to gradual draining of the battery. We must wait for a full charge
# before starting a run in order to keep the devices online.
try:
battery_info = device.old_interface.GetBatteryInfo()
except Exception as e:
battery_info = {}
logging.error('Unable to obtain battery info for %s, %s',
str(device), e)
while int(battery_info.get('level',
100)) < options.min_battery_level:
if not device.old_interface.IsDeviceCharging():
if device.old_interface.CanControlUsbCharging():
device.old_interface.EnableUsbCharging()
else:
logging.error('Device is not charging')
break
logging.info('Waiting for device to charge. Current level=%s',
battery_info.get('level', 0))
time.sleep(60)
battery_info = device.old_interface.GetBatteryInfo()
if not options.skip_wipe:
# TODO(jbudorick): Tune the timeout per OS version.
device.Reboot(True, timeout=600, retries=0)
device.RunShellCommand('date -s %s' %
time.strftime('%Y%m%d.%H%M%S', time.gmtime()),
as_root=True)
props = device.RunShellCommand('getprop')
for prop in props:
logging.info(' %s' % prop)
if options.auto_reconnect:
PushAndLaunchAdbReboot(device, options.target)
except (errors.WaitForResponseTimedOutError,
device_errors.CommandTimeoutError):
logging.info('Timed out waiting for device %s. Adding to blacklist.',
str(device))
# Device black list is reset by bb_device_status_check.py per build.
device_blacklist.ExtendBlacklist([str(device)])
except device_errors.CommandFailedError:
logging.exception(
'Failed to provision device %s. Adding to blacklist.', str(device))
device_blacklist.ExtendBlacklist([str(device)])
def main():
parser = optparse.OptionParser()
parser.add_option('', '--out-dir',
help='Directory where the device path is stored',
default=os.path.join(constants.DIR_SOURCE_ROOT, 'out'))
parser.add_option('--no-provisioning-check', action='store_true',
help='Will not check if devices are provisioned properly.')
parser.add_option('--device-status-dashboard', action='store_true',
help='Output device status data for dashboard.')
parser.add_option('--restart-usb', action='store_true',
help='Restart USB ports before running device check.')
parser.add_option('--json-output',
help='Output JSON information into a specified file.')
options, args = parser.parse_args()
if args:
parser.error('Unknown options %s' % args)
# Remove the last build's "bad devices" before checking device statuses.
device_blacklist.ResetBlacklist()
try:
expected_devices = device_list.GetPersistentDeviceList(
os.path.join(options.out_dir, device_list.LAST_DEVICES_FILENAME))
except IOError:
expected_devices = []
devices = android_commands.GetAttachedDevices()
# Only restart usb if devices are missing.
if set(expected_devices) != set(devices):
print 'expected_devices: %s, devices: %s' % (expected_devices, devices)
KillAllAdb()
retries = 5
usb_restarted = True
if options.restart_usb:
if not RestartUsb():
usb_restarted = False
bb_annotations.PrintWarning()
print 'USB reset stage failed, wait for any device to come back.'
while retries:
print 'retry adb devices...'
time.sleep(1)
devices = android_commands.GetAttachedDevices()
if set(expected_devices) == set(devices):
# All devices are online, keep going.
break
if not usb_restarted and devices:
# The USB wasn't restarted, but there's at least one device online.
# No point in trying to wait for all devices.
break
retries -= 1
# TODO(navabi): Test to make sure this fails and then fix call
offline_devices = android_commands.GetAttachedDevices(
hardware=False, emulator=False, offline=True)
types, builds, batteries, reports, errors, json_data = [], [], [], [], [], []
fail_step_lst = []
if devices:
types, builds, batteries, reports, errors, fail_step_lst, json_data = (
zip(*[DeviceInfo(dev, options) for dev in devices]))
err_msg = CheckForMissingDevices(options, devices) or []
unique_types = list(set(types))
unique_builds = list(set(builds))
bb_annotations.PrintMsg('Online devices: %d. Device types %s, builds %s'
% (len(devices), unique_types, unique_builds))
print '\n'.join(reports)
for serial, dev_errors in zip(devices, errors):
if dev_errors:
err_msg += ['%s errors:' % serial]
err_msg += [' %s' % error for error in dev_errors]
if err_msg:
bb_annotations.PrintWarning()
msg = '\n'.join(err_msg)
print msg
from_address = '*****@*****.**'
to_addresses = ['*****@*****.**']
bot_name = os.environ.get('BUILDBOT_BUILDERNAME')
slave_name = os.environ.get('BUILDBOT_SLAVENAME')
subject = 'Device status check errors on %s, %s.' % (slave_name, bot_name)
SendEmail(from_address, to_addresses, [], subject, msg)
if options.device_status_dashboard:
perf_tests_results_helper.PrintPerfResult('BotDevices', 'OnlineDevices',
[len(devices)], 'devices')
perf_tests_results_helper.PrintPerfResult('BotDevices', 'OfflineDevices',
[len(offline_devices)], 'devices',
'unimportant')
for serial, battery in zip(devices, batteries):
perf_tests_results_helper.PrintPerfResult('DeviceBattery', serial,
[battery], '%',
'unimportant')
if options.json_output:
with open(options.json_output, 'wb') as f:
f.write(json.dumps(json_data, indent=4))
num_failed_devs = 0
for fail_status, device in zip(fail_step_lst, devices):
if not fail_status:
device_blacklist.ExtendBlacklist([str(device)])
num_failed_devs += 1
if num_failed_devs == len(devices):
return 2
if not devices:
return 1