def ProvisionDevices(options): is_perf = 'perf' in os.environ.get('BUILDBOT_BUILDERNAME', '').lower() # TODO(jbudorick): Parallelize provisioning of all attached devices after # switching from AndroidCommands. if options.device is not None: devices = [options.device] else: devices = android_commands.GetAttachedDevices() # Wipe devices (unless --skip-wipe was specified) if not options.skip_wipe: WipeDevicesIfPossible(devices) bad_devices = [] # Provision devices for device_serial in devices: try: ProvisionDevice(device_serial, is_perf, options.disable_location) except errors.WaitForResponseTimedOutError: logging.info( 'Timed out waiting for device %s. Adding to blacklist.', device_serial) bad_devices.append(device_serial) # Device black list is reset by bb_device_status_check.py per build. device_blacklist.ExtendBlacklist([device_serial]) devices = [device for device in devices if device not in bad_devices] # If there are no good devices if not devices: raise device_errors.NoDevicesError try: device_utils.DeviceUtils.parallel(devices).Reboot(True) except errors.DeviceUnresponsiveError: pass bad_devices = [] for device_serial in devices: device = device_utils.DeviceUtils(device_serial) try: device.WaitUntilFullyBooted(timeout=90) (_, prop ) = device.old_interface.GetShellCommandStatusAndOutput('getprop') for p in prop: print p except errors.WaitForResponseTimedOutError: logging.info( 'Timed out waiting for device %s. Adding to blacklist.', device_serial) bad_devices.append(device_serial) # Device black list is reset by bb_device_status_check.py per build. device_blacklist.ExtendBlacklist([device_serial]) devices = [device for device in devices if device not in bad_devices] # If there are no good devices if not devices: raise device_errors.NoDevicesError if options.auto_reconnect: PushAndLaunchAdbReboot(devices, options.target)
def ProvisionDevice(device, options): try: if not options.skip_wipe: WipeDeviceIfPossible(device, options.reboot_timeout) try: device.EnableRoot() except device_errors.CommandFailedError as e: logging.warning(str(e)) _ConfigureLocalProperties(device, options.enable_java_debug) device_settings.ConfigureContentSettings( device, device_settings.DETERMINISTIC_DEVICE_SETTINGS) if options.disable_location: device_settings.ConfigureContentSettings( device, device_settings.DISABLE_LOCATION_SETTINGS) else: device_settings.ConfigureContentSettings( device, device_settings.ENABLE_LOCATION_SETTINGS) device_settings.SetLockScreenSettings(device) if options.disable_network: device_settings.ConfigureContentSettings( device, device_settings.NETWORK_DISABLED_SETTINGS) if options.wait_for_battery: try: battery_info = device.old_interface.GetBatteryInfo() except Exception as e: battery_info = {} logging.error('Unable to obtain battery info for %s, %s', str(device), e) while int(battery_info.get('level', 100)) < options.min_battery_level: if not device.old_interface.IsDeviceCharging(): if device.old_interface.CanControlUsbCharging(): device.old_interface.EnableUsbCharging() else: logging.error('Device is not charging') break logging.info('Waiting for device to charge. Current level=%s', battery_info.get('level', 0)) time.sleep(60) battery_info = device.old_interface.GetBatteryInfo() if not options.skip_wipe: device.Reboot(True, timeout=options.reboot_timeout, retries=0) device.RunShellCommand('date -s %s' % time.strftime('%Y%m%d.%H%M%S', time.gmtime()), as_root=True) props = device.RunShellCommand('getprop') for prop in props: logging.info(' %s' % prop) if options.auto_reconnect: PushAndLaunchAdbReboot(device, options.target) except (errors.WaitForResponseTimedOutError, device_errors.CommandTimeoutError): logging.info('Timed out waiting for device %s. Adding to blacklist.', str(device)) # Device black list is reset by bb_device_status_check.py per build. device_blacklist.ExtendBlacklist([str(device)]) except device_errors.CommandFailedError: logging.exception('Failed to provision device %s. Adding to blacklist.', str(device)) device_blacklist.ExtendBlacklist([str(device)])
def ProvisionDevice(device, options): if options.reboot_timeout: reboot_timeout = options.reboot_timeout elif (device.build_version_sdk >= constants.ANDROID_SDK_VERSION_CODES.LOLLIPOP): reboot_timeout = _DEFAULT_TIMEOUTS.LOLLIPOP else: reboot_timeout = _DEFAULT_TIMEOUTS.PRE_LOLLIPOP try: if not options.skip_wipe: WipeDeviceIfPossible(device, reboot_timeout, options) try: device.EnableRoot() except device_errors.CommandFailedError as e: logging.warning(str(e)) _ConfigureLocalProperties(device, options.enable_java_debug) device_settings.ConfigureContentSettings( device, device_settings.DETERMINISTIC_DEVICE_SETTINGS) if options.disable_location: device_settings.ConfigureContentSettings( device, device_settings.DISABLE_LOCATION_SETTINGS) else: device_settings.ConfigureContentSettings( device, device_settings.ENABLE_LOCATION_SETTINGS) device_settings.SetLockScreenSettings(device) if options.disable_network: device_settings.ConfigureContentSettings( device, device_settings.NETWORK_DISABLED_SETTINGS) if options.min_battery_level is not None: try: battery = battery_utils.BatteryUtils(device) battery.ChargeDeviceToLevel(options.min_battery_level) except device_errors.CommandFailedError as e: logging.exception( 'Unable to charge device to specified level.') if not options.skip_wipe: device.Reboot(True, timeout=reboot_timeout, retries=0) device.RunShellCommand('date -s %s' % time.strftime('%Y%m%d.%H%M%S', time.gmtime()), as_root=True) props = device.RunShellCommand('getprop') for prop in props: logging.info(' %s' % prop) if options.auto_reconnect: PushAndLaunchAdbReboot(device, options.target) except (errors.WaitForResponseTimedOutError, device_errors.CommandTimeoutError): logging.info('Timed out waiting for device %s. Adding to blacklist.', str(device)) # Device black list is reset by bb_device_status_check.py per build. device_blacklist.ExtendBlacklist([str(device)]) except device_errors.CommandFailedError: logging.exception( 'Failed to provision device %s. Adding to blacklist.', str(device)) device_blacklist.ExtendBlacklist([str(device)])
def blacklisting_install(device): try: device.Install(apk, reinstall=args.keep_data) except device_errors.CommandFailedError: logging.exception('Failed to install %s', args.apk) device_blacklist.ExtendBlacklist([str(device)]) logging.warning('Blacklisting %s', str(device)) except device_errors.CommandTimeoutError: logging.exception('Timed out while installing %s', args.apk) device_blacklist.ExtendBlacklist([str(device)]) logging.warning('Blacklisting %s', str(device))
def ProvisionDevice(device, options): if options.reboot_timeout: reboot_timeout = options.reboot_timeout elif (device.build_version_sdk >= constants.ANDROID_SDK_VERSION_CODES.LOLLIPOP): reboot_timeout = _DEFAULT_TIMEOUTS.LOLLIPOP else: reboot_timeout = _DEFAULT_TIMEOUTS.PRE_LOLLIPOP def should_run_phase(phase_name): return not options.phases or phase_name in options.phases def run_phase(phase_func, reboot=True): try: device.WaitUntilFullyBooted(timeout=reboot_timeout, retries=0) except device_errors.CommandTimeoutError: logging.error('Device did not finish booting. Will try to reboot.') device.Reboot(timeout=reboot_timeout) phase_func(device, options) if reboot: device.Reboot(False, retries=0) device.adb.WaitForDevice() try: if should_run_phase(_PHASES.WIPE): run_phase(WipeDevice) if should_run_phase(_PHASES.PROPERTIES): run_phase(SetProperties) if should_run_phase(_PHASES.FINISH): run_phase(FinishProvisioning, reboot=False) except (errors.WaitForResponseTimedOutError, device_errors.CommandTimeoutError): logging.exception( 'Timed out waiting for device %s. Adding to blacklist.', str(device)) device_blacklist.ExtendBlacklist([str(device)]) except device_errors.CommandFailedError: logging.exception( 'Failed to provision device %s. Adding to blacklist.', str(device)) device_blacklist.ExtendBlacklist([str(device)])
def main(): parser = optparse.OptionParser() parser.add_option('', '--out-dir', help='Directory where the device path is stored', default=os.path.join(constants.DIR_SOURCE_ROOT, 'out')) parser.add_option( '--no-provisioning-check', action='store_true', help='Will not check if devices are provisioned properly.') parser.add_option('--device-status-dashboard', action='store_true', help='Output device status data for dashboard.') parser.add_option('--restart-usb', action='store_true', help='Restart USB ports before running device check.') parser.add_option('--json-output', help='Output JSON information into a specified file.') parser.add_option('-v', '--verbose', action='count', default=1, help='Log more information.') options, args = parser.parse_args() if args: parser.error('Unknown options %s' % args) run_tests_helper.SetLogLevel(options.verbose) # Remove the last build's "bad devices" before checking device statuses. device_blacklist.ResetBlacklist() try: expected_devices = device_list.GetPersistentDeviceList( os.path.join(options.out_dir, device_list.LAST_DEVICES_FILENAME)) except IOError: expected_devices = [] devices = device_utils.DeviceUtils.HealthyDevices() device_serials = [d.adb.GetDeviceSerial() for d in devices] # Only restart usb if devices are missing. if set(expected_devices) != set(device_serials): logging.warning('expected_devices: %s', expected_devices) logging.warning('devices: %s', device_serials) KillAllAdb() retries = 5 usb_restarted = True if options.restart_usb: if not RestartUsb(): usb_restarted = False bb_annotations.PrintWarning() logging.error('USB reset stage failed, ' 'wait for any device to come back.') while retries: logging.info('retry adb devices...') time.sleep(1) devices = device_utils.DeviceUtils.HealthyDevices() device_serials = [d.adb.GetDeviceSerial() for d in devices] if set(expected_devices) == set(device_serials): # All devices are online, keep going. break if not usb_restarted and devices: # The USB wasn't restarted, but there's at least one device online. # No point in trying to wait for all devices. break retries -= 1 types, builds, batteries, errors, devices_ok, json_data = ([], [], [], [], [], []) if devices: types, builds, batteries, errors, devices_ok, json_data = (zip( *[DeviceInfo(dev, options) for dev in devices])) # Write device info to file for buildbot info display. if os.path.exists('/home/chrome-bot'): with open('/home/chrome-bot/.adb_device_info', 'w') as f: for device in json_data: try: f.write('%s %s %s %.1fC %s%%\n' % (device['serial'], device['type'], device['build'], float(device['battery']['temperature']) / 10, device['battery']['level'])) except Exception: pass err_msg = CheckForMissingDevices(options, devices) or [] unique_types = list(set(types)) unique_builds = list(set(builds)) bb_annotations.PrintMsg('Online devices: %d. Device types %s, builds %s' % (len(devices), unique_types, unique_builds)) for j in json_data: logging.info('Device %s (%s)', j.get('serial'), j.get('type')) logging.info(' Build: %s (%s)', j.get('build'), j.get('build_detail')) logging.info(' Current Battery Service state:') for k, v in j.get('battery', {}).iteritems(): logging.info(' %s: %s', k, v) logging.info(' IMEI slice: %s', j.get('imei_slice')) logging.info(' WiFi IP: %s', j.get('wifi_ip')) for dev, dev_errors in zip(devices, errors): if dev_errors: err_msg += ['%s errors:' % str(dev)] err_msg += [' %s' % error for error in dev_errors] if err_msg: bb_annotations.PrintWarning() for e in err_msg: logging.error(e) from_address = '*****@*****.**' to_addresses = ['*****@*****.**'] bot_name = os.environ.get('BUILDBOT_BUILDERNAME') slave_name = os.environ.get('BUILDBOT_SLAVENAME') subject = 'Device status check errors on %s, %s.' % (slave_name, bot_name) SendEmail(from_address, to_addresses, [], subject, '\n'.join(err_msg)) if options.device_status_dashboard: offline_devices = [ device_utils.DeviceUtils(a) for a in adb_wrapper.AdbWrapper.Devices(is_ready=False) if a.GetState() == 'offline' ] perf_tests_results_helper.PrintPerfResult('BotDevices', 'OnlineDevices', [len(devices)], 'devices') perf_tests_results_helper.PrintPerfResult('BotDevices', 'OfflineDevices', [len(offline_devices)], 'devices', 'unimportant') for dev, battery in zip(devices, batteries): perf_tests_results_helper.PrintPerfResult('DeviceBattery', str(dev), [battery], '%', 'unimportant') if options.json_output: with open(options.json_output, 'wb') as f: f.write(json.dumps(json_data, indent=4)) num_failed_devs = 0 for device_ok, device in zip(devices_ok, devices): if not device_ok: logging.warning('Blacklisting %s', str(device)) device_blacklist.ExtendBlacklist([str(device)]) num_failed_devs += 1 if num_failed_devs == len(devices): return 2 if not devices: return 1
def ProvisionDevice(device, options, is_perf): try: if not options.skip_wipe: WipeDeviceIfPossible(device) try: device.EnableRoot() except device_errors.CommandFailedError as e: logging.warning(str(e)) _ConfigureLocalProperties(device, is_perf) device_settings.ConfigureContentSettings( device, device_settings.DETERMINISTIC_DEVICE_SETTINGS) if options.disable_location: device_settings.ConfigureContentSettings( device, device_settings.DISABLE_LOCATION_SETTINGS) else: device_settings.ConfigureContentSettings( device, device_settings.ENABLE_LOCATION_SETTINGS) device_settings.SetLockScreenSettings(device) if is_perf: # TODO(tonyg): We eventually want network on. However, currently radios # can cause perfbots to drain faster than they charge. device_settings.ConfigureContentSettings( device, device_settings.NETWORK_DISABLED_SETTINGS) # Some perf bots run benchmarks with USB charging disabled which leads # to gradual draining of the battery. We must wait for a full charge # before starting a run in order to keep the devices online. try: battery_info = device.old_interface.GetBatteryInfo() except Exception as e: battery_info = {} logging.error('Unable to obtain battery info for %s, %s', str(device), e) while int(battery_info.get('level', 100)) < options.min_battery_level: if not device.old_interface.IsDeviceCharging(): if device.old_interface.CanControlUsbCharging(): device.old_interface.EnableUsbCharging() else: logging.error('Device is not charging') break logging.info('Waiting for device to charge. Current level=%s', battery_info.get('level', 0)) time.sleep(60) battery_info = device.old_interface.GetBatteryInfo() if not options.skip_wipe: # TODO(jbudorick): Tune the timeout per OS version. device.Reboot(True, timeout=600, retries=0) device.RunShellCommand('date -s %s' % time.strftime('%Y%m%d.%H%M%S', time.gmtime()), as_root=True) props = device.RunShellCommand('getprop') for prop in props: logging.info(' %s' % prop) if options.auto_reconnect: PushAndLaunchAdbReboot(device, options.target) except (errors.WaitForResponseTimedOutError, device_errors.CommandTimeoutError): logging.info('Timed out waiting for device %s. Adding to blacklist.', str(device)) # Device black list is reset by bb_device_status_check.py per build. device_blacklist.ExtendBlacklist([str(device)]) except device_errors.CommandFailedError: logging.exception( 'Failed to provision device %s. Adding to blacklist.', str(device)) device_blacklist.ExtendBlacklist([str(device)])
def main(): parser = optparse.OptionParser() parser.add_option('', '--out-dir', help='Directory where the device path is stored', default=os.path.join(constants.DIR_SOURCE_ROOT, 'out')) parser.add_option('--no-provisioning-check', action='store_true', help='Will not check if devices are provisioned properly.') parser.add_option('--device-status-dashboard', action='store_true', help='Output device status data for dashboard.') parser.add_option('--restart-usb', action='store_true', help='Restart USB ports before running device check.') parser.add_option('--json-output', help='Output JSON information into a specified file.') options, args = parser.parse_args() if args: parser.error('Unknown options %s' % args) # Remove the last build's "bad devices" before checking device statuses. device_blacklist.ResetBlacklist() try: expected_devices = device_list.GetPersistentDeviceList( os.path.join(options.out_dir, device_list.LAST_DEVICES_FILENAME)) except IOError: expected_devices = [] devices = android_commands.GetAttachedDevices() # Only restart usb if devices are missing. if set(expected_devices) != set(devices): print 'expected_devices: %s, devices: %s' % (expected_devices, devices) KillAllAdb() retries = 5 usb_restarted = True if options.restart_usb: if not RestartUsb(): usb_restarted = False bb_annotations.PrintWarning() print 'USB reset stage failed, wait for any device to come back.' while retries: print 'retry adb devices...' time.sleep(1) devices = android_commands.GetAttachedDevices() if set(expected_devices) == set(devices): # All devices are online, keep going. break if not usb_restarted and devices: # The USB wasn't restarted, but there's at least one device online. # No point in trying to wait for all devices. break retries -= 1 # TODO(navabi): Test to make sure this fails and then fix call offline_devices = android_commands.GetAttachedDevices( hardware=False, emulator=False, offline=True) types, builds, batteries, reports, errors, json_data = [], [], [], [], [], [] fail_step_lst = [] if devices: types, builds, batteries, reports, errors, fail_step_lst, json_data = ( zip(*[DeviceInfo(dev, options) for dev in devices])) err_msg = CheckForMissingDevices(options, devices) or [] unique_types = list(set(types)) unique_builds = list(set(builds)) bb_annotations.PrintMsg('Online devices: %d. Device types %s, builds %s' % (len(devices), unique_types, unique_builds)) print '\n'.join(reports) for serial, dev_errors in zip(devices, errors): if dev_errors: err_msg += ['%s errors:' % serial] err_msg += [' %s' % error for error in dev_errors] if err_msg: bb_annotations.PrintWarning() msg = '\n'.join(err_msg) print msg from_address = '*****@*****.**' to_addresses = ['*****@*****.**'] bot_name = os.environ.get('BUILDBOT_BUILDERNAME') slave_name = os.environ.get('BUILDBOT_SLAVENAME') subject = 'Device status check errors on %s, %s.' % (slave_name, bot_name) SendEmail(from_address, to_addresses, [], subject, msg) if options.device_status_dashboard: perf_tests_results_helper.PrintPerfResult('BotDevices', 'OnlineDevices', [len(devices)], 'devices') perf_tests_results_helper.PrintPerfResult('BotDevices', 'OfflineDevices', [len(offline_devices)], 'devices', 'unimportant') for serial, battery in zip(devices, batteries): perf_tests_results_helper.PrintPerfResult('DeviceBattery', serial, [battery], '%', 'unimportant') if options.json_output: with open(options.json_output, 'wb') as f: f.write(json.dumps(json_data, indent=4)) num_failed_devs = 0 for fail_status, device in zip(fail_step_lst, devices): if not fail_status: device_blacklist.ExtendBlacklist([str(device)]) num_failed_devs += 1 if num_failed_devs == len(devices): return 2 if not devices: return 1