def _GetAllDevices(): devices_path = os.path.join(os.environ.get('CHROMIUM_OUT_DIR', 'out'), device_list.LAST_DEVICES_FILENAME) try: devices = device_list.GetPersistentDeviceList(devices_path) except IOError as e: logging.error('Unable to find %s [%s]', devices_path, e) devices = android_commands.GetAttachedDevices() return sorted(devices)
def RecoverDevices(blacklist, output_directory): # Remove the last build's "bad devices" before checking device statuses. blacklist.Reset() previous_devices = set(a.GetDeviceSerial() for a in adb_wrapper.AdbWrapper.Devices()) KillAllAdb() reset_usb.reset_all_android_devices() try: expected_devices = set( device_list.GetPersistentDeviceList( os.path.join(output_directory, device_list.LAST_DEVICES_FILENAME))) except IOError: expected_devices = set() all_devices = [ device_utils.DeviceUtils(d) for d in previous_devices.union(expected_devices) ] def blacklisting_recovery(device): try: device.WaitUntilFullyBooted() except device_errors.CommandFailedError: logging.exception( 'Failure while waiting for %s. Adding to blacklist.', str(device)) blacklist.Extend([str(device)]) except device_errors.CommandTimeoutError: logging.exception( 'Timed out while waiting for %s. Adding to blacklist.', str(device)) blacklist.Extend([str(device)]) device_utils.DeviceUtils.parallel(all_devices).pMap(blacklisting_recovery) devices = device_utils.DeviceUtils.HealthyDevices(blacklist) device_serials = set(d.adb.GetDeviceSerial() for d in devices) missing_devices = expected_devices.difference(device_serials) new_devices = device_serials.difference(expected_devices) if missing_devices or new_devices: logging.warning('expected_devices:') for d in sorted(expected_devices): logging.warning(' %s', d) logging.warning('devices:') for d in sorted(device_serials): logging.warning(' %s', d) return devices
def _GetAllDevices(active_devices): devices_path = os.path.join(os.environ.get('CHROMIUM_OUT_DIR', 'out'), device_list.LAST_DEVICES_FILENAME) try: devices = [ device_utils.DeviceUtils(s) for s in device_list.GetPersistentDeviceList(devices_path) ] except IOError as e: logging.error('Unable to find %s [%s]', devices_path, e) devices = active_devices return sorted(devices)
def main(): parser = optparse.OptionParser() parser.add_option('', '--out-dir', help='Directory where the device path is stored', default=os.path.join(constants.DIR_SOURCE_ROOT, 'out')) parser.add_option( '--no-provisioning-check', action='store_true', help='Will not check if devices are provisioned properly.') parser.add_option('--device-status-dashboard', action='store_true', help='Output device status data for dashboard.') parser.add_option('--restart-usb', action='store_true', help='Restart USB ports before running device check.') parser.add_option('--json-output', help='Output JSON information into a specified file.') parser.add_option('-v', '--verbose', action='count', default=1, help='Log more information.') options, args = parser.parse_args() if args: parser.error('Unknown options %s' % args) run_tests_helper.SetLogLevel(options.verbose) # Remove the last build's "bad devices" before checking device statuses. device_blacklist.ResetBlacklist() try: expected_devices = device_list.GetPersistentDeviceList( os.path.join(options.out_dir, device_list.LAST_DEVICES_FILENAME)) except IOError: expected_devices = [] devices = device_utils.DeviceUtils.HealthyDevices() device_serials = [d.adb.GetDeviceSerial() for d in devices] # Only restart usb if devices are missing. if set(expected_devices) != set(device_serials): logging.warning('expected_devices: %s', expected_devices) logging.warning('devices: %s', device_serials) KillAllAdb() retries = 5 usb_restarted = True if options.restart_usb: if not RestartUsb(): usb_restarted = False bb_annotations.PrintWarning() logging.error('USB reset stage failed, ' 'wait for any device to come back.') while retries: logging.info('retry adb devices...') time.sleep(1) devices = device_utils.DeviceUtils.HealthyDevices() device_serials = [d.adb.GetDeviceSerial() for d in devices] if set(expected_devices) == set(device_serials): # All devices are online, keep going. break if not usb_restarted and devices: # The USB wasn't restarted, but there's at least one device online. # No point in trying to wait for all devices. break retries -= 1 types, builds, batteries, errors, devices_ok, json_data = ([], [], [], [], [], []) if devices: types, builds, batteries, errors, devices_ok, json_data = (zip( *[DeviceInfo(dev, options) for dev in devices])) # Write device info to file for buildbot info display. if os.path.exists('/home/chrome-bot'): with open('/home/chrome-bot/.adb_device_info', 'w') as f: for device in json_data: try: f.write('%s %s %s %.1fC %s%%\n' % (device['serial'], device['type'], device['build'], float(device['battery']['temperature']) / 10, device['battery']['level'])) except Exception: pass err_msg = CheckForMissingDevices(options, devices) or [] unique_types = list(set(types)) unique_builds = list(set(builds)) bb_annotations.PrintMsg('Online devices: %d. Device types %s, builds %s' % (len(devices), unique_types, unique_builds)) for j in json_data: logging.info('Device %s (%s)', j.get('serial'), j.get('type')) logging.info(' Build: %s (%s)', j.get('build'), j.get('build_detail')) logging.info(' Current Battery Service state:') for k, v in j.get('battery', {}).iteritems(): logging.info(' %s: %s', k, v) logging.info(' IMEI slice: %s', j.get('imei_slice')) logging.info(' WiFi IP: %s', j.get('wifi_ip')) for dev, dev_errors in zip(devices, errors): if dev_errors: err_msg += ['%s errors:' % str(dev)] err_msg += [' %s' % error for error in dev_errors] if err_msg: bb_annotations.PrintWarning() for e in err_msg: logging.error(e) from_address = '*****@*****.**' to_addresses = ['*****@*****.**'] bot_name = os.environ.get('BUILDBOT_BUILDERNAME') slave_name = os.environ.get('BUILDBOT_SLAVENAME') subject = 'Device status check errors on %s, %s.' % (slave_name, bot_name) SendEmail(from_address, to_addresses, [], subject, '\n'.join(err_msg)) if options.device_status_dashboard: offline_devices = [ device_utils.DeviceUtils(a) for a in adb_wrapper.AdbWrapper.Devices(is_ready=False) if a.GetState() == 'offline' ] perf_tests_results_helper.PrintPerfResult('BotDevices', 'OnlineDevices', [len(devices)], 'devices') perf_tests_results_helper.PrintPerfResult('BotDevices', 'OfflineDevices', [len(offline_devices)], 'devices', 'unimportant') for dev, battery in zip(devices, batteries): perf_tests_results_helper.PrintPerfResult('DeviceBattery', str(dev), [battery], '%', 'unimportant') if options.json_output: with open(options.json_output, 'wb') as f: f.write(json.dumps(json_data, indent=4)) num_failed_devs = 0 for device_ok, device in zip(devices_ok, devices): if not device_ok: logging.warning('Blacklisting %s', str(device)) device_blacklist.ExtendBlacklist([str(device)]) num_failed_devs += 1 if num_failed_devs == len(devices): return 2 if not devices: return 1
def CheckForMissingDevices(options, devices): """Uses file of previous online devices to detect broken phones. Args: options: out_dir parameter of options argument is used as the base directory to load and update the cache file. devices: A list of DeviceUtils instance for the currently visible and online attached devices. """ out_dir = os.path.abspath(options.out_dir) device_serials = set(d.adb.GetDeviceSerial() for d in devices) # last_devices denotes all known devices prior to this run last_devices_path = os.path.join(out_dir, device_list.LAST_DEVICES_FILENAME) last_missing_devices_path = os.path.join( out_dir, device_list.LAST_MISSING_DEVICES_FILENAME) try: last_devices = device_list.GetPersistentDeviceList(last_devices_path) except IOError: # Ignore error, file might not exist last_devices = [] try: last_missing_devices = device_list.GetPersistentDeviceList( last_missing_devices_path) except IOError: last_missing_devices = [] missing_devs = list(set(last_devices) - device_serials) new_missing_devs = list(set(missing_devs) - set(last_missing_devices)) if new_missing_devs and os.environ.get('BUILDBOT_SLAVENAME'): logging.info('new_missing_devs %s' % new_missing_devs) devices_missing_msg = '%d devices not detected.' % len(missing_devs) bb_annotations.PrintSummaryText(devices_missing_msg) from_address = '*****@*****.**' to_addresses = [ '*****@*****.**', '*****@*****.**' ] cc_addresses = ['*****@*****.**'] subject = 'Devices offline on %s, %s, %s' % ( os.environ.get('BUILDBOT_SLAVENAME'), os.environ.get('BUILDBOT_BUILDERNAME'), os.environ.get('BUILDBOT_BUILDNUMBER')) msg = ('Please reboot the following devices:\n%s' % '\n'.join(map(str, new_missing_devs))) SendEmail(from_address, to_addresses, cc_addresses, subject, msg) all_known_devices = list(device_serials | set(last_devices)) device_list.WritePersistentDeviceList(last_devices_path, all_known_devices) device_list.WritePersistentDeviceList(last_missing_devices_path, missing_devs) if not all_known_devices: # This can happen if for some reason the .last_devices file is not # present or if it was empty. return ['No online devices. Have any devices been plugged in?'] if missing_devs: devices_missing_msg = '%d devices not detected.' % len(missing_devs) bb_annotations.PrintSummaryText(devices_missing_msg) return [ 'Current online devices: %s' % ', '.join(d for d in device_serials), '%s are no longer visible. Were they removed?' % missing_devs ] else: new_devs = device_serials - set(last_devices) if new_devs and os.path.exists(last_devices_path): bb_annotations.PrintWarning() bb_annotations.PrintSummaryText('%d new devices detected' % len(new_devs)) logging.info('New devices detected:') for d in new_devs: logging.info(' %s', d)
def main(): parser = optparse.OptionParser() parser.add_option('', '--out-dir', help='Directory where the device path is stored', default=os.path.join(constants.DIR_SOURCE_ROOT, 'out')) parser.add_option( '--no-provisioning-check', action='store_true', help='Will not check if devices are provisioned properly.') parser.add_option('--device-status-dashboard', action='store_true', help='Output device status data for dashboard.') parser.add_option('--restart-usb', action='store_true', help='Restart USB ports before running device check.') parser.add_option('--json-output', help='Output JSON information into a specified file.') options, args = parser.parse_args() if args: parser.error('Unknown options %s' % args) # Remove the last build's "bad devices" before checking device statuses. device_blacklist.ResetBlacklist() try: expected_devices = device_list.GetPersistentDeviceList( os.path.join(options.out_dir, device_list.LAST_DEVICES_FILENAME)) except IOError: expected_devices = [] devices = android_commands.GetAttachedDevices() # Only restart usb if devices are missing. if set(expected_devices) != set(devices): print 'expected_devices: %s, devices: %s' % (expected_devices, devices) KillAllAdb() retries = 5 usb_restarted = True if options.restart_usb: if not RestartUsb(): usb_restarted = False bb_annotations.PrintWarning() print 'USB reset stage failed, wait for any device to come back.' while retries: print 'retry adb devices...' time.sleep(1) devices = android_commands.GetAttachedDevices() if set(expected_devices) == set(devices): # All devices are online, keep going. break if not usb_restarted and devices: # The USB wasn't restarted, but there's at least one device online. # No point in trying to wait for all devices. break retries -= 1 # TODO(navabi): Test to make sure this fails and then fix call offline_devices = android_commands.GetAttachedDevices(hardware=False, emulator=False, offline=True) types, builds, batteries, reports, errors = [], [], [], [], [] fail_step_lst = [] if devices: types, builds, batteries, reports, errors, fail_step_lst = (zip( *[DeviceInfo(dev, options) for dev in devices])) err_msg = CheckForMissingDevices(options, devices) or [] unique_types = list(set(types)) unique_builds = list(set(builds)) bb_annotations.PrintMsg('Online devices: %d. Device types %s, builds %s' % (len(devices), unique_types, unique_builds)) print '\n'.join(reports) for serial, dev_errors in zip(devices, errors): if dev_errors: err_msg += ['%s errors:' % serial] err_msg += [' %s' % error for error in dev_errors] if err_msg: bb_annotations.PrintWarning() msg = '\n'.join(err_msg) print msg from_address = '*****@*****.**' to_addresses = ['*****@*****.**'] bot_name = os.environ.get('BUILDBOT_BUILDERNAME') slave_name = os.environ.get('BUILDBOT_SLAVENAME') subject = 'Device status check errors on %s, %s.' % (slave_name, bot_name) SendEmail(from_address, to_addresses, [], subject, msg) if options.device_status_dashboard: perf_tests_results_helper.PrintPerfResult('BotDevices', 'OnlineDevices', [len(devices)], 'devices') perf_tests_results_helper.PrintPerfResult('BotDevices', 'OfflineDevices', [len(offline_devices)], 'devices', 'unimportant') for serial, battery in zip(devices, batteries): perf_tests_results_helper.PrintPerfResult('DeviceBattery', serial, [battery], '%', 'unimportant') if options.json_output: with open(options.json_output, 'wb') as f: f.write( json.dumps({ 'online_devices': devices, 'offline_devices': offline_devices, 'expected_devices': expected_devices, 'unique_types': unique_types, 'unique_builds': unique_builds, })) if False in fail_step_lst: # TODO(navabi): Build fails on device status check step if there exists any # devices with critically low battery. Remove those devices from testing, # allowing build to continue with good devices. return 2 if not devices: return 1
def CheckForMissingDevices(options, adb_online_devs): """Uses file of previous online devices to detect broken phones. Args: options: out_dir parameter of options argument is used as the base directory to load and update the cache file. adb_online_devs: A list of serial numbers of the currently visible and online attached devices. """ # TODO(navabi): remove this once the bug that causes different number # of devices to be detected between calls is fixed. logger = logging.getLogger() logger.setLevel(logging.INFO) out_dir = os.path.abspath(options.out_dir) # last_devices denotes all known devices prior to this run last_devices_path = os.path.join(out_dir, device_list.LAST_DEVICES_FILENAME) last_missing_devices_path = os.path.join( out_dir, device_list.LAST_MISSING_DEVICES_FILENAME) try: last_devices = device_list.GetPersistentDeviceList(last_devices_path) except IOError: # Ignore error, file might not exist last_devices = [] try: last_missing_devices = device_list.GetPersistentDeviceList( last_missing_devices_path) except IOError: last_missing_devices = [] missing_devs = list(set(last_devices) - set(adb_online_devs)) new_missing_devs = list(set(missing_devs) - set(last_missing_devices)) if new_missing_devs and os.environ.get('BUILDBOT_SLAVENAME'): logging.info('new_missing_devs %s' % new_missing_devs) devices_missing_msg = '%d devices not detected.' % len(missing_devs) bb_annotations.PrintSummaryText(devices_missing_msg) from_address = '*****@*****.**' to_addresses = [ '*****@*****.**', '*****@*****.**' ] cc_addresses = ['*****@*****.**'] subject = 'Devices offline on %s, %s, %s' % ( os.environ.get('BUILDBOT_SLAVENAME'), os.environ.get('BUILDBOT_BUILDERNAME'), os.environ.get('BUILDBOT_BUILDNUMBER')) msg = ('Please reboot the following devices:\n%s' % '\n'.join(map(str, new_missing_devs))) SendEmail(from_address, to_addresses, cc_addresses, subject, msg) all_known_devices = list(set(adb_online_devs) | set(last_devices)) device_list.WritePersistentDeviceList(last_devices_path, all_known_devices) device_list.WritePersistentDeviceList(last_missing_devices_path, missing_devs) if not all_known_devices: # This can happen if for some reason the .last_devices file is not # present or if it was empty. return ['No online devices. Have any devices been plugged in?'] if missing_devs: devices_missing_msg = '%d devices not detected.' % len(missing_devs) bb_annotations.PrintSummaryText(devices_missing_msg) # TODO(navabi): Debug by printing both output from GetCmdOutput and # GetAttachedDevices to compare results. crbug_link = ( 'https://code.google.com/p/chromium/issues/entry?summary=' '%s&comment=%s&labels=Restrict-View-Google,OS-Android,Infra' % (urllib.quote('Device Offline'), urllib.quote('Buildbot: %s %s\n' 'Build: %s\n' '(please don\'t change any labels)' % (os.environ.get('BUILDBOT_BUILDERNAME'), os.environ.get('BUILDBOT_SLAVENAME'), os.environ.get('BUILDBOT_BUILDNUMBER'))))) return [ 'Current online devices: %s' % adb_online_devs, '%s are no longer visible. Were they removed?\n' % missing_devs, 'SHERIFF:\n', '@@@STEP_LINK@Click here to file a bug@%s@@@\n' % crbug_link, 'Cache file: %s\n\n' % last_devices_path, 'adb devices: %s' % GetCmdOutput(['adb', 'devices']), 'adb devices(GetAttachedDevices): %s' % adb_online_devs ] else: new_devs = set(adb_online_devs) - set(last_devices) if new_devs and os.path.exists(last_devices_path): bb_annotations.PrintWarning() bb_annotations.PrintSummaryText('%d new devices detected' % len(new_devs)) print( 'New devices detected %s. And now back to your ' 'regularly scheduled program.' % list(new_devs))