def RecoverDevices(blacklist, output_directory):
# Remove the last build's "bad devices" before checking device statuses.
blacklist.Reset()
previous_devices = set(a.GetDeviceSerial()
for a in adb_wrapper.AdbWrapper.Devices())
KillAllAdb()
reset_usb.reset_all_android_devices()
try:
expected_devices = set(
device_list.GetPersistentDeviceList(
os.path.join(output_directory,
device_list.LAST_DEVICES_FILENAME)))
except IOError:
expected_devices = set()
all_devices = [
device_utils.DeviceUtils(d)
for d in previous_devices.union(expected_devices)
]
def blacklisting_recovery(device):
try:
device.WaitUntilFullyBooted()
except device_errors.CommandFailedError:
logging.exception(
'Failure while waiting for %s. Adding to blacklist.',
str(device))
blacklist.Extend([str(device)])
except device_errors.CommandTimeoutError:
logging.exception(
'Timed out while waiting for %s. Adding to blacklist.',
str(device))
blacklist.Extend([str(device)])
device_utils.DeviceUtils.parallel(all_devices).pMap(blacklisting_recovery)
devices = device_utils.DeviceUtils.HealthyDevices(blacklist)
device_serials = set(d.adb.GetDeviceSerial() for d in devices)
missing_devices = expected_devices.difference(device_serials)
new_devices = device_serials.difference(expected_devices)
if missing_devices or new_devices:
logging.warning('expected_devices:')
for d in sorted(expected_devices):
logging.warning(' %s', d)
logging.warning('devices:')
for d in sorted(device_serials):
logging.warning(' %s', d)
return devices
def RecoverDevices(blacklist, output_directory):
# Remove the last build's "bad devices" before checking device statuses.
blacklist.Reset()
previous_devices = set(a.GetDeviceSerial()
for a in adb_wrapper.AdbWrapper.Devices())
KillAllAdb()
reset_usb.reset_all_android_devices()
try:
expected_devices = set(device_list.GetPersistentDeviceList(
os.path.join(output_directory, device_list.LAST_DEVICES_FILENAME)))
except IOError:
expected_devices = set()
all_devices = [device_utils.DeviceUtils(d)
for d in previous_devices.union(expected_devices)]
def blacklisting_recovery(device):
try:
device.WaitUntilFullyBooted()
except device_errors.CommandFailedError:
logging.exception('Failure while waiting for %s. Adding to blacklist.',
str(device))
blacklist.Extend([str(device)])
except device_errors.CommandTimeoutError:
logging.exception('Timed out while waiting for %s. Adding to blacklist.',
str(device))
blacklist.Extend([str(device)])
device_utils.DeviceUtils.parallel(all_devices).pMap(blacklisting_recovery)
devices = device_utils.DeviceUtils.HealthyDevices(blacklist)
device_serials = set(d.adb.GetDeviceSerial() for d in devices)
missing_devices = expected_devices.difference(device_serials)
new_devices = device_serials.difference(expected_devices)
if missing_devices or new_devices:
logging.warning('expected_devices:')
for d in sorted(expected_devices):
logging.warning(' %s', d)
logging.warning('devices:')
for d in sorted(device_serials):
logging.warning(' %s', d)
return devices
def main():
parser = optparse.OptionParser()
parser.add_option('',
'--out-dir',
help='Directory where the device path is stored',
default=os.path.join(constants.DIR_SOURCE_ROOT, 'out'))
parser.add_option(
'--no-provisioning-check',
action='store_true',
help='Will not check if devices are provisioned properly.')
parser.add_option('--device-status-dashboard',
action='store_true',
help='Output device status data for dashboard.')
parser.add_option('--restart-usb',
action='store_true',
help='DEPRECATED. '
'This script now always tries to reset USB.')
parser.add_option('--json-output',
help='Output JSON information into a specified file.')
parser.add_option('-v',
'--verbose',
action='count',
default=1,
help='Log more information.')
options, args = parser.parse_args()
if args:
parser.error('Unknown options %s' % args)
run_tests_helper.SetLogLevel(options.verbose)
# Remove the last build's "bad devices" before checking device statuses.
device_blacklist.ResetBlacklist()
KillAllAdb()
reset_usb.reset_all_android_devices()
try:
expected_devices = set(
device_list.GetPersistentDeviceList(
os.path.join(options.out_dir,
device_list.LAST_DEVICES_FILENAME)))
except IOError:
expected_devices = set()
def all_devices_found():
devices = device_utils.DeviceUtils.HealthyDevices()
device_serials = set(d.adb.GetDeviceSerial() for d in devices)
return not bool(expected_devices.difference(device_serials))
timeout_retry.WaitFor(all_devices_found, wait_period=1, max_tries=5)
devices = device_utils.DeviceUtils.HealthyDevices()
device_serials = set(d.adb.GetDeviceSerial() for d in devices)
missing_devices = expected_devices.difference(device_serials)
new_devices = device_serials.difference(expected_devices)
if missing_devices or new_devices:
logging.warning('expected_devices:')
for d in sorted(expected_devices):
logging.warning(' %s', d)
logging.warning('devices:')
for d in sorted(device_serials):
logging.warning(' %s', d)
types, builds, batteries, errors, devices_ok, json_data = ([], [], [], [],
[], [])
if devices:
types, builds, batteries, errors, devices_ok, json_data = (zip(
*[DeviceInfo(dev, options) for dev in devices]))
# Write device info to file for buildbot info display.
if os.path.exists('/home/chrome-bot'):
with open('/home/chrome-bot/.adb_device_info', 'w') as f:
for device in json_data:
try:
f.write('%s %s %s %.1fC %s%%\n' %
(device['serial'], device['type'], device['build'],
float(device['battery']['temperature']) / 10,
device['battery']['level']))
except Exception:
pass
err_msg = CheckForMissingDevices(options, devices) or []
unique_types = list(set(types))
unique_builds = list(set(builds))
bb_annotations.PrintMsg('Online devices: %d. Device types %s, builds %s' %
(len(devices), unique_types, unique_builds))
for j in json_data:
logging.info('Device %s (%s)', j.get('serial'), j.get('type'))
logging.info(' Build: %s (%s)', j.get('build'), j.get('build_detail'))
logging.info(' Current Battery Service state:')
for k, v in j.get('battery', {}).iteritems():
logging.info(' %s: %s', k, v)
logging.info(' IMEI slice: %s', j.get('imei_slice'))
logging.info(' WiFi IP: %s', j.get('wifi_ip'))
for dev, dev_errors in zip(devices, errors):
if dev_errors:
err_msg += ['%s errors:' % str(dev)]
err_msg += [' %s' % error for error in dev_errors]
if err_msg:
bb_annotations.PrintWarning()
for e in err_msg:
logging.error(e)
from_address = '*****@*****.**'
to_addresses = ['*****@*****.**']
bot_name = os.environ.get('BUILDBOT_BUILDERNAME')
slave_name = os.environ.get('BUILDBOT_SLAVENAME')
subject = 'Device status check errors on %s, %s.' % (slave_name,
bot_name)
SendEmail(from_address, to_addresses, [], subject, '\n'.join(err_msg))
if options.device_status_dashboard:
offline_devices = [
device_utils.DeviceUtils(a)
for a in adb_wrapper.AdbWrapper.Devices(is_ready=False)
if a.GetState() == 'offline'
]
perf_tests_results_helper.PrintPerfResult('BotDevices',
'OnlineDevices',
[len(devices)], 'devices')
perf_tests_results_helper.PrintPerfResult('BotDevices',
'OfflineDevices',
[len(offline_devices)],
'devices', 'unimportant')
for dev, battery in zip(devices, batteries):
perf_tests_results_helper.PrintPerfResult('DeviceBattery',
str(dev), [battery], '%',
'unimportant')
if options.json_output:
with open(options.json_output, 'wb') as f:
f.write(json.dumps(json_data, indent=4))
num_failed_devs = 0
for device_ok, device in zip(devices_ok, devices):
if not device_ok:
logging.warning('Blacklisting %s', str(device))
device_blacklist.ExtendBlacklist([str(device)])
num_failed_devs += 1
if num_failed_devs == len(devices):
return 2
if not devices:
return 1
def main():
parser = optparse.OptionParser()
parser.add_option(
"",
"--out-dir",
help="Directory where the device path is stored",
default=os.path.join(constants.DIR_SOURCE_ROOT, "out"),
)
parser.add_option(
"--no-provisioning-check", action="store_true", help="Will not check if devices are provisioned properly."
)
parser.add_option("--device-status-dashboard", action="store_true", help="Output device status data for dashboard.")
parser.add_option(
"--restart-usb", action="store_true", help="DEPRECATED. " "This script now always tries to reset USB."
)
parser.add_option("--json-output", help="Output JSON information into a specified file.")
parser.add_option("-v", "--verbose", action="count", default=1, help="Log more information.")
options, args = parser.parse_args()
if args:
parser.error("Unknown options %s" % args)
run_tests_helper.SetLogLevel(options.verbose)
# Remove the last build's "bad devices" before checking device statuses.
device_blacklist.ResetBlacklist()
KillAllAdb()
reset_usb.reset_all_android_devices()
try:
expected_devices = set(
device_list.GetPersistentDeviceList(os.path.join(options.out_dir, device_list.LAST_DEVICES_FILENAME))
)
except IOError:
expected_devices = set()
def all_devices_found():
devices = device_utils.DeviceUtils.HealthyDevices()
device_serials = set(d.adb.GetDeviceSerial() for d in devices)
return not bool(expected_devices.difference(device_serials))
timeout_retry.WaitFor(all_devices_found, wait_period=1, max_tries=5)
devices = device_utils.DeviceUtils.HealthyDevices()
device_serials = set(d.adb.GetDeviceSerial() for d in devices)
missing_devices = expected_devices.difference(device_serials)
new_devices = device_serials.difference(expected_devices)
if missing_devices or new_devices:
logging.warning("expected_devices:")
for d in sorted(expected_devices):
logging.warning(" %s", d)
logging.warning("devices:")
for d in sorted(device_serials):
logging.warning(" %s", d)
types, builds, batteries, errors, devices_ok, json_data = ([], [], [], [], [], [])
if devices:
types, builds, batteries, errors, devices_ok, json_data = zip(*[DeviceInfo(dev, options) for dev in devices])
# Write device info to file for buildbot info display.
if os.path.exists("/home/chrome-bot"):
with open("/home/chrome-bot/.adb_device_info", "w") as f:
for device in json_data:
try:
f.write(
"%s %s %s %.1fC %s%%\n"
% (
device["serial"],
device["type"],
device["build"],
float(device["battery"]["temperature"]) / 10,
device["battery"]["level"],
)
)
except Exception:
pass
err_msg = CheckForMissingDevices(options, devices) or []
unique_types = list(set(types))
unique_builds = list(set(builds))
bb_annotations.PrintMsg(
"Online devices: %d. Device types %s, builds %s" % (len(devices), unique_types, unique_builds)
)
for j in json_data:
logging.info("Device %s (%s)", j.get("serial"), j.get("type"))
logging.info(" Build: %s (%s)", j.get("build"), j.get("build_detail"))
logging.info(" Current Battery Service state:")
for k, v in j.get("battery", {}).iteritems():
logging.info(" %s: %s", k, v)
logging.info(" IMEI slice: %s", j.get("imei_slice"))
logging.info(" WiFi IP: %s", j.get("wifi_ip"))
for dev, dev_errors in zip(devices, errors):
if dev_errors:
err_msg += ["%s errors:" % str(dev)]
err_msg += [" %s" % error for error in dev_errors]
if err_msg:
bb_annotations.PrintWarning()
for e in err_msg:
logging.error(e)
from_address = "*****@*****.**"
to_addresses = ["*****@*****.**"]
bot_name = os.environ.get("BUILDBOT_BUILDERNAME")
slave_name = os.environ.get("BUILDBOT_SLAVENAME")
subject = "Device status check errors on %s, %s." % (slave_name, bot_name)
SendEmail(from_address, to_addresses, [], subject, "\n".join(err_msg))
if options.device_status_dashboard:
offline_devices = [
device_utils.DeviceUtils(a)
for a in adb_wrapper.AdbWrapper.Devices(is_ready=False)
if a.GetState() == "offline"
]
perf_tests_results_helper.PrintPerfResult("BotDevices", "OnlineDevices", [len(devices)], "devices")
perf_tests_results_helper.PrintPerfResult(
"BotDevices", "OfflineDevices", [len(offline_devices)], "devices", "unimportant"
)
for dev, battery in zip(devices, batteries):
perf_tests_results_helper.PrintPerfResult("DeviceBattery", str(dev), [battery], "%", "unimportant")
if options.json_output:
with open(options.json_output, "wb") as f:
f.write(json.dumps(json_data, indent=4))
num_failed_devs = 0
for device_ok, device in zip(devices_ok, devices):
if not device_ok:
logging.warning("Blacklisting %s", str(device))
device_blacklist.ExtendBlacklist([str(device)])
num_failed_devs += 1
if num_failed_devs == len(devices):
return 2
if not devices:
return 1
