def ProvisionDevices(options):
if options.device is not None:
devices = [options.device]
else:
devices = android_commands.GetAttachedDevices()
for device_serial in devices:
device = device_utils.DeviceUtils(device_serial)
device.old_interface.EnableAdbRoot()
install_output = GetCmdOutput(
['%s/build/android/adb_install_apk.py' % constants.DIR_SOURCE_ROOT,
'--apk',
'%s/build/android/CheckInstallApk-debug.apk' % constants.DIR_SOURCE_ROOT
])
failure_string = 'Failure [INSTALL_FAILED_INSUFFICIENT_STORAGE]'
if failure_string in install_output:
WipeDeviceData(device)
_ConfigureLocalProperties(device)
device_settings.ConfigureContentSettingsDict(
device, device_settings.DETERMINISTIC_DEVICE_SETTINGS)
# TODO(tonyg): We eventually want network on. However, currently radios
# can cause perfbots to drain faster than they charge.
if 'perf' in os.environ.get('BUILDBOT_BUILDERNAME', '').lower():
device_settings.ConfigureContentSettingsDict(
device, device_settings.NETWORK_DISABLED_SETTINGS)
device.old_interface.RunShellCommandWithSU('date -u %f' % time.time())
(_, props) = device.old_interface.GetShellCommandStatusAndOutput('getprop')
for prop in props:
print prop
if options.auto_reconnect:
PushAndLaunchAdbReboot(devices, options.target)
def CheckForMissingDevices(options, adb_online_devs):
"""Uses file of previous online devices to detect broken phones.
Args:
options: out_dir parameter of options argument is used as the base
directory to load and update the cache file.
adb_online_devs: A list of serial numbers of the currently visible
and online attached devices.
"""
out_dir = os.path.abspath(options.out_dir)
last_devices_path = os.path.join(out_dir, '.last_devices')
last_devices = []
try:
with open(last_devices_path) as f:
last_devices = f.read().splitlines()
except IOError:
# Ignore error, file might not exist
pass
missing_devs = list(set(last_devices) - set(adb_online_devs))
if missing_devs:
buildbot_report.PrintWarning()
buildbot_report.PrintSummaryText('%d devices not detected.' %
len(missing_devs))
print 'Current online devices: %s' % adb_online_devs
print '%s are no longer visible. Were they removed?\n' % missing_devs
print 'SHERIFF: See go/chrome_device_monitor'
print 'Cache file: %s\n\n' % last_devices_path
print 'adb devices'
print GetCmdOutput(['adb', 'devices'])
else:
new_devs = set(adb_online_devs) - set(last_devices)
if new_devs and os.path.exists(last_devices_path):
buildbot_report.PrintWarning()
buildbot_report.PrintSummaryText('%d new devices detected' %
len(new_devs))
print(
'New devices detected %s. And now back to your '
'regularly scheduled program.' % list(new_devs))
if not os.path.exists(out_dir):
os.makedirs(out_dir)
with open(last_devices_path, 'w') as f:
# Write devices currently visible plus devices previously seen.
f.write('\n'.join(set(adb_online_devs + last_devices)))
def CheckForMissingDevices(options, adb_online_devs):
"""Uses file of previous online devices to detect broken phones.
Args:
options: out_dir parameter of options argument is used as the base
directory to load and update the cache file.
adb_online_devs: A list of serial numbers of the currently visible
and online attached devices.
"""
# TODO(navabi): remove this once the bug that causes different number
# of devices to be detected between calls is fixed.
logger = logging.getLogger()
logger.setLevel(logging.INFO)
out_dir = os.path.abspath(options.out_dir)
# last_devices denotes all known devices prior to this run
last_devices_path = os.path.join(out_dir,
device_list.LAST_DEVICES_FILENAME)
last_missing_devices_path = os.path.join(
out_dir, device_list.LAST_MISSING_DEVICES_FILENAME)
try:
last_devices = device_list.GetPersistentDeviceList(last_devices_path)
except IOError:
# Ignore error, file might not exist
last_devices = []
try:
last_missing_devices = device_list.GetPersistentDeviceList(
last_missing_devices_path)
except IOError:
last_missing_devices = []
missing_devs = list(set(last_devices) - set(adb_online_devs))
new_missing_devs = list(set(missing_devs) - set(last_missing_devices))
if new_missing_devs and os.environ.get('BUILDBOT_SLAVENAME'):
logging.info('new_missing_devs %s' % new_missing_devs)
devices_missing_msg = '%d devices not detected.' % len(missing_devs)
bb_annotations.PrintSummaryText(devices_missing_msg)
from_address = '*****@*****.**'
to_addresses = [
'*****@*****.**',
'*****@*****.**'
]
cc_addresses = ['*****@*****.**']
subject = 'Devices offline on %s, %s, %s' % (
os.environ.get('BUILDBOT_SLAVENAME'),
os.environ.get('BUILDBOT_BUILDERNAME'),
os.environ.get('BUILDBOT_BUILDNUMBER'))
msg = ('Please reboot the following devices:\n%s' %
'\n'.join(map(str, new_missing_devs)))
SendEmail(from_address, to_addresses, cc_addresses, subject, msg)
all_known_devices = list(set(adb_online_devs) | set(last_devices))
device_list.WritePersistentDeviceList(last_devices_path, all_known_devices)
device_list.WritePersistentDeviceList(last_missing_devices_path,
missing_devs)
if not all_known_devices:
# This can happen if for some reason the .last_devices file is not
# present or if it was empty.
return ['No online devices. Have any devices been plugged in?']
if missing_devs:
devices_missing_msg = '%d devices not detected.' % len(missing_devs)
bb_annotations.PrintSummaryText(devices_missing_msg)
# TODO(navabi): Debug by printing both output from GetCmdOutput and
# GetAttachedDevices to compare results.
crbug_link = (
'https://code.google.com/p/chromium/issues/entry?summary='
'%s&comment=%s&labels=Restrict-View-Google,OS-Android,Infra' %
(urllib.quote('Device Offline'),
urllib.quote('Buildbot: %s %s\n'
'Build: %s\n'
'(please don\'t change any labels)' %
(os.environ.get('BUILDBOT_BUILDERNAME'),
os.environ.get('BUILDBOT_SLAVENAME'),
os.environ.get('BUILDBOT_BUILDNUMBER')))))
return [
'Current online devices: %s' % adb_online_devs,
'%s are no longer visible. Were they removed?\n' % missing_devs,
'SHERIFF:\n',
'@@@STEP_LINK@Click here to file a bug@%s@@@\n' % crbug_link,
'Cache file: %s\n\n' % last_devices_path,
'adb devices: %s' % GetCmdOutput(['adb', 'devices']),
'adb devices(GetAttachedDevices): %s' % adb_online_devs
]
else:
new_devs = set(adb_online_devs) - set(last_devices)
if new_devs and os.path.exists(last_devices_path):
bb_annotations.PrintWarning()
bb_annotations.PrintSummaryText('%d new devices detected' %
len(new_devs))
print(
'New devices detected %s. And now back to your '
'regularly scheduled program.' % list(new_devs))
def CheckForMissingDevices(options, adb_online_devs):
"""Uses file of previous online devices to detect broken phones.
Args:
options: out_dir parameter of options argument is used as the base
directory to load and update the cache file.
adb_online_devs: A list of serial numbers of the currently visible
and online attached devices.
"""
# TODO(navabi): remove this once the bug that causes different number
# of devices to be detected between calls is fixed.
logger = logging.getLogger()
logger.setLevel(logging.INFO)
out_dir = os.path.abspath(options.out_dir)
def ReadDeviceList(file_name):
devices_path = os.path.join(out_dir, file_name)
devices = []
try:
with open(devices_path) as f:
devices = f.read().splitlines()
except IOError:
# Ignore error, file might not exist
pass
return devices
def WriteDeviceList(file_name, device_list):
path = os.path.join(out_dir, file_name)
if not os.path.exists(out_dir):
os.makedirs(out_dir)
with open(path, 'w') as f:
# Write devices currently visible plus devices previously seen.
f.write('\n'.join(set(device_list)))
last_devices_path = os.path.join(out_dir, '.last_devices')
last_devices = ReadDeviceList('.last_devices')
missing_devs = list(set(last_devices) - set(adb_online_devs))
if missing_devs:
from_address = '*****@*****.**'
to_address = '*****@*****.**'
bot_name = os.environ['BUILDBOT_BUILDERNAME']
slave_name = os.environ['BUILDBOT_SLAVENAME']
num_online_devs = len(adb_online_devs)
subject = 'Devices offline on %s, %s (%d remaining).' % (
slave_name, bot_name, num_online_devs)
buildbot_report.PrintWarning()
devices_missing_msg = '%d devices not detected.' % len(missing_devs)
buildbot_report.PrintSummaryText(devices_missing_msg)
# TODO(navabi): Debug by printing both output from GetCmdOutput and
# GetAttachedDevices to compare results.
body = '\n'.join([
'Current online devices: %s' % adb_online_devs,
'%s are no longer visible. Were they removed?\n' % missing_devs,
'SHERIFF: See go/chrome_device_monitor',
'Cache file: %s\n\n' % last_devices_path,
'adb devices: %s' % GetCmdOutput(['adb', 'devices']),
'adb devices(GetAttachedDevices): %s' % GetAttachedDevices()
])
print body
# Only send email if the first time a particular device goes offline
last_missing = ReadDeviceList('.last_missing')
new_missing_devs = set(missing_devs) - set(last_missing)
if new_missing_devs:
msg_body = '\r\n'.join([
'From: %s' % from_address,
'To: %s' % to_address,
'Subject: %s' % subject, '', body
])
try:
server = smtplib.SMTP('localhost')
server.sendmail(from_address, [to_address], msg_body)
server.quit()
except Exception as e:
print 'Failed to send alert email. Error: %s' % e
else:
new_devs = set(adb_online_devs) - set(last_devices)
if new_devs and os.path.exists(last_devices_path):
buildbot_report.PrintWarning()
buildbot_report.PrintSummaryText('%d new devices detected' %
len(new_devs))
print(
'New devices detected %s. And now back to your '
'regularly scheduled program.' % list(new_devs))
WriteDeviceList('.last_devices', (adb_online_devs + last_devices))
WriteDeviceList('.last_missing', missing_devs)
def AdbShellCmd(cmd):
return GetCmdOutput('adb -s %s shell %s' % (serial, cmd),
shell=True).strip()
def CheckForMissingDevices(options, adb_online_devs):
"""Uses file of previous online devices to detect broken phones.
Args:
options: out_dir parameter of options argument is used as the base
directory to load and update the cache file.
adb_online_devs: A list of serial numbers of the currently visible
and online attached devices.
"""
# TODO(navabi): remove this once the bug that causes different number
# of devices to be detected between calls is fixed.
logger = logging.getLogger()
logger.setLevel(logging.INFO)
out_dir = os.path.abspath(options.out_dir)
def ReadDeviceList(file_name):
devices_path = os.path.join(out_dir, file_name)
devices = []
try:
with open(devices_path) as f:
devices = f.read().splitlines()
except IOError:
# Ignore error, file might not exist
pass
return devices
def WriteDeviceList(file_name, device_list):
path = os.path.join(out_dir, file_name)
if not os.path.exists(out_dir):
os.makedirs(out_dir)
with open(path, 'w') as f:
# Write devices currently visible plus devices previously seen.
f.write('\n'.join(set(device_list)))
last_devices_path = os.path.join(out_dir, '.last_devices')
last_devices = ReadDeviceList('.last_devices')
missing_devs = list(set(last_devices) - set(adb_online_devs))
all_known_devices = list(set(adb_online_devs) | set(last_devices))
WriteDeviceList('.last_devices', all_known_devices)
WriteDeviceList('.last_missing', missing_devs)
if not all_known_devices:
# This can happen if for some reason the .last_devices file is not
# present or if it was empty.
return ['No online devices. Have any devices been plugged in?']
if missing_devs:
devices_missing_msg = '%d devices not detected.' % len(missing_devs)
bb_annotations.PrintSummaryText(devices_missing_msg)
# TODO(navabi): Debug by printing both output from GetCmdOutput and
# GetAttachedDevices to compare results.
return [
'Current online devices: %s' % adb_online_devs,
'%s are no longer visible. Were they removed?\n' % missing_devs,
'SHERIFF: See go/chrome_device_monitor',
'Cache file: %s\n\n' % last_devices_path,
'adb devices: %s' % GetCmdOutput(['adb', 'devices']),
'adb devices(GetAttachedDevices): %s' %
android_commands.GetAttachedDevices()
]
else:
new_devs = set(adb_online_devs) - set(last_devices)
if new_devs and os.path.exists(last_devices_path):
bb_annotations.PrintWarning()
bb_annotations.PrintSummaryText('%d new devices detected' %
len(new_devs))
print(
'New devices detected %s. And now back to your '
'regularly scheduled program.' % list(new_devs))
def DeviceInfo(serial, options):
"""Gathers info on a device via various adb calls.
Args:
serial: The serial of the attached device to construct info about.
Returns:
Tuple of device type, build id, report as a string, error messages, and
boolean indicating whether or not device can be used for testing.
"""
device_adb = android_commands.AndroidCommands(serial)
# TODO(navabi): Replace AdbShellCmd with device_adb.
device_type = device_adb.GetBuildProduct()
device_build = device_adb.GetBuildId()
device_build_type = device_adb.GetBuildType()
device_product_name = device_adb.GetProductName()
setup_wizard_disabled = device_adb.GetSetupWizardStatus() == 'DISABLED'
battery = device_adb.GetBatteryInfo()
install_output = GetCmdOutput([
'%s/build/android/adb_install_apk.py' % constants.DIR_SOURCE_ROOT,
'--apk',
'%s/build/android/CheckInstallApk-debug.apk' %
constants.DIR_SOURCE_ROOT
])
def _GetData(re_expression, line, lambda_function=lambda x: x):
if not line:
return 'Unknown'
found = re.findall(re_expression, line)
if found and len(found):
return lambda_function(found[0])
return 'Unknown'
install_speed = _GetData('(\d+) KB/s', install_output)
ac_power = _GetData('AC powered: (\w+)', battery)
battery_level = _GetData('level: (\d+)', battery)
battery_temp = _GetData('temperature: (\d+)', battery,
lambda x: float(x) / 10.0)
imei_slice = _GetData('Device ID = (\d+)', device_adb.GetSubscriberInfo(),
lambda x: x[-6:])
report = [
'Device %s (%s)' % (serial, device_type),
' Build: %s (%s)' % (device_build, device_adb.GetBuildFingerprint()),
' Battery: %s%%' % battery_level,
' Battery temp: %s' % battery_temp,
' IMEI slice: %s' % imei_slice,
' Wifi IP: %s' % device_adb.GetWifiIP(),
' Install Speed: %s KB/s' % install_speed, ''
]
errors = []
if battery_level < 15:
errors += ['Device critically low in battery. Turning off device.']
if (not setup_wizard_disabled and device_build_type != 'user'
and not options.no_provisioning_check):
errors += ['Setup wizard not disabled. Was it provisioned correctly?']
if device_product_name == 'mantaray' and ac_power != 'true':
errors += ['Mantaray device not connected to AC power.']
# TODO(navabi): Insert warning once we have a better handle of what install
# speeds to expect. The following lines were causing too many alerts.
# if install_speed < 500:
# errors += ['Device install speed too low. Do not use for testing.']
# Causing the device status check step fail for slow install speed or low
# battery currently is too disruptive to the bots (especially try bots).
# Turn off devices with low battery and the step does not fail.
if battery_level < 15:
device_adb.EnableAdbRoot()
device_adb.Shutdown()
full_report = '\n'.join(report)
return device_type, device_build, battery_level, full_report, errors, True
def DeviceInfo(serial, options):
"""Gathers info on a device via various adb calls.
Args:
serial: The serial of the attached device to construct info about.
Returns:
Tuple of device type, build id, report as a string, error messages, and
boolean indicating whether or not device can be used for testing.
"""
def AdbShellCmd(cmd):
return GetCmdOutput('adb -s %s shell %s' % (serial, cmd),
shell=True).strip()
device_adb = android_commands.AndroidCommands(serial)
# TODO(navabi): Replace AdbShellCmd with device_adb.
device_type = AdbShellCmd('getprop ro.build.product')
device_build = AdbShellCmd('getprop ro.build.id')
device_build_type = AdbShellCmd('getprop ro.build.type')
device_product_name = AdbShellCmd('getprop ro.product.name')
setup_wizard_disabled = AdbShellCmd(
'getprop ro.setupwizard.mode') == 'DISABLED'
battery = AdbShellCmd('dumpsys battery')
install_output = GetCmdOutput([
'%s/build/android/adb_install_apk.py' % constants.DIR_SOURCE_ROOT,
'--apk',
'%s/build/android/CheckInstallApk-debug.apk' %
constants.DIR_SOURCE_ROOT
])
install_speed_found = re.findall('(\d+) KB/s', install_output)
if install_speed_found:
install_speed = int(install_speed_found[0])
else:
install_speed = 'Unknown'
if 'Error' in battery:
ac_power = 'Unknown'
battery_level = 'Unknown'
battery_temp = 'Unknown'
else:
ac_power = re.findall('AC powered: (\w+)', battery)[0]
battery_level = int(re.findall('level: (\d+)', battery)[0])
battery_temp = float(re.findall('temperature: (\d+)', battery)[0]) / 10
report = [
'Device %s (%s)' % (serial, device_type),
' Build: %s (%s)' %
(device_build, AdbShellCmd('getprop ro.build.fingerprint')),
' Battery: %s%%' % battery_level,
' Battery temp: %s' % battery_temp,
' IMEI slice: %s' % AdbShellCmd('dumpsys iphonesubinfo '
'| grep Device'
"| awk '{print $4}'")[-6:],
' Wifi IP: %s' % AdbShellCmd('getprop dhcp.wlan0.ipaddress'),
' Install Speed: %s KB/s' % install_speed, ''
]
errors = []
if battery_level < 15:
errors += ['Device critically low in battery. Turning off device.']
if (not setup_wizard_disabled and device_build_type != 'user'
and not options.no_provisioning_check):
errors += ['Setup wizard not disabled. Was it provisioned correctly?']
if device_product_name == 'mantaray' and ac_power != 'true':
errors += ['Mantaray device not connected to AC power.']
# TODO(navabi): Insert warning once we have a better handle of what install
# speeds to expect. The following lines were causing too many alerts.
# if install_speed < 500:
# errors += ['Device install speed too low. Do not use for testing.']
# Causing the device status check step fail for slow install speed or low
# battery currently is too disruptive to the bots (especially try bots).
# Turn off devices with low battery and the step does not fail.
if battery_level < 15:
device_adb.EnableAdbRoot()
AdbShellCmd('reboot -p')
return device_type, device_build, '\n'.join(report), errors, True
def CheckForMissingDevices(options, adb_online_devs):
"""Uses file of previous online devices to detect broken phones.
Args:
options: out_dir parameter of options argument is used as the base
directory to load and update the cache file.
adb_online_devs: A list of serial numbers of the currently visible
and online attached devices.
"""
# TODO(navabi): remove this once the bug that causes different number
# of devices to be detected between calls is fixed.
logger = logging.getLogger()
logger.setLevel(logging.INFO)
out_dir = os.path.abspath(options.out_dir)
def WriteDeviceList(file_name, device_list):
path = os.path.join(out_dir, file_name)
if not os.path.exists(out_dir):
os.makedirs(out_dir)
with open(path, 'w') as f:
# Write devices currently visible plus devices previously seen.
f.write('\n'.join(set(device_list)))
last_devices_path = os.path.join(out_dir, '.last_devices')
last_devices = GetLastDevices(out_dir)
missing_devs = list(set(last_devices) - set(adb_online_devs))
all_known_devices = list(set(adb_online_devs) | set(last_devices))
WriteDeviceList('.last_devices', all_known_devices)
WriteDeviceList('.last_missing', missing_devs)
if not all_known_devices:
# This can happen if for some reason the .last_devices file is not
# present or if it was empty.
return ['No online devices. Have any devices been plugged in?']
if missing_devs:
devices_missing_msg = '%d devices not detected.' % len(missing_devs)
bb_annotations.PrintSummaryText(devices_missing_msg)
# TODO(navabi): Debug by printing both output from GetCmdOutput and
# GetAttachedDevices to compare results.
crbug_link = (
'https://code.google.com/p/chromium/issues/entry?summary='
'%s&comment=%s&labels=Restrict-View-Google,OS-Android,Infra' %
(urllib.quote('Device Offline'),
urllib.quote('Buildbot: %s %s\n'
'Build: %s\n'
'(please don\'t change any labels)' %
(os.environ.get('BUILDBOT_BUILDERNAME'),
os.environ.get('BUILDBOT_SLAVENAME'),
os.environ.get('BUILDBOT_BUILDNUMBER')))))
return [
'Current online devices: %s' % adb_online_devs,
'%s are no longer visible. Were they removed?\n' % missing_devs,
'SHERIFF:\n',
'@@@STEP_LINK@Click here to file a bug@%s@@@\n' % crbug_link,
'Cache file: %s\n\n' % last_devices_path,
'adb devices: %s' % GetCmdOutput(['adb', 'devices']),
'adb devices(GetAttachedDevices): %s' %
android_commands.GetAttachedDevices()
]
else:
new_devs = set(adb_online_devs) - set(last_devices)
if new_devs and os.path.exists(last_devices_path):
bb_annotations.PrintWarning()
bb_annotations.PrintSummaryText('%d new devices detected' %
len(new_devs))
print(
'New devices detected %s. And now back to your '
'regularly scheduled program.' % list(new_devs))
def DeviceInfo(serial):
"""Gathers info on a device via various adb calls.
Args:
serial: The serial of the attached device to construct info about.
Returns:
Tuple of device type, build id, report as a string, error messages, and
boolean indicating whether or not device can be used for testing.
"""
def AdbShellCmd(cmd):
return GetCmdOutput('adb -s %s shell %s' % (serial, cmd),
shell=True).strip()
device_type = AdbShellCmd('getprop ro.build.product')
device_build = AdbShellCmd('getprop ro.build.id')
device_build_type = AdbShellCmd('getprop ro.build.type')
device_product_name = AdbShellCmd('getprop ro.product.name')
setup_wizard_disabled = AdbShellCmd(
'getprop ro.setupwizard.mode') == 'DISABLED'
battery = AdbShellCmd('dumpsys battery')
install_output = GetCmdOutput([
'%s/build/android/adb_install_apk.py' % constants.DIR_SOURCE_ROOT,
'--apk',
'%s/build/android/CheckInstallApk-debug.apk' %
constants.DIR_SOURCE_ROOT
])
install_speed_found = re.findall('(\d+) KB/s', install_output)
if install_speed_found:
install_speed = int(install_speed_found[0])
else:
install_speed = 'Unknown'
if 'Error' in battery:
ac_power = 'Unknown'
battery_level = 'Unknown'
battery_temp = 'Unknown'
else:
ac_power = re.findall('AC powered: (\w+)', battery)[0]
battery_level = int(re.findall('level: (\d+)', battery)[0])
battery_temp = float(re.findall('temperature: (\d+)', battery)[0]) / 10
report = [
'Device %s (%s)' % (serial, device_type),
' Build: %s (%s)' %
(device_build, AdbShellCmd('getprop ro.build.fingerprint')),
' Battery: %s%%' % battery_level,
' Battery temp: %s' % battery_temp,
' IMEI slice: %s' % AdbShellCmd('dumpsys iphonesubinfo '
'| grep Device'
"| awk '{print $4}'")[-6:],
' Wifi IP: %s' % AdbShellCmd('getprop dhcp.wlan0.ipaddress'),
' Install Speed: %s KB/s' % install_speed, ''
]
errors = []
if battery_level < 5:
errors += ['Device critically low in battery. Do not use for testing.']
if not setup_wizard_disabled and device_build_type != 'user':
errors += ['Setup wizard not disabled. Was it provisioned correctly?']
if device_product_name == 'mantaray' and ac_power != 'true':
errors += ['Mantaray device not connected to AC power.']
if install_speed < 800:
errors += ['Device install speed too low. Do not use for testing.']
# TODO(navabi): Determine if device status check step should fail on slow
# install speed. The original CL caused the step to fail but was reverted
# because it caused too many early failures. Determine if it was just flake.
# Also, do not fail on 'Unknown' caused by offline device, because other
# devices can still be used for tests.
fail_step = (battery_level == 'Unknown' or battery_level >= 5)
return device_type, device_build, '\n'.join(report), errors, fail_step