def PrintAnnotationForTestResults(test_results):
if test_results.timed_out:
buildbot_report.PrintWarning()
elif test_results.failed or test_results.crashed or test_results.overall_fail:
buildbot_report.PrintError()
else:
print 'Step success!' # No annotation needed
def main():
parser = optparse.OptionParser()
parser.add_option('',
'--out-dir',
help='Directory where the device path is stored',
default=os.path.join(os.path.dirname(__file__), '..',
'..', 'out'))
options, args = parser.parse_args()
if args:
parser.error('Unknown options %s' % args)
buildbot_report.PrintNamedStep('Device Status Check')
devices = android_commands.GetAttachedDevices()
types, builds, reports, errors = [], [], [], []
if devices:
types, builds, reports, errors = zip(
*[DeviceInfo(dev) for dev in devices])
unique_types = list(set(types))
unique_builds = list(set(builds))
buildbot_report.PrintMsg('Online devices: %d. Device types %s, builds %s' %
(len(devices), unique_types, unique_builds))
print '\n'.join(reports)
full_errors = []
for serial, device_errors in zip(devices, errors):
full_errors.extend('%s: %s' % (serial, error)
for error in device_errors)
if full_errors:
buildbot_report.PrintWarning()
print '\n'.join(full_errors)
CheckForMissingDevices(options, devices)
def CheckForMissingDevices(options, adb_online_devs):
"""Uses file of previous online devices to detect broken phones.
Args:
options: out_dir parameter of options argument is used as the base
directory to load and update the cache file.
adb_online_devs: A list of serial numbers of the currently visible
and online attached devices.
"""
out_dir = os.path.abspath(options.out_dir)
last_devices_path = os.path.join(out_dir, '.last_devices')
last_devices = []
try:
with open(last_devices_path) as f:
last_devices = f.read().splitlines()
except IOError:
# Ignore error, file might not exist
pass
missing_devs = list(set(last_devices) - set(adb_online_devs))
if missing_devs:
buildbot_report.PrintWarning()
buildbot_report.PrintSummaryText('%d devices not detected.' %
len(missing_devs))
print 'Current online devices: %s' % adb_online_devs
print '%s are no longer visible. Were they removed?\n' % missing_devs
print 'SHERIFF: See go/chrome_device_monitor'
print 'Cache file: %s\n\n' % last_devices_path
print 'adb devices'
print GetCmdOutput(['adb', 'devices'])
else:
new_devs = set(adb_online_devs) - set(last_devices)
if new_devs and os.path.exists(last_devices_path):
buildbot_report.PrintWarning()
buildbot_report.PrintSummaryText('%d new devices detected' %
len(new_devs))
print(
'New devices detected %s. And now back to your '
'regularly scheduled program.' % list(new_devs))
if not os.path.exists(out_dir):
os.makedirs(out_dir)
with open(last_devices_path, 'w') as f:
# Write devices currently visible plus devices previously seen.
f.write('\n'.join(set(adb_online_devs + last_devices)))
def RunCmd(command, flunk_on_failure=True):
"""Run a command relative to the chrome source root."""
code = SpawnCmd(command).wait()
print '<', ' '.join(map(pipes.quote, command))
if code != 0:
print 'ERROR: process exited with code %d' % code
if flunk_on_failure:
buildbot_report.PrintError()
else:
buildbot_report.PrintWarning()
return code
def RunCmd(command, flunk_on_failure=True):
"""Run a command relative to the chrome source root."""
code = SpawnCmd(command).wait()
print '<', ' '.join(map(pipes.quote, command))
if code != 0:
print 'ERROR: non-zero status %d from %s' % (code, command)
if flunk_on_failure:
buildbot_report.PrintError()
else:
buildbot_report.PrintWarning()
return code
def main():
parser = optparse.OptionParser()
parser.add_option('',
'--out-dir',
help='Directory where the device path is stored',
default=os.path.join(os.path.dirname(__file__), '..',
'..', 'out'))
parser.add_option(
'--no-provisioning-check',
help='Will not check if devices are provisioned properly.')
options, args = parser.parse_args()
if args:
parser.error('Unknown options %s' % args)
devices = android_commands.GetAttachedDevices()
types, builds, reports, errors = [], [], [], []
fail_step_lst = []
if devices:
types, builds, reports, errors, fail_step_lst = (zip(
*[DeviceInfo(dev, options) for dev in devices]))
err_msg = CheckForMissingDevices(options, devices) or []
unique_types = list(set(types))
unique_builds = list(set(builds))
buildbot_report.PrintMsg('Online devices: %d. Device types %s, builds %s' %
(len(devices), unique_types, unique_builds))
print '\n'.join(reports)
for serial, dev_errors in zip(devices, errors):
if dev_errors:
err_msg += ['%s errors:' % serial]
err_msg += [' %s' % error for error in dev_errors]
if err_msg:
buildbot_report.PrintWarning()
msg = '\n'.join(err_msg)
print msg
SendDeviceStatusAlert(msg)
if False in fail_step_lst:
# TODO(navabi): Build fails on device status check step if there exists any
# devices with critically low battery or install speed. Remove those devices
# from testing, allowing build to continue with good devices.
return 1
if not devices:
return 1
def RunCmd(command,
flunk_on_failure=True,
halt_on_failure=False,
warning_code=88):
"""Run a command relative to the chrome source root."""
code = SpawnCmd(command).wait()
print '<', CommandToString(command)
if code != 0:
print 'ERROR: process exited with code %d' % code
if code != warning_code and flunk_on_failure:
buildbot_report.PrintError()
else:
buildbot_report.PrintWarning()
# Allow steps to have both halting (i.e. 1) and non-halting exit codes.
if code != warning_code and halt_on_failure:
raise OSError()
return code
def RebootDevices():
"""Reboot all attached and online devices."""
# Early return here to avoid presubmit dependence on adb,
# which might not exist in this checkout.
if bb_utils.TESTING:
return
devices = android_commands.GetAttachedDevices()
print 'Rebooting: %s' % devices
if devices:
pool = multiprocessing.Pool(len(devices))
results = pool.map_async(RebootDeviceSafe, devices).get(99999)
for device, result in zip(devices, results):
if result:
print '%s failed to startup.' % device
if any(results):
buildbot_report.PrintWarning()
else:
print 'Reboots complete.'
def main():
parser = optparse.OptionParser()
parser.add_option('', '--out-dir',
help='Directory where the device path is stored',
default=os.path.join(os.path.dirname(__file__), '..',
'..', 'out'))
options, args = parser.parse_args()
if args:
parser.error('Unknown options %s' % args)
devices = android_commands.GetAttachedDevices()
types, builds, reports, errors = [], [], [], []
if devices:
types, builds, reports, errors = zip(*[DeviceInfo(dev) for dev in devices])
err_msg = CheckForMissingDevices(options, devices) or []
unique_types = list(set(types))
unique_builds = list(set(builds))
buildbot_report.PrintMsg('Online devices: %d. Device types %s, builds %s'
% (len(devices), unique_types, unique_builds))
print '\n'.join(reports)
for serial, dev_errors in zip(devices, errors):
if dev_errors:
err_msg += ['%s errors:' % serial]
err_msg += [' %s' % error for error in dev_errors]
if err_msg:
buildbot_report.PrintWarning()
msg = '\n'.join(err_msg)
print msg
SendDeviceStatusAlert(msg)
if not devices:
return 1
def CheckForMissingDevices(options, adb_online_devs):
"""Uses file of previous online devices to detect broken phones.
Args:
options: out_dir parameter of options argument is used as the base
directory to load and update the cache file.
adb_online_devs: A list of serial numbers of the currently visible
and online attached devices.
"""
# TODO(navabi): remove this once the bug that causes different number
# of devices to be detected between calls is fixed.
logger = logging.getLogger()
logger.setLevel(logging.INFO)
out_dir = os.path.abspath(options.out_dir)
def ReadDeviceList(file_name):
devices_path = os.path.join(out_dir, file_name)
devices = []
try:
with open(devices_path) as f:
devices = f.read().splitlines()
except IOError:
# Ignore error, file might not exist
pass
return devices
def WriteDeviceList(file_name, device_list):
path = os.path.join(out_dir, file_name)
if not os.path.exists(out_dir):
os.makedirs(out_dir)
with open(path, 'w') as f:
# Write devices currently visible plus devices previously seen.
f.write('\n'.join(set(device_list)))
last_devices_path = os.path.join(out_dir, '.last_devices')
last_devices = ReadDeviceList('.last_devices')
missing_devs = list(set(last_devices) - set(adb_online_devs))
if missing_devs:
from_address = '*****@*****.**'
to_address = '*****@*****.**'
bot_name = os.environ['BUILDBOT_BUILDERNAME']
slave_name = os.environ['BUILDBOT_SLAVENAME']
num_online_devs = len(adb_online_devs)
subject = 'Devices offline on %s, %s (%d remaining).' % (
slave_name, bot_name, num_online_devs)
buildbot_report.PrintWarning()
devices_missing_msg = '%d devices not detected.' % len(missing_devs)
buildbot_report.PrintSummaryText(devices_missing_msg)
# TODO(navabi): Debug by printing both output from GetCmdOutput and
# GetAttachedDevices to compare results.
body = '\n'.join([
'Current online devices: %s' % adb_online_devs,
'%s are no longer visible. Were they removed?\n' % missing_devs,
'SHERIFF: See go/chrome_device_monitor',
'Cache file: %s\n\n' % last_devices_path,
'adb devices: %s' % GetCmdOutput(['adb', 'devices']),
'adb devices(GetAttachedDevices): %s' % GetAttachedDevices()
])
print body
# Only send email if the first time a particular device goes offline
last_missing = ReadDeviceList('.last_missing')
new_missing_devs = set(missing_devs) - set(last_missing)
if new_missing_devs:
msg_body = '\r\n'.join([
'From: %s' % from_address,
'To: %s' % to_address,
'Subject: %s' % subject, '', body
])
try:
server = smtplib.SMTP('localhost')
server.sendmail(from_address, [to_address], msg_body)
server.quit()
except Exception as e:
print 'Failed to send alert email. Error: %s' % e
else:
new_devs = set(adb_online_devs) - set(last_devices)
if new_devs and os.path.exists(last_devices_path):
buildbot_report.PrintWarning()
buildbot_report.PrintSummaryText('%d new devices detected' %
len(new_devs))
print(
'New devices detected %s. And now back to your '
'regularly scheduled program.' % list(new_devs))
WriteDeviceList('.last_devices', (adb_online_devs + last_devices))
WriteDeviceList('.last_missing', missing_devs)
def CheckForMissingDevices(options, adb_online_devs):
"""Uses file of previous online devices to detect broken phones.
Args:
options: out_dir parameter of options argument is used as the base
directory to load and update the cache file.
adb_online_devs: A list of serial numbers of the currently visible
and online attached devices.
"""
# TODO(navabi): remove this once the bug that causes different number
# of devices to be detected between calls is fixed.
logger = logging.getLogger()
logger.setLevel(logging.INFO)
out_dir = os.path.abspath(options.out_dir)
def ReadDeviceList(file_name):
devices_path = os.path.join(out_dir, file_name)
devices = []
try:
with open(devices_path) as f:
devices = f.read().splitlines()
except IOError:
# Ignore error, file might not exist
pass
return devices
def WriteDeviceList(file_name, device_list):
path = os.path.join(out_dir, file_name)
if not os.path.exists(out_dir):
os.makedirs(out_dir)
with open(path, 'w') as f:
# Write devices currently visible plus devices previously seen.
f.write('\n'.join(set(device_list)))
last_devices_path = os.path.join(out_dir, '.last_devices')
last_devices = ReadDeviceList('.last_devices')
missing_devs = list(set(last_devices) - set(adb_online_devs))
all_known_devices = list(set(adb_online_devs) | set(last_devices))
WriteDeviceList('.last_devices', all_known_devices)
WriteDeviceList('.last_missing', missing_devs)
if not all_known_devices:
# This can happen if for some reason the .last_devices file is not
# present or if it was empty.
return ['No online devices. Have any devices been plugged in?']
if missing_devs:
devices_missing_msg = '%d devices not detected.' % len(missing_devs)
buildbot_report.PrintSummaryText(devices_missing_msg)
# TODO(navabi): Debug by printing both output from GetCmdOutput and
# GetAttachedDevices to compare results.
return [
'Current online devices: %s' % adb_online_devs,
'%s are no longer visible. Were they removed?\n' % missing_devs,
'SHERIFF: See go/chrome_device_monitor',
'Cache file: %s\n\n' % last_devices_path,
'adb devices: %s' % GetCmdOutput(['adb', 'devices']),
'adb devices(GetAttachedDevices): %s' %
android_commands.GetAttachedDevices()
]
else:
new_devs = set(adb_online_devs) - set(last_devices)
if new_devs and os.path.exists(last_devices_path):
buildbot_report.PrintWarning()
buildbot_report.PrintSummaryText('%d new devices detected' %
len(new_devs))
print(
'New devices detected %s. And now back to your '
'regularly scheduled program.' % list(new_devs))
