def verify(self, host):
# Temporarily work around a problem caused by some old FSI
# builds that don't have the power_supply_info command by
# ignoring failures. The repair triggers believe that this
# verifier can't be fixed by re-installing, which means if a DUT
# gets stuck with one of those old builds, it can't be repaired.
#
# TODO(jrbarnette): This is for crbug.com/599158; we need a
# better solution.
try:
info = host.get_power_supply_info()
except:
logging.exception('get_power_supply_info() failed')
return
try:
if info['Line Power']['online'] != 'yes':
raise hosts.AutoservVerifyError('AC power is not plugged in')
except KeyError:
logging.info('Cannot determine AC power status - '
'skipping check.')
try:
if float(info['Battery']['percentage']) < 50.0:
raise hosts.AutoservVerifyError('Battery is less than 50%')
except KeyError:
logging.info('Cannot determine battery status - '
'skipping check.')
def verify(self, host):
# Test 1 - The DUT is not excluded from updates.
if not _is_firmware_update_supported(host):
return
# Test 2 - The DUT has an assigned stable firmware version.
info = host.host_info_store.get()
if info.model is None:
raise hosts.AutoservVerifyError(
'Can not verify firmware version. '
'No model label value found')
stable_firmware = afe_utils.get_stable_firmware_version(info.model)
if stable_firmware is None:
# This DUT doesn't have a firmware update target
return
# For tests 3 and 4: If the output from `crossystem` or
# `chromeos-firmwareupdate` isn't what we expect, we log an
# error, but don't fail: We don't want DUTs unable to test a
# build merely because of a bug or change in either of those
# commands.
# Test 3 - The DUT is not running the target stable firmware.
current_firmware = self._get_rw_firmware(host)
if current_firmware is None:
logging.error('DUT firmware version can\'t be determined.')
return
if current_firmware == stable_firmware:
return
# Test 4 - The firmware supplied in the running OS build is not
# the assigned stable firmware.
available_firmware = _get_available_firmware(host, info.model)
if available_firmware is None:
logging.error('Supplied firmware version in OS can\'t be '
'determined.')
return
if available_firmware != stable_firmware:
raise hosts.AutoservVerifyError(
'DUT firmware requires update from %s to %s' %
(current_firmware, stable_firmware))
# Time to update the firmware.
logging.info('Updating firmware from %s to %s',
current_firmware, stable_firmware)
self._check_hardware_match(current_firmware, stable_firmware)
try:
host.run('chromeos-firmwareupdate --mode=autoupdate')
host.reboot()
except Exception as e:
message = ('chromeos-firmwareupdate failed: from '
'%s to %s')
logging.exception(message, current_firmware, stable_firmware)
raise hosts.AutoservVerifyError(
message % (current_firmware, stable_firmware))
final_firmware = self._get_rw_firmware(host)
if final_firmware != stable_firmware:
message = ('chromeos-firmwareupdate failed: tried upgrade '
'to %s, now running %s instead')
raise hosts.AutoservVerifyError(
message % (stable_firmware, final_firmware))
def verify(self, host):
if _is_virual_machine(host):
# We do not forward host TPM / emulated TPM to qemu VMs, so skip
# this verification step.
logging.debug('Skipped verification %s on VM', self)
return
# This cryptohome command emits status information in JSON format. It
# looks something like this:
# {
# "installattrs": {
# ...
# },
# "mounts": [ {
# ...
# } ],
# "tpm": {
# "being_owned": false,
# "can_connect": true,
# "can_decrypt": false,
# "can_encrypt": false,
# "can_load_srk": true,
# "can_load_srk_pubkey": true,
# "enabled": true,
# "has_context": true,
# "has_cryptohome_key": false,
# "has_key_handle": false,
# "last_error": 0,
# "owned": true
# }
# }
output = host.run('cryptohome --action=status').stdout.strip()
try:
status = json.loads(output)
except ValueError:
logging.info('Cannot determine the Crytohome valid status - '
'skipping check.')
return
try:
tpm = status['tpm']
if not tpm['enabled']:
raise hosts.AutoservVerifyError(
'TPM is not enabled -- Hardware is not working.')
if not tpm['can_connect']:
raise hosts.AutoservVerifyError(
('TPM connect failed -- '
'last_error=%d.' % tpm['last_error']))
if tpm['owned'] and not tpm['can_load_srk']:
raise hosts.AutoservVerifyError('Cannot load the TPM SRK')
if tpm['can_load_srk'] and not tpm['can_load_srk_pubkey']:
raise hosts.AutoservVerifyError(
'Cannot load the TPM SRK public key')
except KeyError:
logging.info('Cannot determine the Crytohome valid status - '
'skipping check.')
def verify(self, host):
try:
if not host.upstart_status('ap-controller'):
raise hosts.AutoservVerifyError(
'ap-controller service is not running')
except error.AutoservRunError:
raise hosts.AutoservVerifyError('ap-controller service not found')
try:
host.run('pgrep ap-controller')
except error.AutoservRunError:
raise hosts.AutoservVerifyError(
'ap-controller process is not running')
def verify(self, host):
# pylint: disable=missing-docstring
try:
# This output is in text protobuf format.
result = host.run('cryptohome --action=tpm_more_status')
if 'attestation_prepared: true' not in result.stdout:
raise hosts.AutoservVerifyError(
'Attestation has not been prepared')
result = host.run('cryptohome --action=tpm_attestation_get_ek')
if 'EK Certificate' not in result.stdout:
raise hosts.AutoservVerifyError(
'Endorsement certificate not found')
except error.AutoservRunError:
raise hosts.AutoservVerifyError(
'Unable to fetch endorsement certificate')
def verify(self, host):
"""
Test whether the `host` has a `SERIAL` setting configured.
This tests the config file names used by the `servod` upstart
job for a valid setting of the `SERIAL` variable. The following
conditions raise errors:
* The SERIAL setting doesn't match the DUT's entry in the AFE
database.
* There is no config file.
"""
if not host.is_cros_host():
return
# Not all servo hosts will have a servo serial so don't verify if it's
# not set.
if host.servo_serial is None:
return
for config in self._get_configs(host):
serialval = self._get_config_val(host, config, self.ATTR)
if serialval is not None:
self._validate_attr(host, serialval, host.servo_serial,
self.ATTR, config)
return
msg = 'Servo serial is unconfigured; should be %s' % host.servo_serial
raise hosts.AutoservVerifyError(msg)
def repair(self, host):
host.get_servo().get_power_state_controller().reset()
# Get the lid_open value which requires EC console.
lid_open = host.get_servo().get('lid_open')
if lid_open != 'yes' and lid_open != 'not_applicable':
raise hosts.AutoservVerifyError(
'Still fail to contact EC console after rebooting DUT')
def verify(self, host):
"""
Test whether the `host` has a `BOARD` setting configured.
This tests the config file names used by the `servod` upstart
job for a valid setting of the `BOARD` variable. The following
conditions raise errors:
* A config file exists, but the content contains no setting
for BOARD.
* The BOARD setting doesn't match the DUT's entry in the AFE
database.
* There is no config file.
"""
if not host.is_cros_host():
return
for config in self._get_configs(host):
boardval = self._get_config_val(host, config, self.ATTR)
if boardval is not None:
self._validate_attr(host, boardval, host.servo_board,
self.ATTR, config)
return
msg = 'Servo board is unconfigured'
if host.servo_board is not None:
msg += '; should be %s' % host.servo_board
raise hosts.AutoservVerifyError(msg)
def verify(self, host):
if not _is_firmware_repair_supported(host):
return
try:
# Read the AP firmware and dump the sections that we're
# interested in.
cmd = ('mkdir /tmp/verify_firmware; '
'cd /tmp/verify_firmware; '
'for section in VBLOCK_A VBLOCK_B FW_MAIN_A FW_MAIN_B; '
'do flashrom -r image.bin -i $section:$section; '
'done')
host.run(cmd)
# Verify the firmware blocks A and B.
cmd = ('vbutil_firmware --verify /tmp/verify_firmware/VBLOCK_%c'
' --signpubkey /usr/share/vboot/devkeys/root_key.vbpubk'
' --fv /tmp/verify_firmware/FW_MAIN_%c')
for c in ('A', 'B'):
rv = host.run(cmd % (c, c), ignore_status=True)
if rv.exit_status:
raise hosts.AutoservVerifyError(
'Firmware %c is in a bad state.' % c)
finally:
# Remove the temporary files.
host.run('rm -rf /tmp/verify_firmware')
def verify(self, host):
if host.servo_board in self._BOARDS_WO_PWR_BUTTON:
return
button = host.get_servo().get('pwr_button')
if button != 'release':
raise hosts.AutoservVerifyError(
'Check ribbon cable: \'pwr_button\' is stuck')
def verify(self, host):
if not host.is_cros_host():
return
status_cmd = 'status servod PORT=%d' % host.servo_port
job_status = host.run(status_cmd, ignore_status=True).stdout
if 'start/running' not in job_status:
raise hosts.AutoservVerifyError(
'servod not running on %s port %d' %
(host.hostname, host.servo_port))
def verify(self, host):
# Some pools are allowed to be in dev mode
info = host.host_info_store.get()
if (bool(info.pools & _DEV_MODE_ALLOWED_POOLS)):
return
result = host.run('crossystem devsw_boot', ignore_status=True).stdout
if result != '0':
raise hosts.AutoservVerifyError('The host is in dev mode')
def verify(self, host):
# pylint: disable=missing-docstring
try:
info = host.get_power_supply_info()
except error.AutoservRunError:
raise hosts.AutoservVerifyError('Failed to get power supply info')
try:
if info['Line Power']['online'] != 'yes':
raise hosts.AutoservVerifyError('AC power is not plugged in')
except KeyError:
raise hosts.AutoservVerifyError('Cannot determine AC power status')
try:
if float(info['Battery']['percentage']) < 50.0:
raise hosts.AutoservVerifyError('Battery is less than 50%')
except KeyError:
logging.info('Cannot determine battery status - '
'skipping check.')
def verify(self, host):
# grep for stateful FS errors of the type "EXT4-fs error (device sda1):"
command = ("dmesg | grep -E \"EXT4-fs error \(device "
"$(cut -d ' ' -f 5,9 /proc/$$/mountinfo | "
"grep -e '^/mnt/stateful_partition ' | "
"cut -d ' ' -f 2 | cut -d '/' -f 3)\):\"")
output = host.run(command=command, ignore_status=True).stdout
if output:
sample = output.splitlines()[0]
message = 'Saw file system error: %s' % sample
raise hosts.AutoservVerifyError(message)
# Check for other critical FS errors.
command = 'dmesg | grep "This should not happen!! Data will be lost"'
output = host.run(command=command, ignore_status=True).stdout
if output:
message = 'Saw file system error: Data will be lost'
raise hosts.AutoservVerifyError(message)
else:
logging.error('Could not determine stateful mount.')
def verify(self, host):
result = host.run('python -c "import cPickle"', ignore_status=True)
if result.exit_status != 0:
message = 'The python interpreter is broken'
if result.exit_status == 127:
search = host.run('which python', ignore_status=True)
if search.exit_status != 0 or not search.stdout:
message = ('Python is missing; may be caused by '
'powerwash')
raise hosts.AutoservVerifyError(message)
def verify(self, host):
# This deliberately stops looking after the first error.
# See above for the details.
for testdir in self._TEST_DIRECTORIES:
filename = os.path.join(testdir, 'writable_test')
command = 'touch %s && rm %s' % (filename, filename)
rv = host.run(command=command, ignore_status=True)
if rv.exit_status != 0:
msg = 'Can\'t create a file in %s' % testdir
raise hosts.AutoservVerifyError(msg)
def _get_cryptohome_status(host):
"""Returns a dictionary containing the cryptohome status.
@param host: a hosts.Host object.
@returns A dictionary containing the cryptohome status.
@raises AutoservVerifyError: if the output could not be parsed or the TPM
status is missing.
@raises hosts.AutoservRunError: if the cryptohome command failed.
"""
# This cryptohome command emits status information in JSON format. It
# looks something like this:
# {
# "installattrs": {
# ...
# },
# "mounts": [ {
# ...
# } ],
# "tpm": {
# "being_owned": false,
# "can_connect": true,
# "can_decrypt": false,
# "can_encrypt": false,
# "can_load_srk": true,
# "can_load_srk_pubkey": true,
# "enabled": true,
# "has_context": true,
# "has_cryptohome_key": false,
# "has_key_handle": false,
# "last_error": 0,
# "owned": true
# }
# }
try:
output = host.run('cryptohome --action=status').stdout.strip()
status = json.loads(output)
if 'tpm' not in status:
raise hosts.AutoservVerifyError('TPM status is missing')
return status
except ValueError:
raise hosts.AutoservVerifyError('Unable to parse cryptohome status')
def verify(self, host):
if host.is_up():
return
msg = 'No answer to ssh from %s'
try:
socket.gethostbyname(host.hostname)
except Exception as e:
logging.exception('DNS lookup failure')
msg = 'Unable to look up %%s in DNS: %s' % e
else:
if utils.ping(host.hostname, tries=1, deadline=1) != 0:
msg = 'No answer to ping from %s'
raise hosts.AutoservVerifyError(msg % host.hostname)
def _validate_attr(host, val, expected_val, attr, config_file):
"""
Check that the attr setting is valid for the host.
This presupposes that a valid config file was found. Raise an
execption if:
* There was no attr setting from the file (i.e. the setting
is an empty string), or
* The attr setting is valid, the attr is known,
and the setting doesn't match the DUT.
@param host Host to be checked for `config_file`.
@param val Value to be tested.
@param expected_val Expected value.
@param attr Attribute we're validating.
@param config_file Path to the config file to be tested.
"""
if not val:
raise hosts.AutoservVerifyError('config file %s exists, but %s '
'is not set' % (attr, config_file))
if expected_val is not None and val != expected_val:
raise hosts.AutoservVerifyError('%s is %s; it should be %s' %
(attr, val, expected_val))
def connect_servo(self):
"""Establish a connection to the servod server on this host.
Initializes `self._servo` and then verifies that all network
connections are working. This will create an ssh tunnel if
it's required.
As a side effect of testing the connection, all signals on the
target servo are reset to default values, and the USB stick is
set to the neutral (off) position.
"""
servo_obj = servo.Servo(servo_host=self,
servo_serial=self.servo_serial)
timeout, _ = retry.timeout(
servo_obj.initialize_dut,
timeout_sec=self.INITIALIZE_SERVO_TIMEOUT_SECS)
if timeout:
raise hosts.AutoservVerifyError('Servo initialize timed out.')
self._servo = servo_obj
def _check_hardware_match(version_a, version_b):
"""
Check that two firmware versions identify the same hardware.
Firmware version strings look like this:
Google_Gnawty.5216.239.34
The part before the numbers identifies the hardware for which
the firmware was built. This function checks that the hardware
identified by `version_a` and `version_b` is the same.
This is a sanity check to protect us from installing the wrong
firmware on a DUT when a board label has somehow gone astray.
@param version_a First firmware version for the comparison.
@param version_b Second firmware version for the comparison.
"""
hardware_a = version_a.split('.')[0]
hardware_b = version_b.split('.')[0]
if hardware_a != hardware_b:
message = 'Hardware/Firmware mismatch updating %s to %s'
raise hosts.AutoservVerifyError(message % (version_a, version_b))
def verify(self, host):
# pylint: disable=missing-docstring
try:
status = CryptohomeStatus(host)
if not status.tpm_enabled:
raise hosts.AutoservVerifyError('TPM is not enabled')
if not status.tpm_owned:
raise hosts.AutoservVerifyError('TPM is not owned')
if not status.tpm_can_load_srk:
raise hosts.AutoservVerifyError('TPM cannot load SRK')
if not status.tpm_can_load_srk_pubkey:
raise hosts.AutoservVerifyError('TPM cannot load SRK pubkey')
# Check that the TPM is fully initialized. The output of this
# command is line-oriented property/value pairs.
result = host.run('cryptohome --action=tpm_status')
if 'TPM Ready: true' not in result.stdout:
raise hosts.AutoservVerifyError('TPM is not ready')
except error.AutoservRunError:
raise hosts.AutoservVerifyError('Could not determine TPM status')
def verify(self, host):
# pylint: disable=missing-docstring
result = host.run('[ ! -e /dev/kvm -a -f /usr/bin/vm_concierge ]',
ignore_status=True)
if result.exit_status == 0:
raise hosts.AutoservVerifyError('/dev/kvm is missing')
def verify(self, host):
# pylint: disable=missing-docstring
result = host.run('test -f %s' % autoupdater.PROVISION_FAILED,
ignore_status=True)
if result.exit_status == 0:
raise hosts.AutoservVerifyError('Last AU on this DUT failed')
def verify(self, host):
result = host.run('test -f %s' % host.PROVISION_FAILED,
ignore_status=True)
if result.exit_status == 0:
raise hosts.AutoservVerifyError('Last AU on this DUT failed')
def verify(self, host):
lid_open = host.get_servo().get('lid_open')
if lid_open != 'yes' and lid_open != 'not_applicable':
raise hosts.AutoservVerifyError(
'Check lid switch: lid_open is %s' % lid_open)