def _update_via_p2p(self, host, update_url):
"""
Update the second DUT via P2P from the first DUT.
We perform a non-interactive update and update_engine will check
for other devices that have P2P enabled and download from them instead.
@param host: The second DUT.
@param update_url: the url to call for updating the DUT.
"""
logging.info('Updating second host via p2p.')
host.reboot()
self._set_active_p2p_host(self._hosts[1])
utils.poll_for_condition(condition=self._is_update_engine_idle,
desc='Waiting for update engine idle')
try:
# Start a non-interactive update which is required for p2p.
updater = autoupdater.ChromiumOSUpdater(update_url, host,
interactive=False)
updater.update_image()
except autoupdater.RootFSUpdateError:
logging.exception('Failed to update the second DUT via P2P.')
raise error.TestFail('Failed to update the second DUT. Error: %s' %
self._get_last_error_string())
finally:
logging.info('Saving update engine logs to results dir.')
host.get_file(self._UPDATE_ENGINE_LOG,
os.path.join(self.resultsdir,
'update_engine.log_second_dut'))
# Return the update_engine logs so we can check for p2p entries.
return host.run('cat %s' % self._UPDATE_ENGINE_LOG).stdout
def machine_install_and_update_labels(host,
update_url,
use_quick_provision=False,
with_cheets=False):
"""Install a build and update the version labels on a host.
@param host: Host object where the build is to be installed.
@param update_url: URL of the build to install.
@param use_quick_provision: If true, then attempt to use
quick-provision for the update.
@param with_cheets: If true, installation is for a specific, custom
version of Android for a target running ARC.
"""
info = host.host_info_store.get()
info.clear_version_labels()
_clear_host_attributes_before_provision(host, info)
host.host_info_store.commit(info)
updater = autoupdater.ChromiumOSUpdater(
update_url, host=host, use_quick_provision=use_quick_provision)
image_name, host_attributes = updater.run_update()
info = host.host_info_store.get()
info.attributes.update(host_attributes)
if with_cheets:
image_name += provision.CHEETS_SUFFIX
info.set_version_label(host.VERSION_PREFIX, image_name)
host.host_info_store.commit(info)
def _update_dut(self, host, update_url):
"""
Update the first DUT normally and save the update engine logs.
@param host: the host object for the first DUT.
@param update_url: the url to call for updating the DUT.
"""
logging.info('Updating first DUT with a regular update.')
host.reboot()
# Sometimes update request is lost if checking right after reboot so
# make sure update_engine is ready.
self._set_active_p2p_host(self._hosts[0])
utils.poll_for_condition(condition=self._is_update_engine_idle,
desc='Waiting for update engine idle')
try:
updater = autoupdater.ChromiumOSUpdater(update_url, host)
updater.update_image()
except autoupdater.RootFSUpdateError:
logging.exception('Failed to update the first DUT.')
raise error.TestFail('Updating the first DUT failed. Error: %s.' %
self._get_last_error_string())
finally:
logging.info('Saving update engine logs to results dir.')
host.get_file(self._UPDATE_ENGINE_LOG,
os.path.join(self.resultsdir,
'update_engine.log_first_dut'))
host.reboot()
def testGetRemoteScript(self):
"""Test _get_remote_script() behaviors."""
update_url = ('http://172.22.50.205:8082/update/lumpy-chrome-perf/'
'R28-4444.0.0-b2996')
script_name = 'fubar'
local_script = '/usr/local/bin/%s' % script_name
host = self.mox.CreateMockAnything()
updater = autoupdater.ChromiumOSUpdater(update_url, host=host)
host.path_exists(local_script).AndReturn(True)
self.mox.ReplayAll()
# Simple case: file exists on DUT
self.assertEqual(updater._get_remote_script(script_name),
local_script)
self.mox.VerifyAll()
self.mox.ResetAll()
fake_shell = '/bin/ash'
tmp_script = '/tmp/%s' % script_name
fake_result = self.mox.CreateMockAnything()
fake_result.stdout = ' %s\n' % fake_shell
host.path_exists(local_script).AndReturn(False)
host.run(mox.IgnoreArg(),
ignore_status=True).AndReturn(fake_result)
self.mox.ReplayAll()
# Complicated case: script not on DUT, so try to download it.
self.assertEqual(
updater._get_remote_script(script_name),
'%s %s' % (fake_shell, tmp_script))
self.mox.VerifyAll()
def testUpdateStateful(self):
"""Tests that we call the stateful update script with the correct args.
"""
self.mox.StubOutWithMock(autoupdater.ChromiumOSUpdater, '_run')
self.mox.StubOutWithMock(autoupdater.ChromiumOSUpdater,
'_get_stateful_update_script')
update_url = ('http://172.22.50.205:8082/update/lumpy-chrome-perf/'
'R28-4444.0.0-b2996')
static_update_url = ('http://172.22.50.205:8082/static/'
'lumpy-chrome-perf/R28-4444.0.0-b2996')
update_script = '/usr/local/bin/stateful_update'
# Test with clobber=False.
autoupdater.ChromiumOSUpdater._get_stateful_update_script().AndReturn(
update_script)
autoupdater.ChromiumOSUpdater._run(
mox.And(
mox.StrContains(update_script),
mox.StrContains(static_update_url),
mox.Not(mox.StrContains('--stateful_change=clean'))),
timeout=mox.IgnoreArg())
self.mox.ReplayAll()
updater = autoupdater.ChromiumOSUpdater(update_url)
updater.update_stateful(clobber=False)
self.mox.VerifyAll()
# Test with clobber=True.
self.mox.ResetAll()
autoupdater.ChromiumOSUpdater._get_stateful_update_script().AndReturn(
update_script)
autoupdater.ChromiumOSUpdater._run(
mox.And(
mox.StrContains(update_script),
mox.StrContains(static_update_url),
mox.StrContains('--stateful_change=clean')),
timeout=mox.IgnoreArg())
self.mox.ReplayAll()
updater = autoupdater.ChromiumOSUpdater(update_url)
updater.update_stateful(clobber=True)
self.mox.VerifyAll()
def testRollbackRootfs(self):
"""Tests that we correctly rollback the rootfs when requested."""
self.mox.StubOutWithMock(autoupdater.ChromiumOSUpdater, '_run')
self.mox.StubOutWithMock(autoupdater.ChromiumOSUpdater,
'_verify_update_completed')
host = self.mox.CreateMockAnything()
update_url = 'http://server/test/url'
host.hostname = 'test_host'
can_rollback_cmd = ('/usr/bin/update_engine_client --can_rollback')
rollback_cmd = ('/usr/bin/update_engine_client --rollback '
'--follow')
updater = autoupdater.ChromiumOSUpdater(update_url, host=host)
# Return an old build which shouldn't call can_rollback.
updater.host.get_release_version().AndReturn('1234.0.0')
autoupdater.ChromiumOSUpdater._run(rollback_cmd)
autoupdater.ChromiumOSUpdater._verify_update_completed()
self.mox.ReplayAll()
updater.rollback_rootfs(powerwash=True)
self.mox.VerifyAll()
self.mox.ResetAll()
cmd_result_1 = self.mox.CreateMockAnything()
cmd_result_1.exit_status = 1
# Rollback but can_rollback says we can't -- return an error.
updater.host.get_release_version().AndReturn('5775.0.0')
autoupdater.ChromiumOSUpdater._run(can_rollback_cmd).AndRaise(
error.AutoservRunError('can_rollback failed', cmd_result_1))
self.mox.ReplayAll()
self.assertRaises(autoupdater.RootFSUpdateError,
updater.rollback_rootfs, True)
self.mox.VerifyAll()
self.mox.ResetAll()
# Rollback >= version blacklisted.
updater.host.get_release_version().AndReturn('5775.0.0')
autoupdater.ChromiumOSUpdater._run(can_rollback_cmd)
autoupdater.ChromiumOSUpdater._run(rollback_cmd)
autoupdater.ChromiumOSUpdater._verify_update_completed()
self.mox.ReplayAll()
updater.rollback_rootfs(powerwash=True)
self.mox.VerifyAll()
def update_device(self, device_host):
"""Update router and pcap associated with host.
@param device_host: router / pcap host object
@param device_board: router / pcap board name
"""
device_board = device_host.get_board().split(':', 1)[1]
desired = self.STABLE_VERSIONS.get(device_board, None)
if desired is None:
raise error.TestFail(
'No stable version found for %s with board=%s.' %
(device_host.hostname, device_board))
logging.info('Checking whether %s is at the latest stable version: %s',
device_host.hostname, desired.release_version)
current_release_version = self.get_release_version(device_host)
if desired.release_version == current_release_version:
raise error.TestNAError(
'%s is already at latest version %s.' %
(device_host.hostname, desired.release_version))
logging.info('Updating %s to image %s from %s', device_host.hostname,
desired.release_version, current_release_version)
logging.info('Staging artifacts.')
try:
ds = dev_server.ImageServer.resolve(desired.builder_version,
device_host.hostname)
ds.stage_artifacts(desired.builder_version,
['full_payload', 'stateful'])
except dev_server.DevServerException as e:
logging.error(e)
raise error.TestFail(str(e))
url = self.get_update_url(ds.url(), desired.builder_version)
autoupdater.ChromiumOSUpdater(url, host=device_host).run_update()
def get_updater_from_repo_url(host, job_repo_url=None):
"""Returns the autoupdater instance to use for a given autoupdate test.
All server-side tests that run in the lab have an associated job_repo_url
assigned to their host that is associated with the version of the build that
is currently installed on them. Given most autoupdate tests need to
update to some build as part of the test, we conveniently re-update to the
same version installed. This method serves as a helper to get the
instantiated autoupdater instance for that build.
This method guarantees that the devserver associated with the autoupdater
has already staged the necessary files for autoupdate.
@param host: The host for the DUT of the server-side test.
@param job_repo_url: If set, the job_repo_url to use.
@raise error.TestError: If we fail to get a job_repo_url.
"""
# Get the job_repo_url -- if not present, attempt to use the one
# specified in the host attributes for the host.
if not job_repo_url:
info = host.host_info_store.get()
job_repo_url = info.attributes.get(host.job_repo_url_attribute, '')
if not job_repo_url:
raise error.TestError(
'Could not find a job_repo_url for the given host.')
# Get the devserver url and build (image) from the repo url e.g.
# 'http://mydevserver:8080', 'x86-alex-release/R27-123.0.0'
ds_url, build = tools.get_devserver_build_from_package_url(job_repo_url)
devserver = dev_server.ImageServer(ds_url)
devserver.stage_artifacts(build, ['full_payload', 'stateful'])
# We only need to update stateful to do this test.
return autoupdater.ChromiumOSUpdater(devserver.get_update_url(build),
host=host)
def testTriggerUpdate(self):
"""Tests that we correctly handle updater errors."""
update_url = 'http://server/test/url'
self.host = self.mox.CreateMockAnything()
self.mox.StubOutWithMock(self.host, 'run')
self.mox.StubOutWithMock(autoupdater.ChromiumOSUpdater,
'_get_last_update_error')
self.host.hostname = 'test_host'
updater_control_bin = '/usr/bin/update_engine_client'
test_url = 'http://server/test/url'
expected_wait_cmd = ('%s -status | grep CURRENT_OP' %
updater_control_bin)
expected_cmd = ('%s --check_for_update --omaha_url=%s' %
(updater_control_bin, test_url))
self.mox.StubOutWithMock(time, "sleep")
UPDATE_ENGINE_RETRY_WAIT_TIME=5
# Generic SSH Error.
cmd_result_255 = self.mox.CreateMockAnything()
cmd_result_255.exit_status = 255
# Command Failed Error
cmd_result_1 = self.mox.CreateMockAnything()
cmd_result_1.exit_status = 1
# Error 37
cmd_result_37 = self.mox.CreateMockAnything()
cmd_result_37.exit_status = 37
updater = autoupdater.ChromiumOSUpdater(update_url, host=self.host)
# (SUCCESS) Expect one wait command and one status command.
self._host_run_for_update(expected_wait_cmd)
self._host_run_for_update(expected_cmd)
# (SUCCESS) Test with one retry to wait for update-engine.
self._host_run_for_update(expected_wait_cmd, exception=
error.AutoservRunError('non-zero status', cmd_result_1))
time.sleep(UPDATE_ENGINE_RETRY_WAIT_TIME)
self._host_run_for_update(expected_wait_cmd)
self._host_run_for_update(expected_cmd)
# (SUCCESS) One-time SSH timeout, then success on retry.
self._host_run_for_update(expected_wait_cmd)
self._host_run_for_update(expected_cmd, exception=
error.AutoservSSHTimeout('ssh timed out', cmd_result_255))
self._host_run_for_update(expected_cmd)
# (SUCCESS) One-time ERROR 37, then success.
self._host_run_for_update(expected_wait_cmd)
self._host_run_for_update(expected_cmd, exception=
error.AutoservRunError('ERROR_CODE=37', cmd_result_37))
self._host_run_for_update(expected_cmd)
# (FAILURE) Bad status of update engine.
self._host_run_for_update(expected_wait_cmd)
self._host_run_for_update(expected_cmd, bad_update_status=True,
exception=error.InstallError(
'host is not in installable state'))
# (FAILURE) Two-time SSH timeout.
self._host_run_for_update(expected_wait_cmd)
self._host_run_for_update(expected_cmd, exception=
error.AutoservSSHTimeout('ssh timed out', cmd_result_255))
self._host_run_for_update(expected_cmd, exception=
error.AutoservSSHTimeout('ssh timed out', cmd_result_255))
# (FAILURE) SSH Permission Error
self._host_run_for_update(expected_wait_cmd)
self._host_run_for_update(expected_cmd, exception=
error.AutoservSshPermissionDeniedError('no permission',
cmd_result_255))
# (FAILURE) Other ssh failure
self._host_run_for_update(expected_wait_cmd)
self._host_run_for_update(expected_cmd, exception=
error.AutoservSshPermissionDeniedError('no permission',
cmd_result_255))
# (FAILURE) Other error
self._host_run_for_update(expected_wait_cmd)
self._host_run_for_update(expected_cmd, exception=
error.AutoservRunError("unknown error", cmd_result_1))
self.mox.ReplayAll()
# Expect success
updater.trigger_update()
updater.trigger_update()
updater.trigger_update()
updater.trigger_update()
# Expect errors as listed above
self.assertRaises(autoupdater.RootFSUpdateError, updater.trigger_update)
self.assertRaises(autoupdater.RootFSUpdateError, updater.trigger_update)
self.assertRaises(autoupdater.RootFSUpdateError, updater.trigger_update)
self.assertRaises(autoupdater.RootFSUpdateError, updater.trigger_update)
self.assertRaises(autoupdater.RootFSUpdateError, updater.trigger_update)
self.mox.VerifyAll()
def update_image(self, wait_for_update=False):
"""Update the image on the servo host, if needed.
This method recognizes the following cases:
* If the Host is not running Chrome OS, do nothing.
* If a previously triggered update is now complete, reboot
to the new version.
* If the host is processing a previously triggered update,
do nothing.
* If the host is running a version of Chrome OS different
from the default for servo Hosts, trigger an update, but
don't wait for it to complete.
@param wait_for_update If an update needs to be applied and
this is true, then don't return until the update is
downloaded and finalized, and the host rebooted.
@raises dev_server.DevServerException: If all the devservers are down.
@raises site_utils.ParseBuildNameException: If the devserver returns
an invalid build name.
@raises AutoservRunError: If the update_engine_client isn't present on
the host, and the host is a cros_host.
"""
# servod could be running in a Ubuntu workstation.
if not self.is_cros_host():
logging.info(
'Not attempting an update, either %s is not running '
'chromeos or we cannot find enough information about '
'the host.', self.hostname)
return
if lsbrelease_utils.is_moblab():
logging.info('Not attempting an update, %s is running moblab.',
self.hostname)
return
target_build = afe_utils.get_stable_cros_image_name(self.get_board())
target_build_number = server_utils.ParseBuildName(target_build)[3]
# For servo image staging, we want it as more widely distributed as
# possible, so that devservers' load can be evenly distributed. So use
# hostname instead of target_build as hash.
ds = dev_server.ImageServer.resolve(self.hostname,
hostname=self.hostname)
url = ds.get_update_url(target_build)
updater = autoupdater.ChromiumOSUpdater(update_url=url, host=self)
self._maybe_reboot_post_upgrade(updater)
current_build_number = self._get_release_version()
status = updater.check_update_status()
update_pending = True
if status in autoupdater.UPDATER_PROCESSING_UPDATE:
logging.info(
'servo host %s already processing an update, update '
'engine client status=%s', self.hostname, status)
elif status == autoupdater.UPDATER_NEED_REBOOT:
return
elif current_build_number != target_build_number:
logging.info(
'Using devserver url: %s to trigger update on '
'servo host %s, from %s to %s', url, self.hostname,
current_build_number, target_build_number)
try:
ds.stage_artifacts(target_build, artifacts=['full_payload'])
except Exception as e:
logging.error('Staging artifacts failed: %s', str(e))
logging.error('Abandoning update for this cycle.')
else:
try:
updater.trigger_update()
except autoupdater.RootFSUpdateError as e:
trigger_download_status = 'failed with %s' % str(e)
metrics.Counter('chromeos/autotest/servo/'
'rootfs_update_failed').increment()
else:
trigger_download_status = 'passed'
logging.info(
'Triggered download and update %s for %s, '
'update engine currently in status %s',
trigger_download_status, self.hostname,
updater.check_update_status())
else:
logging.info('servo host %s does not require an update.',
self.hostname)
update_pending = False
if update_pending and wait_for_update:
logging.info('Waiting for servo update to complete.')
self.run('update_engine_client --follow', ignore_status=True)