def testRetryExceptionBadArgs(self):
"""Verify we reject non-classes or tuples of classes"""
with self.assertRaises(TypeError):
retry_util.RetryException('', 3, map)
with self.assertRaises(TypeError):
retry_util.RetryException(123, 3, map)
with self.assertRaises(TypeError):
retry_util.RetryException(None, 3, map)
with self.assertRaises(TypeError):
retry_util.RetryException([None], 3, map)
def vpd_set(host,
vpd_dict,
partition='RW_VPD',
dump=False,
force_dump=False,
retries=3):
"""
Sets the given key/value pairs in the specified VPD partition.
@param host: Host to run the command on.
@param vpd_dict: Dictionary containing the VPD key/value pairs to set.
Dictionary keys should be the VPD key strings, and values
should be the desired values to write.
@param partition: Which partition to access. 'RO_VPD' or 'RW_VPD'.
@param dump: If True, also run dump_vpd_log command after setting the
vpd values.
@param force_dump: Whether or not to forcefully dump the vpd log.
@param retries: Number of times to try rerunning the command in case of
error.
"""
_check_partition(partition)
for vpd_key in vpd_dict:
set_cmd = _VPD_BASE_CMD % (partition, '-s',
(vpd_key + '=' + str(vpd_dict[vpd_key])))
retry_util.RetryException(error.AutoservRunError, retries, host.run,
set_cmd).stdout
if dump:
dump_vpd_log(host, force=force_dump, retries=retries)
def Send(self, message):
"""Send an email via SMTP
If we get a socket error (e.g. the SMTP server is not listening or
timesout), we will retry a few times. All socket errors will be
caught here.
Args:
message: A MIMEMultipart() object containing the body of the message.
Returns:
True if the email was sent, else False.
"""
def _Send():
smtp_client = smtplib.SMTP(self._smtp_server)
recipients = [s.strip() for s in message['To'].split(',')]
smtp_client.sendmail(message['From'], recipients,
message.as_string())
smtp_client.quit()
try:
retry_util.RetryException(socket.error,
self.SMTP_RETRY_COUNT,
_Send,
sleep=self.SMTP_RETRY_DELAY)
return True
except socket.error as e:
logging.warning('Could not send e-mail from %s to %s via %r: %s',
message['From'], message['To'], self._smtp_server,
e)
return False
def _WaitForSSHPort(self, sleep=5):
"""Wait for SSH port to become available."""
class _SSHPortInUseError(Exception):
"""Exception for _CheckSSHPortBusy to throw."""
def _CheckSSHPortBusy(ssh_port):
"""Check if the SSH port is in use."""
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
sock.bind((remote_access.LOCALHOST_IP, ssh_port))
except socket.error as e:
if e.errno == errno.EADDRINUSE:
logging.info('SSH port %d in use...', self.ssh_port)
raise _SSHPortInUseError()
finally:
sock.close()
try:
retry_util.RetryException(
exception=_SSHPortInUseError,
max_retry=10,
functor=lambda: _CheckSSHPortBusy(self.ssh_port),
sleep=sleep)
except _SSHPortInUseError:
raise VMError('SSH port %d in use' % self.ssh_port)
def close(self):
if self.dev:
for part in list(self._mounted):
self._Unmount(part)
# We still need to remove some directories, since _Unmount did not.
for link in self._symlinks:
osutils.SafeUnlink(link)
self._symlinks = set()
for path in self._to_be_rmdir:
retry_util.RetryException(cros_build_lib.RunCommandError,
60,
osutils.RmDir,
path,
sudo=True,
sleep=1)
self._to_be_rmdir = set()
cmd = ['partx', '-d', self.dev]
cros_build_lib.sudo_run(cmd, quiet=True, error_code_ok=True)
cros_build_lib.sudo_run(['losetup', '--detach', self.dev])
self.dev = None
self.parts = {}
self._gpt_table = None
if self._destination_created:
self.destination = None
self._destination_created = False
def TransferDevServerPackage(self):
"""Transfer devserver package to work directory of the remote device."""
retry_util.RetryException(cros_build_lib.RunCommandError,
MAX_RETRY,
super(ChromiumOSUpdater,
self).TransferDevServerPackage,
delay_sec=DELAY_SEC_FOR_RETRY)
def createPatch(self, clone_path, project, **kwargs):
"""Create a patch in the given git checkout and upload it to gerrit.
Args:
clone_path: The directory on disk of the git clone.
project: The associated project.
**kwargs: Additional keyword arguments to pass to createCommit.
Returns:
A GerritPatch object.
"""
(revision, changeid) = self.createCommit(clone_path, **kwargs)
self.uploadChange(clone_path)
def PatchQuery():
return self._GetHelper().QuerySingleRecord(change=changeid,
project=project,
branch='master')
# 'RetryException' is needed because there is a race condition between
# uploading the change and querying for the change.
gpatch = retry_util.RetryException(gerrit.QueryHasNoResults,
5,
PatchQuery,
sleep=1)
self.assertEqual(gpatch.change_id, changeid)
self.assertEqual(gpatch.revision, revision)
return gpatch
def testRetryException(self):
"""Verify we retry only when certain exceptions get thrown"""
source = functools.partial(next, iter(range(6)))
def _TestMain():
val = source()
if val < 2:
raise OSError()
if val < 5:
raise ValueError()
return val
with self.assertRaises(OSError):
retry_util.RetryException((OSError, ValueError), 2, _TestMain)
with self.assertRaises(ValueError):
retry_util.RetryException((OSError, ValueError), 1, _TestMain)
self.assertEqual(5, retry_util.RetryException(ValueError, 1, _TestMain))
with self.assertRaises(StopIteration):
retry_util.RetryException(ValueError, 3, _TestMain)
def cleanup(self):
logging.info('Disabling p2p_update on hosts.')
for host in self._hosts:
try:
cmd = 'update_engine_client --p2p_update=no'
retry_util.RetryException(error.AutoservRunError, 2, host.run,
cmd)
except Exception:
logging.info('Failed to disable P2P in cleanup.')
super(autoupdate_P2P, self).cleanup()
def _enable_p2p_update_on_hosts(self):
"""Turn on the option to enable p2p updating on both DUTs."""
logging.info('Enabling p2p_update on hosts.')
for host in self._hosts:
try:
cmd = 'update_engine_client --p2p_update=yes'
retry_util.RetryException(error.AutoservRunError, 2, host.run,
cmd)
except Exception:
raise error.TestFail('Failed to enable p2p on %s' % host)
def TransferStatefulUpdate(self):
"""Transfer files for stateful update.
The stateful update bin and the corresponding payloads are copied to the
target remote device for stateful update.
"""
retry_util.RetryException(cros_build_lib.RunCommandError,
_MAX_RETRY,
self._TransferStatefulUpdate,
delay_sec=_DELAY_SEC_FOR_RETRY)
def _WaitForProc(exe, numpids):
try:
retry_util.RetryException(
exception=_TooFewPidsException,
max_retry=5,
functor=lambda: _GetRunningPids(exe, numpids),
sleep=sleep)
except _TooFewPidsException:
raise VMError('_WaitForProcs failed: timed out while waiting for '
'%d %s processes to start.' % (numpids, exe))
def TransferRootfsUpdate(self):
"""Transfer files for rootfs update.
The corresponding payload are copied to the remote device for rootfs
update.
"""
retry_util.RetryException(cros_build_lib.RunCommandError,
MAX_RETRY,
super(ChromiumOSUpdater,
self).TransferRootfsUpdate,
delay_sec=DELAY_SEC_FOR_RETRY)
def _change_cellular_setting_in_update_engine(self,
update_over_cellular=True):
"""
Toggles the update_over_cellular setting in update_engine.
@param update_over_cellular: True to enable, False to disable.
"""
answer = 'yes' if update_over_cellular else 'no'
cmd = 'update_engine_client --update_over_cellular=%s' % answer
retry_util.RetryException(error.AutoservRunError, 2, self._run, cmd)
def cleanup(self):
self._host.run('rm %s' % self._CUSTOM_LSB_RELEASE, ignore_status=True)
# Get the last two update_engine logs: before and after reboot.
files = self._host.run('ls -t -1 '
'/var/log/update_engine/').stdout.splitlines()
for i in range(2):
self._host.get_file('/var/log/update_engine/%s' % files[i],
self.resultsdir)
cmd = 'update_engine_client --update_over_cellular=no'
retry_util.RetryException(error.AutoservRunError, 2, self._host.run,
cmd)
super(autoupdate_ForcedOOBEUpdate, self).cleanup()
def dump_vpd_log(host, force=True, retries=3):
"""
Applies changes to the VPD settings by flushing them to the VPD cache and
output files.
@param host: Host to run the command on.
@param force: True to pass in the --force parameter to forcefully dump
the log. False to omit it.
@param retries: Number of times to try rerunning the command in case of
error.
"""
vpd_dump_cmd = 'dump_vpd_log%s' % (' --force' if force else '')
retry_util.RetryException(error.AutoservRunError, retries, host.run,
vpd_dump_cmd)
def Execute(self, func, **kwargs):
"""Build and execute the RPC to the Mob* Monitor.
Args:
func: The remote function to call.
kwargs: Arguments to above function.
Returns:
The result of the remote call.
"""
def urllib_call():
call = self.ConstructUrllibCall(func, **kwargs)
return urllib2.urlopen(call).read()
return retry_util.RetryException(urllib2.URLError, RPC_RETRY_TIMES,
urllib_call, sleep=RPC_SLEEP_SECS)
def WaitForBoot(self, sleep=5):
"""Wait for the device to boot up.
Wait for the ssh connection to become active.
"""
try:
result = retry_util.RetryException(
exception=remote_access.SSHConnectionError,
max_retry=10,
functor=lambda: self.RemoteCommand(cmd=['true']),
sleep=sleep)
except remote_access.SSHConnectionError:
raise DeviceError(
'WaitForBoot timed out trying to connect to the device.')
if result.returncode != 0:
raise DeviceError('WaitForBoot failed: %s.' % result.error)
def _RetryCommand(self, cmd, **kwargs):
"""Retry commands if SSHConnectionError happens.
Args:
cmd: the command to be run by device.
kwargs: the parameters for device to run the command.
Returns:
the output of running the command.
"""
return retry_util.RetryException(remote_access.SSHConnectionError,
MAX_RETRY,
self.device.run,
cmd,
delay_sec=DELAY_SEC_FOR_RETRY,
shell=isinstance(
cmd, six.string_types),
**kwargs)
def _UploadPerfValues(self, *args, **kwargs):
"""Helper for uploading perf values.
This currently handles common checks only. We could make perf values more
integrated in the overall stage running process in the future though if we
had more stages that cared about this.
"""
# Only upload perf data for buildbots as the data from local tryjobs
# probably isn't useful to us.
if not self._run.options.buildbot:
return
try:
retry_util.RetryException(perf_uploader.PerfUploadingError, 3,
perf_uploader.UploadPerfValues, *args,
**kwargs)
except perf_uploader.PerfUploadingError:
logging.exception('Uploading perf data failed')
def _ResetUpdateEngine(self):
"""Resets the host to prepare for a clean update regardless of state."""
self.device.RunCommand(['rm', '-f', self.UPDATED_MARKER],
**self._cmd_kwargs)
self.device.RunCommand(['stop', 'ui'], **self._cmd_kwargs_omit_error)
self.device.RunCommand(['stop', 'update-engine'],
**self._cmd_kwargs_omit_error)
self.device.RunCommand(['start', 'update-engine'], **self._cmd_kwargs)
status = retry_util.RetryException(Exception,
MAX_RETRY,
self.GetUpdateStatus,
self.device,
delay_sec=DELAY_SEC_FOR_RETRY)
if status[0] != UPDATE_STATUS_IDLE:
raise PreSetupUpdateError('%s is not in an installable state' %
self.device.hostname)
def vpd_delete(host, key, partition='RW_VPD', retries=3):
"""
Deletes the specified key from the specified VPD partition.
@param host: Host to run the command on.
@param key: The VPD value to delete.
@param partition: Which partition to access. 'RO_VPD' or 'RW_VPD'.
@param retries: Number of times to try rerunning the command in case of
error.
"""
_check_partition(partition)
if not vpd_get(host, key, partition=partition, retries=retries):
return
del_cmd = _VPD_BASE_CMD % (partition, '-d', key)
retry_util.RetryException(error.AutoservRunError, retries, host.run,
del_cmd).stdout
def __enter__(self):
fd, self._target_path = tempfile.mkstemp(prefix=self._path +
'.chromite.portablelock.')
os.close(fd)
try:
retry_util.RetryException(OSError,
self._max_retry,
os.link,
self._target_path,
self._path,
sleep=self._sleep)
except OSError:
raise LockNotAcquiredError('Timeout while trying to lock %s' %
self._path)
finally:
osutils.SafeUnlink(self._target_path)
return self
def _SendPerfValues(buildroot, sdk_tarball, buildbot_uri_log, version,
platform_name):
"""Generate & upload perf data for the build"""
perf_path = SdkPerfPath(buildroot)
test_name = 'sdk'
units = 'bytes'
# Make sure the file doesn't contain previous data.
osutils.SafeUnlink(perf_path)
common_kwargs = {
'higher_is_better': False,
'graph': 'cros-sdk-size',
'stdio_uri': buildbot_uri_log,
}
sdk_size = os.path.getsize(sdk_tarball)
perf_uploader.OutputPerfValue(perf_path, 'base', sdk_size, units,
**common_kwargs)
for tarball in glob.glob(
os.path.join(buildroot, constants.DEFAULT_CHROOT_DIR,
constants.SDK_TOOLCHAINS_OUTPUT, '*.tar.*')):
name = os.path.basename(tarball).rsplit('.', 2)[0]
size = os.path.getsize(tarball)
perf_uploader.OutputPerfValue(perf_path, name, size, units,
**common_kwargs)
perf_uploader.OutputPerfValue(perf_path, 'base_plus_%s' % name,
sdk_size + size, units,
**common_kwargs)
# Due to limitations in the perf dashboard, we have to create an integer
# based on the current timestamp. This field only accepts integers, and
# the perf dashboard accepts this or CrOS+Chrome official versions.
revision = int(version.replace('.', ''))
perf_values = perf_uploader.LoadPerfValues(perf_path)
retry_util.RetryException(perf_uploader.PerfUploadingError,
3,
perf_uploader.UploadPerfValues,
perf_values,
platform_name,
test_name,
revision=revision)
def vpd_get(host, key, partition='RW_VPD', retries=3):
"""
Gets the VPD value associated with the input key.
@param host: Host to run the command on.
@param key: Key of the desired VPD value.
@param partition: Which partition to access. 'RO_VPD' or 'RW_VPD'.
@param retries: Number of times to try rerunning the command in case of
error.
"""
_check_partition(partition)
get_cmd = _VPD_BASE_CMD % (partition, '-g', key)
try:
return retry_util.RetryException(error.AutoservRunError, retries,
host.run, get_cmd).stdout
except error.AutoservRunError as e:
if 'was not found' in str(e.result_obj.stderr):
return None
else:
raise e
def _enable_p2p_update_on_hosts(self):
"""Turn on the option to enable p2p updating on both DUTs."""
logging.info('Enabling p2p_update on hosts.')
for host in self._hosts:
try:
cmd = 'update_engine_client --p2p_update=yes'
retry_util.RetryException(error.AutoservRunError, 2, host.run,
cmd)
except Exception:
raise error.TestFail('Failed to enable p2p on %s' % host)
if self._too_many_attempts:
host.run('echo 11 > %s' % self._P2P_ATTEMPTS_FILE)
else:
host.run('rm %s' % self._P2P_ATTEMPTS_FILE, ignore_status=True)
if self._deadline_expired:
host.run('echo 1 > %s' % self._P2P_FIRST_ATTEMPT_FILE)
else:
host.run('rm %s' % self._P2P_FIRST_ATTEMPT_FILE,
ignore_status=True)
host.reboot()
def WaitForBoot(self):
"""Wait for the VM to boot up.
Wait for ssh connection to become active, and wait for all expected chrome
processes to be launched.
"""
if not os.path.exists(self.vm_dir):
self.Start()
try:
result = retry_util.RetryException(
exception=remote_access.SSHConnectionError,
max_retry=10,
functor=lambda: self.RemoteCommand(cmd=['echo']),
sleep=5)
except remote_access.SSHConnectionError:
raise VMError('WaitForBoot timed out trying to connect to VM.')
if result.returncode != 0:
raise VMError('WaitForBoot failed: %s.' % result.error)
# Chrome can take a while to start with software emulation.
if not self.enable_kvm:
self._WaitForProcs()
def run_once(self, host, job_repo_url=None, full_payload=True):
self._host = host
update_url = self.get_update_url_for_test(job_repo_url,
full_payload=full_payload,
cellular=True)
logging.info('Update url: %s', update_url)
# gs://chromeos-image-archive does not contain a handy .json file
# with the payloads size and SHA256 info like we have for payloads in
# gs://chromeos-releases. So in order to get this information we need
# to use a devserver to stage the payloads and then call the fileinfo
# API with the staged file URL. This will return the fields we need.
payload = self._get_payload_url(full_payload=full_payload)
staged_url = self._stage_payload_by_uri(payload)
payload_info = self._get_staged_file_info(staged_url)
cmd = 'update_engine_client --update_over_cellular=yes'
retry_util.RetryException(error.AutoservRunError, 2, self._host.run,
cmd)
client_at = autotest.Autotest(self._host)
client_at.run_test('autoupdate_CannedOmahaUpdate',
image_url=update_url,
image_size=payload_info['size'],
image_sha256=payload_info['sha256'],
use_cellular=True,
is_delta=not full_payload)
client_at._check_client_test_result(self._host,
'autoupdate_CannedOmahaUpdate')
# Verify there are cellular entries in the update_engine log.
update_engine_log = self._host.run('cat %s ' %
self._UPDATE_ENGINE_LOG_FILE).stdout
self._check_for_cellular_entries_in_update_log(update_engine_log)
def Run(self):
"""Retry the given stage multiple times to see if it passes."""
self.attempt = 1
retry_util.RetryException(failures_lib.RetriableStepFailure,
self.max_retry, self._PerformStage)
def TransferUpdateUtilsPackage(self):
"""Transfer update-utils package to work directory of the remote device."""
retry_util.RetryException(cros_build_lib.RunCommandError,
_MAX_RETRY,
self._TransferUpdateUtilsPackage,
delay_sec=_DELAY_SEC_FOR_RETRY)