def extract(self, host):
"""Extract the kernel package.
This function is only useful to access the content of the
package (for example the kernel image) without
installing it. It is not necessary to run this function to
install the kernel.
Args:
host: the host on which to extract the kernel package.
Returns:
The full path to the temporary directory on host where
the package was extracted.
Raises:
AutoservError: no package has yet been obtained. Call
DEBKernel.get() with a .deb package.
"""
if self.source_material is None:
raise error.AutoservError("A kernel must first be "
"specified via get()")
remote_tmpdir = host.get_tmp_dir()
basename = os.path.basename(self.source_material)
remote_filename = os.path.join(remote_tmpdir, basename)
host.send_file(self.source_material, remote_filename)
content_dir= os.path.join(remote_tmpdir, "contents")
host.run('dpkg -x "%s" "%s"' % (utils.sh_escape(remote_filename),
utils.sh_escape(content_dir),))
return content_dir
def _run(self, command, timeout, ignore_status, stdout, stderr,
connect_timeout, env, options, stdin, args):
"""Helper function for run()."""
ssh_cmd = self.ssh_command(connect_timeout, options)
if not env.strip():
env = ""
else:
env = "export %s;" % env
for arg in args:
command += ' "%s"' % utils.sh_escape(arg)
full_cmd = '%s "%s %s"' % (ssh_cmd, env, utils.sh_escape(command))
result = utils.run(full_cmd, timeout, True, stdout, stderr,
verbose=False, stdin=stdin,
stderr_is_expected=ignore_status)
# The error messages will show up in band (indistinguishable
# from stuff sent through the SSH connection), so we have the
# remote computer echo the message "Connected." before running
# any command. Since the following 2 errors have to do with
# connecting, it's safe to do these checks.
if result.exit_status == 255:
if re.search(r'^ssh: connect to host .* port .*: '
r'Connection timed out\r$', result.stderr):
raise error.AutoservSSHTimeout("ssh timed out", result)
if "Permission denied." in result.stderr:
msg = "ssh permission denied"
raise error.AutoservSshPermissionDeniedError(msg, result)
if not ignore_status and result.exit_status > 0:
raise error.AutoservRunError("command execution error", result)
return result
def get_installed_autodir(cls, host):
"""
Find where the Autotest client is installed on the host.
:return: an absolute path to an installed Autotest client root.
:raise AutodirNotFoundError if no Autotest installation can be found.
"""
autodir = host.get_autodir()
if autodir:
logging.debug('Using existing host autodir: %s', autodir)
return autodir
if not _server_system_wide_install():
for path in Autotest.get_client_autodir_paths(host):
try:
autotest_binary = os.path.join(path, 'autotest')
host.run('test -x %s' % utils.sh_escape(autotest_binary))
host.run('test -w %s' % utils.sh_escape(path))
logging.debug('Found existing autodir at %s', path)
return path
except error.AutoservRunError:
logging.debug('%s does not exist on %s', autotest_binary,
host.hostname)
else:
for path in Autotest.get_client_autodir_paths(host):
host.run('test -w %s' % utils.sh_escape(path))
logging.debug('Found existing autodir at %s', path)
host.autodir = path
return path
raise AutodirNotFoundError
def install(self, host, **kwargs):
"""
Install a kernel on the remote host.
This will also invoke the guest's bootloader to set this
kernel as the default kernel.
Args:
host: the host on which to install the kernel
[kwargs]: remaining keyword arguments will be passed
to Bootloader.add_kernel()
Raises:
AutoservError: no package has yet been obtained. Call
DEBKernel.get() with a .deb package.
"""
if self.source_material is None:
raise error.AutoservError("A kernel must first be "
"specified via get()")
remote_tmpdir = host.get_tmp_dir()
basename = os.path.basename(self.source_material)
remote_filename = os.path.join(remote_tmpdir, basename)
host.send_file(self.source_material, remote_filename)
host.run('dpkg -i "%s"' % (utils.sh_escape(remote_filename),))
host.run('mkinitramfs -o "%s" "%s"' % (
utils.sh_escape(self.get_initrd_name()),
utils.sh_escape(self.get_version()),))
host.bootloader.add_kernel(self.get_image_name(),
initrd=self.get_initrd_name(), **kwargs)
def test_install(self):
self.common_code()
# record
self.host.run.expect_call('dpkg -i "%s"' %
(utils.sh_escape(self.remote_filename)))
result = common_utils.CmdResult()
result.stdout = "1"
utils.run.expect_call('dpkg-deb -f "%s" version' %
utils.sh_escape(self.kernel.source_material)).and_return(result)
utils.run.expect_call('dpkg-deb -f "%s" version' %
utils.sh_escape(self.kernel.source_material)).and_return(result)
self.host.run.expect_call('mkinitramfs -o "/boot/initrd.img-1" "1"')
utils.run.expect_call('dpkg-deb -f "%s" version' %
utils.sh_escape(self.kernel.source_material)).and_return(result)
utils.run.expect_call('dpkg-deb -f "%s" version' %
utils.sh_escape(self.kernel.source_material)).and_return(result)
self.host.bootloader.add_kernel.expect_call('/boot/vmlinuz-1',
initrd='/boot/initrd.img-1')
# run and check
self.kernel.install(self.host)
self.god.check_playback()
def add_args(self, kernel, args):
"""
Add cmdline arguments for the specified kernel.
:param kernel: can be a position number (index) or title
:param args: argument to be added to the current list of args
"""
return self._run_boottool_exit_status(
"--update-kernel=%s" % utils.sh_escape(str(kernel)), "--args=%s" % utils.sh_escape(args)
)
def remove_args(self, kernel, args):
"""
Removes specified cmdline arguments.
@param kernel: can be a position number (index) or title
@param args: argument to be removed of the current list of args
"""
return self._run_boottool_exit_status('--update-kernel=%s' %
utils.sh_escape(str(kernel)),
'--remove-args=%s' %
utils.sh_escape(args))
def _find_installable_dir(cls, host):
client_autodir_paths = cls.get_client_autodir_paths(host)
for path in client_autodir_paths:
try:
host.run('mkdir -p %s' % utils.sh_escape(path))
host.run('test -w %s' % utils.sh_escape(path))
return path
except error.AutoservRunError:
logging.debug('Failed to create %s', path)
raise error.AutoservInstallError(
'Unable to find a place to install Autotest; tried %s' %
', '.join(client_autodir_paths))
def test_extract(self):
# setup
self.common_code()
content_dir = os.path.join(self.remote_tmpdir, "contents")
# record
self.host.run.expect_call('dpkg -x "%s" "%s"' %
(utils.sh_escape(self.remote_filename),
utils.sh_escape(content_dir)))
# run and test
self.kernel.extract(self.host)
self.god.check_playback()
def _install(self, host=None, autodir=None, use_autoserv=True,
use_packaging=True):
"""
Install autotest. If get() was not called previously, an
attempt will be made to install from the autotest svn
repository.
@param host A Host instance on which autotest will be installed
@param autodir Location on the remote host to install to
@param use_autoserv Enable install modes that depend on the client
running with the autoserv harness
@param use_packaging Enable install modes that use the packaging system
@exception AutoservError If it wasn't possible to install the client
after trying all available methods
"""
if not host:
host = self.host
if not self.got:
self.get()
host.wait_up(timeout=30)
host.setup()
logging.info("Installing autotest on %s", host.hostname)
# set up the autotest directory on the remote machine
if not autodir:
autodir = self.get_install_dir(host)
logging.info('Using installation dir %s', autodir)
host.set_autodir(autodir)
host.run('mkdir -p %s' % utils.sh_escape(autodir))
# make sure there are no files in $AUTODIR/results
results_path = os.path.join(autodir, 'results')
host.run('rm -rf %s/*' % utils.sh_escape(results_path),
ignore_status=True)
# Fetch the autotest client from the nearest repository
if use_packaging:
try:
self._install_using_packaging(host, autodir)
self._create_test_output_dir(host, autodir)
logging.info("Installation of autotest completed")
self.installed = True
return
except (error.PackageInstallError, error.AutoservRunError,
global_config.ConfigError), e:
logging.info("Could not install autotest using the packaging "
"system: %s. Trying other methods", e)
def _install_using_send_file(self, host, autodir):
dirs_to_exclude = set(["tests", "site_tests", "deps", "profilers"])
light_files = [os.path.join(self.source_material, f)
for f in os.listdir(self.source_material)
if f not in dirs_to_exclude]
# needs updating when grubby version is changed
light_files.append(os.path.join(self.source_material,
"deps/grubby/grubby-8.13.tar.bz2"))
host.send_file(light_files, autodir, delete_dest=True)
profilers_autodir = os.path.join(autodir, 'profilers')
profilers_init = os.path.join(self.source_material, 'profilers',
'__init__.py')
host.run("mkdir -p %s" % profilers_autodir)
host.send_file(profilers_init, profilers_autodir, delete_dest=True)
dirs_to_exclude.discard("profilers")
# create empty dirs for all the stuff we excluded
commands = []
for path in dirs_to_exclude:
abs_path = os.path.join(autodir, path)
abs_path = utils.sh_escape(abs_path)
commands.append("mkdir -p '%s'" % abs_path)
commands.append("touch '%s'/__init__.py" % abs_path)
host.run(';'.join(commands))
def _client_system_wide_install(host):
for path in SYSTEM_WIDE_PATHS:
try:
host.run('test -x %s' % utils.sh_escape(path))
except:
return False
return True
def uninstall(self, host=None):
"""
Uninstall (i.e. delete) autotest. Removes the autotest client install
from the specified host.
@params host a Host instance from which the client will be removed
"""
if not self.installed:
return
if self.server_system_wide_install:
uninstall_cmd = UNINSTALL_CLIENT_CMD_MAPPING.get(self.os_vendor,
None)
if uninstall_cmd is not None:
logging.info("Trying to uninstall autotest using distro "
"provided package manager")
host.run(uninstall_cmd)
return
if not host:
host = self.host
autodir = host.get_autodir()
if not autodir:
return
# perform the actual uninstall
host.run("rm -rf %s" % utils.sh_escape(autodir), ignore_status=True)
host.set_autodir(None)
self.installed = False
def _install_using_send_file(self, host, autodir):
dirs_to_exclude = set(["tests", "site_tests", "deps", "profilers"])
light_files = [os.path.join(self.source_material, f)
for f in os.listdir(self.source_material)
if f not in dirs_to_exclude]
# there should be one and only one grubby tarball
grubby_glob = os.path.join(self.source_material,
"deps/grubby/grubby-*.tar.bz2")
grubby_tarball_paths = glob.glob(grubby_glob)
if grubby_tarball_paths:
grubby_tarball_path = grubby_tarball_paths[0]
if os.path.exists(grubby_tarball_path):
light_files.append(grubby_tarball_path)
host.send_file(light_files, autodir, delete_dest=True)
profilers_autodir = os.path.join(autodir, 'profilers')
profilers_init = os.path.join(self.source_material, 'profilers',
'__init__.py')
host.run("mkdir -p %s" % profilers_autodir)
host.send_file(profilers_init, profilers_autodir, delete_dest=True)
dirs_to_exclude.discard("profilers")
# create empty dirs for all the stuff we excluded
commands = []
for path in dirs_to_exclude:
abs_path = os.path.join(autodir, path)
abs_path = utils.sh_escape(abs_path)
commands.append("mkdir -p '%s'" % abs_path)
commands.append("touch '%s'/__init__.py" % abs_path)
host.run(';'.join(commands))
def boot_once(self, title):
if self._host().job:
self._host().job.last_boot_tag = title
title_opt = '--title=%s' % utils.sh_escape(title)
return self._run_boottool_exit_status('--boot-once',
title_opt)
def run_async(self, command, stdout_tee=None, stderr_tee=None, args=(),
connect_timeout=30, options='', verbose=True,
stderr_level=utils.DEFAULT_STDERR_LEVEL,
cmd_outside_subshell=''):
"""
Run a command on the remote host. Returns an AsyncJob object to
interact with the remote process.
This is mostly copied from SSHHost.run and SSHHost._run
"""
if verbose:
logging.debug("Running (async ssh) '%s'" % command)
# Start a master SSH connection if necessary.
self.start_master_ssh()
self.send_file(os.path.join(self.job.clientdir, "shared", "hosts",
"scripts", "run_helper.py"),
os.path.join(self.job.tmpdir, "run_helper.py"))
env = " ".join("=".join(pair) for pair in self.env.iteritems())
ssh_cmd = self.ssh_command(connect_timeout, options)
if not env.strip():
env = ""
else:
env = "export %s;" % env
for arg in args:
command += ' "%s"' % utils.sh_escape(arg)
full_cmd = '{ssh_cmd} "{env} {cmd}"'.format(
ssh_cmd=ssh_cmd, env=env,
cmd=utils.sh_escape("%s (%s '%s')" % (cmd_outside_subshell,
os.path.join(self.job.tmpdir, "run_helper.py"),
utils.sh_escape(command))))
job = utils.AsyncJob(full_cmd, stdout_tee=stdout_tee,
stderr_tee=stderr_tee, verbose=verbose,
stderr_level=stderr_level,
stdin=subprocess.PIPE)
def kill_func():
# this triggers the remote kill
utils.nuke_subprocess(job.sp)
job.kill_func = kill_func
return job
def _run_boottool_cmd(self, *options):
'''
Runs a boottool command, escaping parameters
'''
cmd = self._get_boottool_path()
# FIXME: add unsafe options strings sequence to host.run() parameters
for option in options:
cmd += ' "%s"' % utils.sh_escape(option)
return self._host().run(cmd)
def close(self):
super(RemoteHost, self).close()
self.stop_loggers()
if hasattr(self, 'tmp_dirs'):
for dir in self.tmp_dirs:
try:
self.run('rm -rf "%s"' % (utils.sh_escape(dir)))
except error.AutoservRunError:
pass
def _make_ssh_cmd(self, cmd):
"""
Create a base ssh command string for the host which can be used
to run commands directly on the machine
"""
base_cmd = make_ssh_command(user=self.user, port=self.port,
opts=self.master_ssh_option,
hosts_file=self.known_hosts_file)
return '%s %s "%s"' % (base_cmd, self.hostname, utils.sh_escape(cmd))
def set_default_by_index(self, index):
"""
Sets the given entry number to be the default on every next boot
To set a default only for the next boot, use boot_once() instead.
:param index: entry index number to set as the default.
"""
if self._host().job:
self._host().job.last_boot_tag = None
return self._run_boottool_exit_status("--set-default=%s" % utils.sh_escape(str(index)))
def test_get_version(self):
# record
result = common_utils.CmdResult()
result.exit_status = 0
result.stdout = "image"
cmd = "rpm -qpi %s | grep Version | awk '{print($3);}'" % (utils.sh_escape("source.rpm"))
utils.run.expect_call(cmd).and_return(result)
# run and test
self.assertEquals(self.kernel.get_version(), result.stdout)
self.god.check_playback()
def get_file(self, source, dest, delete_dest=False, preserve_perm=True,
preserve_symlinks=False):
"""
Copy files from the remote host to a local path.
Directories will be copied recursively.
If a source component is a directory with a trailing slash,
the content of the directory will be copied, otherwise, the
directory itself and its content will be copied. This
behavior is similar to that of the program 'rsync'.
Args:
source: either
1) a single file or directory, as a string
2) a list of one or more (possibly mixed)
files or directories
dest: a file or a directory (if source contains a
directory or more than one element, you must
supply a directory dest)
delete_dest: if this is true, the command will also clear
out any old files at dest that are not in the
source
preserve_perm: tells get_file() to try to preserve the sources
permissions on files and dirs
preserve_symlinks: try to preserve symlinks instead of
transforming them into files/dirs on copy
Raises:
AutoservRunError: the scp command failed
"""
# Start a master SSH connection if necessary.
self.start_master_ssh()
if isinstance(source, basestring):
source = [source]
dest = os.path.abspath(dest)
# If rsync is disabled or fails, try scp.
try_scp = True
if self.use_rsync():
try:
remote_source = self._encode_remote_paths(source)
local_dest = utils.sh_escape(dest)
rsync = self._make_rsync_cmd([remote_source], local_dest,
delete_dest, preserve_symlinks)
utils.run(rsync)
try_scp = False
except error.CmdError, e:
logging.warn("trying scp, rsync failed: %s" % e)
def get_installed_autodir(cls, host):
"""
Find where the Autotest client is installed on the host.
@returns an absolute path to an installed Autotest client root.
@raises AutodirNotFoundError if no Autotest installation can be found.
"""
autodir = host.get_autodir()
if autodir:
logging.debug('Using existing host autodir: %s', autodir)
return autodir
system_wide = True
autotest_system_wide = '/usr/bin/autotest-local'
try:
host.run('test -x %s' % utils.sh_escape(autotest_system_wide))
logging.info("System wide install detected")
except:
system_wide = False
for path in Autotest.get_client_autodir_paths(host):
try:
try:
autotest_binary = os.path.join(path, 'autotest')
host.run('test -x %s' % utils.sh_escape(autotest_binary))
except error.AutoservRunError:
if system_wide:
pass
else:
raise
host.run('test -w %s' % utils.sh_escape(path))
logging.debug('Found existing autodir at %s', path)
return path
except error.AutoservRunError:
logging.debug('%s does not exist on %s', autotest_binary,
host.hostname)
raise AutodirNotFoundError
def send_file(self, source, dest, delete_dest=False,
preserve_symlinks=False):
"""
Copy files from a local path to the remote host.
Directories will be copied recursively.
If a source component is a directory with a trailing slash,
the content of the directory will be copied, otherwise, the
directory itself and its content will be copied. This
behavior is similar to that of the program 'rsync'.
Args:
source: either
1) a single file or directory, as a string
2) a list of one or more (possibly mixed)
files or directories
dest: a file or a directory (if source contains a
directory or more than one element, you must
supply a directory dest)
delete_dest: if this is true, the command will also clear
out any old files at dest that are not in the
source
preserve_symlinks: controls if symlinks on the source will be
copied as such on the destination or transformed into the
referenced file/directory
Raises:
AutoservRunError: the scp command failed
"""
# Start a master SSH connection if necessary.
self.start_master_ssh()
if isinstance(source, basestring):
source_is_dir = os.path.isdir(source)
source = [source]
remote_dest = self._encode_remote_paths([dest])
# If rsync is disabled or fails, try scp.
try_scp = True
if self.use_rsync():
try:
local_sources = [utils.sh_escape(path) for path in source]
rsync = self._make_rsync_cmd(local_sources, remote_dest,
delete_dest, preserve_symlinks)
utils.run(rsync)
try_scp = False
except error.CmdError, e:
logging.warn("trying scp, rsync failed: %s" % e)
def get_version(self):
"""Get the version of the kernel to be installed.
Returns:
The version string, as would be returned
by 'make kernelrelease'.
Raises:
AutoservError: no package has yet been obtained. Call
RPMKernel.get() with a .rpm package.
"""
if self.source_material is None:
raise error.AutoservError("A kernel must first be \
specified via get()")
retval = utils.run('rpm -qpi %s | grep Version | awk \'{print($3);}\''
% utils.sh_escape(self.source_material))
return retval.stdout.strip()
def get_version(self):
"""Get the version of the kernel to be installed.
Returns:
The version string, as would be returned
by 'make kernelrelease'.
Raises:
AutoservError: no package has yet been obtained. Call
DEBKernel.get() with a .deb package.
"""
if self.source_material is None:
raise error.AutoservError("A kernel must first be "
"specified via get()")
retval= utils.run('dpkg-deb -f "%s" version' %
utils.sh_escape(self.source_material),)
return retval.stdout.strip()
def uninstall(self, host=None):
"""
Uninstall (i.e. delete) autotest. Removes the autotest client install
from the specified host.
@params host a Host instance from which the client will be removed
"""
if not self.installed:
return
if not host:
host = self.host
autodir = host.get_autodir()
if not autodir:
return
# perform the actual uninstall
host.run("rm -rf %s" % utils.sh_escape(autodir), ignore_status=True)
host.set_autodir(None)
self.installed = False
def _make_rsync_compatible_globs(self, path, is_local):
"""
Given an rsync-style path, returns a list of globbed paths
that will hopefully provide equivalent behaviour for scp. Does not
support the full range of rsync pattern matching behaviour, only that
exposed in the get/send_file interface (trailing slashes).
The is_local param is flag indicating if the paths should be
interpreted as local or remote paths.
"""
# non-trailing slash paths should just work
if len(path) == 0 or path[-1] != "/":
return [path]
# make a function to test if a pattern matches any files
if is_local:
def glob_matches_files(path, pattern):
return len(glob.glob(path + pattern)) > 0
else:
def glob_matches_files(path, pattern):
result = self.run("ls \"%s\"%s" % (utils.sh_escape(path),
pattern),
stdout_tee=None, ignore_status=True)
return result.exit_status == 0
# take a set of globs that cover all files, and see which are needed
patterns = ["*", ".[!.]*"]
patterns = [p for p in patterns if glob_matches_files(path, p)]
# convert them into a set of paths suitable for the commandline
if is_local:
return ["\"%s\"%s" % (utils.sh_escape(path), pattern)
for pattern in patterns]
else:
return [utils.scp_remote_escape(path) + pattern
for pattern in patterns]
def glob_matches_files(path, pattern):
result = self.run("ls \"%s\"%s" %
(utils.sh_escape(path), pattern),
stdout_tee=None,
ignore_status=True)
return result.exit_status == 0
def glob_matches_files(path, pattern):
result = self.run("ls \"%s\"%s" % (utils.sh_escape(path),
pattern),
stdout_tee=None, ignore_status=True)
return result.exit_status == 0
def delete_tmp_dir(self, tmpdir):
"""
Delete the given temporary directory on the remote machine.
"""
self.run('rm -rf "%s"' % utils.sh_escape(tmpdir), ignore_status=True)
self.tmp_dirs.remove(tmpdir)
def get_file(self,
source,
dest,
delete_dest=False,
preserve_perm=True,
preserve_symlinks=False):
"""
Copy files from the remote host to a local path.
Directories will be copied recursively.
If a source component is a directory with a trailing slash,
the content of the directory will be copied, otherwise, the
directory itself and its content will be copied. This
behavior is similar to that of the program 'rsync'.
Args:
source: either
1) a single file or directory, as a string
2) a list of one or more (possibly mixed)
files or directories
dest: a file or a directory (if source contains a
directory or more than one element, you must
supply a directory dest)
delete_dest: if this is true, the command will also clear
out any old files at dest that are not in the
source
preserve_perm: tells get_file() to try to preserve the sources
permissions on files and dirs
preserve_symlinks: try to preserve symlinks instead of
transforming them into files/dirs on copy
Raises:
AutoservRunError: the scp command failed
"""
# Start a master SSH connection if necessary.
self.start_master_ssh()
if isinstance(source, basestring):
source = [source]
dest = os.path.abspath(dest)
# If rsync is disabled or fails, try scp.
try_scp = True
if self.use_rsync():
try:
remote_source = self._encode_remote_paths(source)
local_dest = utils.sh_escape(dest)
rsync = self._make_rsync_cmd([remote_source], local_dest,
delete_dest, preserve_symlinks)
utils.run(rsync)
try_scp = False
except error.CmdError as e:
logging.warn("trying scp, rsync failed: %s" % e)
if try_scp:
# scp has no equivalent to --delete, just drop the entire dest dir
if delete_dest and os.path.isdir(dest):
shutil.rmtree(dest)
os.mkdir(dest)
remote_source = self._make_rsync_compatible_source(source, False)
if remote_source:
# _make_rsync_compatible_source() already did the escaping
remote_source = self._encode_remote_paths(remote_source,
escape=False)
local_dest = utils.sh_escape(dest)
scp = self._make_scp_cmd([remote_source], local_dest)
try:
utils.run(scp)
except error.CmdError as e:
raise error.AutoservRunError(e.args[0], e.args[1])
if not preserve_perm:
# we have no way to tell scp to not try to preserve the
# permissions so set them after copy instead.
# for rsync we could use "--no-p --chmod=ugo=rwX" but those
# options are only in very recent rsync versions
self._set_umask_perms(dest)
