def extract(self, host): """Extract the kernel package. This function is only useful to access the content of the package (for example the kernel image) without installing it. It is not necessary to run this function to install the kernel. Args: host: the host on which to extract the kernel package. Returns: The full path to the temporary directory on host where the package was extracted. Raises: AutoservError: no package has yet been obtained. Call DEBKernel.get() with a .deb package. """ if self.source_material is None: raise error.AutoservError("A kernel must first be " "specified via get()") remote_tmpdir = host.get_tmp_dir() basename = os.path.basename(self.source_material) remote_filename = os.path.join(remote_tmpdir, basename) host.send_file(self.source_material, remote_filename) content_dir= os.path.join(remote_tmpdir, "contents") host.run('dpkg -x "%s" "%s"' % (utils.sh_escape(remote_filename), utils.sh_escape(content_dir),)) return content_dir
def _run(self, command, timeout, ignore_status, stdout, stderr, connect_timeout, env, options, stdin, args): """Helper function for run().""" ssh_cmd = self.ssh_command(connect_timeout, options) if not env.strip(): env = "" else: env = "export %s;" % env for arg in args: command += ' "%s"' % utils.sh_escape(arg) full_cmd = '%s "%s %s"' % (ssh_cmd, env, utils.sh_escape(command)) result = utils.run(full_cmd, timeout, True, stdout, stderr, verbose=False, stdin=stdin, stderr_is_expected=ignore_status) # The error messages will show up in band (indistinguishable # from stuff sent through the SSH connection), so we have the # remote computer echo the message "Connected." before running # any command. Since the following 2 errors have to do with # connecting, it's safe to do these checks. if result.exit_status == 255: if re.search(r'^ssh: connect to host .* port .*: ' r'Connection timed out\r$', result.stderr): raise error.AutoservSSHTimeout("ssh timed out", result) if "Permission denied." in result.stderr: msg = "ssh permission denied" raise error.AutoservSshPermissionDeniedError(msg, result) if not ignore_status and result.exit_status > 0: raise error.AutoservRunError("command execution error", result) return result
def get_installed_autodir(cls, host): """ Find where the Autotest client is installed on the host. :return: an absolute path to an installed Autotest client root. :raise AutodirNotFoundError if no Autotest installation can be found. """ autodir = host.get_autodir() if autodir: logging.debug('Using existing host autodir: %s', autodir) return autodir if not _server_system_wide_install(): for path in Autotest.get_client_autodir_paths(host): try: autotest_binary = os.path.join(path, 'autotest') host.run('test -x %s' % utils.sh_escape(autotest_binary)) host.run('test -w %s' % utils.sh_escape(path)) logging.debug('Found existing autodir at %s', path) return path except error.AutoservRunError: logging.debug('%s does not exist on %s', autotest_binary, host.hostname) else: for path in Autotest.get_client_autodir_paths(host): host.run('test -w %s' % utils.sh_escape(path)) logging.debug('Found existing autodir at %s', path) host.autodir = path return path raise AutodirNotFoundError
def install(self, host, **kwargs): """ Install a kernel on the remote host. This will also invoke the guest's bootloader to set this kernel as the default kernel. Args: host: the host on which to install the kernel [kwargs]: remaining keyword arguments will be passed to Bootloader.add_kernel() Raises: AutoservError: no package has yet been obtained. Call DEBKernel.get() with a .deb package. """ if self.source_material is None: raise error.AutoservError("A kernel must first be " "specified via get()") remote_tmpdir = host.get_tmp_dir() basename = os.path.basename(self.source_material) remote_filename = os.path.join(remote_tmpdir, basename) host.send_file(self.source_material, remote_filename) host.run('dpkg -i "%s"' % (utils.sh_escape(remote_filename),)) host.run('mkinitramfs -o "%s" "%s"' % ( utils.sh_escape(self.get_initrd_name()), utils.sh_escape(self.get_version()),)) host.bootloader.add_kernel(self.get_image_name(), initrd=self.get_initrd_name(), **kwargs)
def test_install(self): self.common_code() # record self.host.run.expect_call('dpkg -i "%s"' % (utils.sh_escape(self.remote_filename))) result = common_utils.CmdResult() result.stdout = "1" utils.run.expect_call('dpkg-deb -f "%s" version' % utils.sh_escape(self.kernel.source_material)).and_return(result) utils.run.expect_call('dpkg-deb -f "%s" version' % utils.sh_escape(self.kernel.source_material)).and_return(result) self.host.run.expect_call('mkinitramfs -o "/boot/initrd.img-1" "1"') utils.run.expect_call('dpkg-deb -f "%s" version' % utils.sh_escape(self.kernel.source_material)).and_return(result) utils.run.expect_call('dpkg-deb -f "%s" version' % utils.sh_escape(self.kernel.source_material)).and_return(result) self.host.bootloader.add_kernel.expect_call('/boot/vmlinuz-1', initrd='/boot/initrd.img-1') # run and check self.kernel.install(self.host) self.god.check_playback()
def add_args(self, kernel, args): """ Add cmdline arguments for the specified kernel. :param kernel: can be a position number (index) or title :param args: argument to be added to the current list of args """ return self._run_boottool_exit_status( "--update-kernel=%s" % utils.sh_escape(str(kernel)), "--args=%s" % utils.sh_escape(args) )
def remove_args(self, kernel, args): """ Removes specified cmdline arguments. @param kernel: can be a position number (index) or title @param args: argument to be removed of the current list of args """ return self._run_boottool_exit_status('--update-kernel=%s' % utils.sh_escape(str(kernel)), '--remove-args=%s' % utils.sh_escape(args))
def _find_installable_dir(cls, host): client_autodir_paths = cls.get_client_autodir_paths(host) for path in client_autodir_paths: try: host.run('mkdir -p %s' % utils.sh_escape(path)) host.run('test -w %s' % utils.sh_escape(path)) return path except error.AutoservRunError: logging.debug('Failed to create %s', path) raise error.AutoservInstallError( 'Unable to find a place to install Autotest; tried %s' % ', '.join(client_autodir_paths))
def test_extract(self): # setup self.common_code() content_dir = os.path.join(self.remote_tmpdir, "contents") # record self.host.run.expect_call('dpkg -x "%s" "%s"' % (utils.sh_escape(self.remote_filename), utils.sh_escape(content_dir))) # run and test self.kernel.extract(self.host) self.god.check_playback()
def _install(self, host=None, autodir=None, use_autoserv=True, use_packaging=True): """ Install autotest. If get() was not called previously, an attempt will be made to install from the autotest svn repository. @param host A Host instance on which autotest will be installed @param autodir Location on the remote host to install to @param use_autoserv Enable install modes that depend on the client running with the autoserv harness @param use_packaging Enable install modes that use the packaging system @exception AutoservError If it wasn't possible to install the client after trying all available methods """ if not host: host = self.host if not self.got: self.get() host.wait_up(timeout=30) host.setup() logging.info("Installing autotest on %s", host.hostname) # set up the autotest directory on the remote machine if not autodir: autodir = self.get_install_dir(host) logging.info('Using installation dir %s', autodir) host.set_autodir(autodir) host.run('mkdir -p %s' % utils.sh_escape(autodir)) # make sure there are no files in $AUTODIR/results results_path = os.path.join(autodir, 'results') host.run('rm -rf %s/*' % utils.sh_escape(results_path), ignore_status=True) # Fetch the autotest client from the nearest repository if use_packaging: try: self._install_using_packaging(host, autodir) self._create_test_output_dir(host, autodir) logging.info("Installation of autotest completed") self.installed = True return except (error.PackageInstallError, error.AutoservRunError, global_config.ConfigError), e: logging.info("Could not install autotest using the packaging " "system: %s. Trying other methods", e)
def _install_using_send_file(self, host, autodir): dirs_to_exclude = set(["tests", "site_tests", "deps", "profilers"]) light_files = [os.path.join(self.source_material, f) for f in os.listdir(self.source_material) if f not in dirs_to_exclude] # needs updating when grubby version is changed light_files.append(os.path.join(self.source_material, "deps/grubby/grubby-8.13.tar.bz2")) host.send_file(light_files, autodir, delete_dest=True) profilers_autodir = os.path.join(autodir, 'profilers') profilers_init = os.path.join(self.source_material, 'profilers', '__init__.py') host.run("mkdir -p %s" % profilers_autodir) host.send_file(profilers_init, profilers_autodir, delete_dest=True) dirs_to_exclude.discard("profilers") # create empty dirs for all the stuff we excluded commands = [] for path in dirs_to_exclude: abs_path = os.path.join(autodir, path) abs_path = utils.sh_escape(abs_path) commands.append("mkdir -p '%s'" % abs_path) commands.append("touch '%s'/__init__.py" % abs_path) host.run(';'.join(commands))
def _client_system_wide_install(host): for path in SYSTEM_WIDE_PATHS: try: host.run('test -x %s' % utils.sh_escape(path)) except: return False return True
def uninstall(self, host=None): """ Uninstall (i.e. delete) autotest. Removes the autotest client install from the specified host. @params host a Host instance from which the client will be removed """ if not self.installed: return if self.server_system_wide_install: uninstall_cmd = UNINSTALL_CLIENT_CMD_MAPPING.get(self.os_vendor, None) if uninstall_cmd is not None: logging.info("Trying to uninstall autotest using distro " "provided package manager") host.run(uninstall_cmd) return if not host: host = self.host autodir = host.get_autodir() if not autodir: return # perform the actual uninstall host.run("rm -rf %s" % utils.sh_escape(autodir), ignore_status=True) host.set_autodir(None) self.installed = False
def _install_using_send_file(self, host, autodir): dirs_to_exclude = set(["tests", "site_tests", "deps", "profilers"]) light_files = [os.path.join(self.source_material, f) for f in os.listdir(self.source_material) if f not in dirs_to_exclude] # there should be one and only one grubby tarball grubby_glob = os.path.join(self.source_material, "deps/grubby/grubby-*.tar.bz2") grubby_tarball_paths = glob.glob(grubby_glob) if grubby_tarball_paths: grubby_tarball_path = grubby_tarball_paths[0] if os.path.exists(grubby_tarball_path): light_files.append(grubby_tarball_path) host.send_file(light_files, autodir, delete_dest=True) profilers_autodir = os.path.join(autodir, 'profilers') profilers_init = os.path.join(self.source_material, 'profilers', '__init__.py') host.run("mkdir -p %s" % profilers_autodir) host.send_file(profilers_init, profilers_autodir, delete_dest=True) dirs_to_exclude.discard("profilers") # create empty dirs for all the stuff we excluded commands = [] for path in dirs_to_exclude: abs_path = os.path.join(autodir, path) abs_path = utils.sh_escape(abs_path) commands.append("mkdir -p '%s'" % abs_path) commands.append("touch '%s'/__init__.py" % abs_path) host.run(';'.join(commands))
def boot_once(self, title): if self._host().job: self._host().job.last_boot_tag = title title_opt = '--title=%s' % utils.sh_escape(title) return self._run_boottool_exit_status('--boot-once', title_opt)
def run_async(self, command, stdout_tee=None, stderr_tee=None, args=(), connect_timeout=30, options='', verbose=True, stderr_level=utils.DEFAULT_STDERR_LEVEL, cmd_outside_subshell=''): """ Run a command on the remote host. Returns an AsyncJob object to interact with the remote process. This is mostly copied from SSHHost.run and SSHHost._run """ if verbose: logging.debug("Running (async ssh) '%s'" % command) # Start a master SSH connection if necessary. self.start_master_ssh() self.send_file(os.path.join(self.job.clientdir, "shared", "hosts", "scripts", "run_helper.py"), os.path.join(self.job.tmpdir, "run_helper.py")) env = " ".join("=".join(pair) for pair in self.env.iteritems()) ssh_cmd = self.ssh_command(connect_timeout, options) if not env.strip(): env = "" else: env = "export %s;" % env for arg in args: command += ' "%s"' % utils.sh_escape(arg) full_cmd = '{ssh_cmd} "{env} {cmd}"'.format( ssh_cmd=ssh_cmd, env=env, cmd=utils.sh_escape("%s (%s '%s')" % (cmd_outside_subshell, os.path.join(self.job.tmpdir, "run_helper.py"), utils.sh_escape(command)))) job = utils.AsyncJob(full_cmd, stdout_tee=stdout_tee, stderr_tee=stderr_tee, verbose=verbose, stderr_level=stderr_level, stdin=subprocess.PIPE) def kill_func(): # this triggers the remote kill utils.nuke_subprocess(job.sp) job.kill_func = kill_func return job
def _run_boottool_cmd(self, *options): ''' Runs a boottool command, escaping parameters ''' cmd = self._get_boottool_path() # FIXME: add unsafe options strings sequence to host.run() parameters for option in options: cmd += ' "%s"' % utils.sh_escape(option) return self._host().run(cmd)
def close(self): super(RemoteHost, self).close() self.stop_loggers() if hasattr(self, 'tmp_dirs'): for dir in self.tmp_dirs: try: self.run('rm -rf "%s"' % (utils.sh_escape(dir))) except error.AutoservRunError: pass
def _make_ssh_cmd(self, cmd): """ Create a base ssh command string for the host which can be used to run commands directly on the machine """ base_cmd = make_ssh_command(user=self.user, port=self.port, opts=self.master_ssh_option, hosts_file=self.known_hosts_file) return '%s %s "%s"' % (base_cmd, self.hostname, utils.sh_escape(cmd))
def set_default_by_index(self, index): """ Sets the given entry number to be the default on every next boot To set a default only for the next boot, use boot_once() instead. :param index: entry index number to set as the default. """ if self._host().job: self._host().job.last_boot_tag = None return self._run_boottool_exit_status("--set-default=%s" % utils.sh_escape(str(index)))
def test_get_version(self): # record result = common_utils.CmdResult() result.exit_status = 0 result.stdout = "image" cmd = "rpm -qpi %s | grep Version | awk '{print($3);}'" % (utils.sh_escape("source.rpm")) utils.run.expect_call(cmd).and_return(result) # run and test self.assertEquals(self.kernel.get_version(), result.stdout) self.god.check_playback()
def get_file(self, source, dest, delete_dest=False, preserve_perm=True, preserve_symlinks=False): """ Copy files from the remote host to a local path. Directories will be copied recursively. If a source component is a directory with a trailing slash, the content of the directory will be copied, otherwise, the directory itself and its content will be copied. This behavior is similar to that of the program 'rsync'. Args: source: either 1) a single file or directory, as a string 2) a list of one or more (possibly mixed) files or directories dest: a file or a directory (if source contains a directory or more than one element, you must supply a directory dest) delete_dest: if this is true, the command will also clear out any old files at dest that are not in the source preserve_perm: tells get_file() to try to preserve the sources permissions on files and dirs preserve_symlinks: try to preserve symlinks instead of transforming them into files/dirs on copy Raises: AutoservRunError: the scp command failed """ # Start a master SSH connection if necessary. self.start_master_ssh() if isinstance(source, basestring): source = [source] dest = os.path.abspath(dest) # If rsync is disabled or fails, try scp. try_scp = True if self.use_rsync(): try: remote_source = self._encode_remote_paths(source) local_dest = utils.sh_escape(dest) rsync = self._make_rsync_cmd([remote_source], local_dest, delete_dest, preserve_symlinks) utils.run(rsync) try_scp = False except error.CmdError, e: logging.warn("trying scp, rsync failed: %s" % e)
def get_installed_autodir(cls, host): """ Find where the Autotest client is installed on the host. @returns an absolute path to an installed Autotest client root. @raises AutodirNotFoundError if no Autotest installation can be found. """ autodir = host.get_autodir() if autodir: logging.debug('Using existing host autodir: %s', autodir) return autodir system_wide = True autotest_system_wide = '/usr/bin/autotest-local' try: host.run('test -x %s' % utils.sh_escape(autotest_system_wide)) logging.info("System wide install detected") except: system_wide = False for path in Autotest.get_client_autodir_paths(host): try: try: autotest_binary = os.path.join(path, 'autotest') host.run('test -x %s' % utils.sh_escape(autotest_binary)) except error.AutoservRunError: if system_wide: pass else: raise host.run('test -w %s' % utils.sh_escape(path)) logging.debug('Found existing autodir at %s', path) return path except error.AutoservRunError: logging.debug('%s does not exist on %s', autotest_binary, host.hostname) raise AutodirNotFoundError
def send_file(self, source, dest, delete_dest=False, preserve_symlinks=False): """ Copy files from a local path to the remote host. Directories will be copied recursively. If a source component is a directory with a trailing slash, the content of the directory will be copied, otherwise, the directory itself and its content will be copied. This behavior is similar to that of the program 'rsync'. Args: source: either 1) a single file or directory, as a string 2) a list of one or more (possibly mixed) files or directories dest: a file or a directory (if source contains a directory or more than one element, you must supply a directory dest) delete_dest: if this is true, the command will also clear out any old files at dest that are not in the source preserve_symlinks: controls if symlinks on the source will be copied as such on the destination or transformed into the referenced file/directory Raises: AutoservRunError: the scp command failed """ # Start a master SSH connection if necessary. self.start_master_ssh() if isinstance(source, basestring): source_is_dir = os.path.isdir(source) source = [source] remote_dest = self._encode_remote_paths([dest]) # If rsync is disabled or fails, try scp. try_scp = True if self.use_rsync(): try: local_sources = [utils.sh_escape(path) for path in source] rsync = self._make_rsync_cmd(local_sources, remote_dest, delete_dest, preserve_symlinks) utils.run(rsync) try_scp = False except error.CmdError, e: logging.warn("trying scp, rsync failed: %s" % e)
def get_version(self): """Get the version of the kernel to be installed. Returns: The version string, as would be returned by 'make kernelrelease'. Raises: AutoservError: no package has yet been obtained. Call RPMKernel.get() with a .rpm package. """ if self.source_material is None: raise error.AutoservError("A kernel must first be \ specified via get()") retval = utils.run('rpm -qpi %s | grep Version | awk \'{print($3);}\'' % utils.sh_escape(self.source_material)) return retval.stdout.strip()
def get_version(self): """Get the version of the kernel to be installed. Returns: The version string, as would be returned by 'make kernelrelease'. Raises: AutoservError: no package has yet been obtained. Call DEBKernel.get() with a .deb package. """ if self.source_material is None: raise error.AutoservError("A kernel must first be " "specified via get()") retval= utils.run('dpkg-deb -f "%s" version' % utils.sh_escape(self.source_material),) return retval.stdout.strip()
def uninstall(self, host=None): """ Uninstall (i.e. delete) autotest. Removes the autotest client install from the specified host. @params host a Host instance from which the client will be removed """ if not self.installed: return if not host: host = self.host autodir = host.get_autodir() if not autodir: return # perform the actual uninstall host.run("rm -rf %s" % utils.sh_escape(autodir), ignore_status=True) host.set_autodir(None) self.installed = False
def _make_rsync_compatible_globs(self, path, is_local): """ Given an rsync-style path, returns a list of globbed paths that will hopefully provide equivalent behaviour for scp. Does not support the full range of rsync pattern matching behaviour, only that exposed in the get/send_file interface (trailing slashes). The is_local param is flag indicating if the paths should be interpreted as local or remote paths. """ # non-trailing slash paths should just work if len(path) == 0 or path[-1] != "/": return [path] # make a function to test if a pattern matches any files if is_local: def glob_matches_files(path, pattern): return len(glob.glob(path + pattern)) > 0 else: def glob_matches_files(path, pattern): result = self.run("ls \"%s\"%s" % (utils.sh_escape(path), pattern), stdout_tee=None, ignore_status=True) return result.exit_status == 0 # take a set of globs that cover all files, and see which are needed patterns = ["*", ".[!.]*"] patterns = [p for p in patterns if glob_matches_files(path, p)] # convert them into a set of paths suitable for the commandline if is_local: return ["\"%s\"%s" % (utils.sh_escape(path), pattern) for pattern in patterns] else: return [utils.scp_remote_escape(path) + pattern for pattern in patterns]
def glob_matches_files(path, pattern): result = self.run("ls \"%s\"%s" % (utils.sh_escape(path), pattern), stdout_tee=None, ignore_status=True) return result.exit_status == 0
def delete_tmp_dir(self, tmpdir): """ Delete the given temporary directory on the remote machine. """ self.run('rm -rf "%s"' % utils.sh_escape(tmpdir), ignore_status=True) self.tmp_dirs.remove(tmpdir)
def get_file(self, source, dest, delete_dest=False, preserve_perm=True, preserve_symlinks=False): """ Copy files from the remote host to a local path. Directories will be copied recursively. If a source component is a directory with a trailing slash, the content of the directory will be copied, otherwise, the directory itself and its content will be copied. This behavior is similar to that of the program 'rsync'. Args: source: either 1) a single file or directory, as a string 2) a list of one or more (possibly mixed) files or directories dest: a file or a directory (if source contains a directory or more than one element, you must supply a directory dest) delete_dest: if this is true, the command will also clear out any old files at dest that are not in the source preserve_perm: tells get_file() to try to preserve the sources permissions on files and dirs preserve_symlinks: try to preserve symlinks instead of transforming them into files/dirs on copy Raises: AutoservRunError: the scp command failed """ # Start a master SSH connection if necessary. self.start_master_ssh() if isinstance(source, basestring): source = [source] dest = os.path.abspath(dest) # If rsync is disabled or fails, try scp. try_scp = True if self.use_rsync(): try: remote_source = self._encode_remote_paths(source) local_dest = utils.sh_escape(dest) rsync = self._make_rsync_cmd([remote_source], local_dest, delete_dest, preserve_symlinks) utils.run(rsync) try_scp = False except error.CmdError as e: logging.warn("trying scp, rsync failed: %s" % e) if try_scp: # scp has no equivalent to --delete, just drop the entire dest dir if delete_dest and os.path.isdir(dest): shutil.rmtree(dest) os.mkdir(dest) remote_source = self._make_rsync_compatible_source(source, False) if remote_source: # _make_rsync_compatible_source() already did the escaping remote_source = self._encode_remote_paths(remote_source, escape=False) local_dest = utils.sh_escape(dest) scp = self._make_scp_cmd([remote_source], local_dest) try: utils.run(scp) except error.CmdError as e: raise error.AutoservRunError(e.args[0], e.args[1]) if not preserve_perm: # we have no way to tell scp to not try to preserve the # permissions so set them after copy instead. # for rsync we could use "--no-p --chmod=ugo=rwX" but those # options are only in very recent rsync versions self._set_umask_perms(dest)