def _run(self, command, timeout, ignore_status, stdout, stderr, connect_timeout, env, options, stdin, args): """Helper function for run().""" ssh_cmd = self.ssh_command(connect_timeout, options) if not env.strip(): env = "" else: env = "export %s;" % env for arg in args: command += ' "%s"' % utils.sh_escape(arg) full_cmd = '%s "%s %s"' % (ssh_cmd, env, utils.sh_escape(command)) result = utils.run(full_cmd, timeout, True, stdout, stderr, verbose=False, stdin=stdin, stderr_is_expected=ignore_status) # The error messages will show up in band (indistinguishable # from stuff sent through the SSH connection), so we have the # remote computer echo the message "Connected." before running # any command. Since the following 2 errors have to do with # connecting, it's safe to do these checks. if result.exit_status == 255: if re.search(r'^ssh: connect to host .* port .*: ' r'Connection timed out\r$', result.stderr): raise error.AutoservSSHTimeout("ssh timed out", result) if "Permission denied." in result.stderr: msg = "ssh permission denied" raise error.AutoservSshPermissionDeniedError(msg, result) if not ignore_status and result.exit_status > 0: raise error.AutoservRunError("command execution error", result) return result
def run(self, command, timeout=3600, ignore_status=False, stdout_tee=utils.TEE_TO_LOGS, stderr_tee=utils.TEE_TO_LOGS, connect_timeout=30, options='', stdin=None, verbose=True, args=()): """ Run a command on the remote host. @see shared.hosts.host.run() :param connect_timeout: connection timeout (in seconds) :param options: string with additional ssh command options :param verbose: log the commands :raise AutoservRunError: if the command failed :raise AutoservSSHTimeout: ssh connection has timed out """ if verbose: logging.debug("Running (ssh) '%s'" % command) # Start a master SSH connection if necessary. self.start_master_ssh() env = " ".join("=".join(pair) for pair in self.env.iteritems()) try: return self._run(command, timeout, ignore_status, stdout_tee, stderr_tee, connect_timeout, env, options, stdin, args) except error.CmdError, cmderr: # We get a CmdError here only if there is timeout of that command. # Catch that and stuff it into AutoservRunError and raise it. raise error.AutoservRunError(cmderr.args[0], cmderr.args[1])
def _expect_failed_run(self, command): (self.host.run.expect_call(command).and_raises( error.AutoservRunError('dummy', object())))
timed_out = True break stdin = self.__send_stdin(channel, stdin) time.sleep(1) if timed_out: exit_status = -signal.SIGTERM else: exit_status = channel.recv_exit_status() channel.settimeout(10) self._exhaust_stream(stdout, raw_stdout, channel.recv) self._exhaust_stream(stderr, raw_stderr, channel.recv_stderr) channel.close() duration = time.time() - start_time # create the appropriate results stdout = "".join(raw_stdout) stderr = "".join(raw_stderr) result = utils.CmdResult(command, stdout, stderr, exit_status, duration) if exit_status == -signal.SIGHUP: msg = "ssh connection unexpectedly terminated" raise error.AutoservRunError(msg, result) if timed_out: logging.warn('Paramiko command timed out after %s sec: %s', timeout, command) raise error.AutoservRunError("command timed out", result) if not ignore_status and exit_status: raise error.AutoservRunError(command, result) return result
class AbstractSSHHost(SiteHost): """ This class represents a generic implementation of most of the framework necessary for controlling a host via ssh. It implements almost all of the abstract Host methods, except for the core Host.run method. """ def _initialize(self, hostname, user="******", port=22, password="", *args, **dargs): super(AbstractSSHHost, self)._initialize(hostname=hostname, *args, **dargs) self.ip = socket.getaddrinfo(self.hostname, None)[0][4][0] self.user = user self.port = port self.password = password self._use_rsync = None self.known_hosts_file = tempfile.mkstemp()[1] """ Master SSH connection background job, socket temp directory and socket control path option. If master-SSH is enabled, these fields will be initialized by start_master_ssh when a new SSH connection is initiated. """ self.master_ssh_job = None self.master_ssh_tempdir = None self.master_ssh_option = '' def use_rsync(self): if self._use_rsync is not None: return self._use_rsync # Check if rsync is available on the remote host. If it's not, # don't try to use it for any future file transfers. self._use_rsync = self._check_rsync() if not self._use_rsync: logging.warn("rsync not available on remote host %s -- disabled", self.hostname) return self._use_rsync def _check_rsync(self): """ Check if rsync is available on the remote host. """ try: self.run("rsync --version", stdout_tee=None, stderr_tee=None) except error.AutoservRunError: return False return True def _encode_remote_paths(self, paths, escape=True): """ Given a list of file paths, encodes it as a single remote path, in the style used by rsync and scp. """ if escape: paths = [utils.scp_remote_escape(path) for path in paths] return '%s@%s:"%s"' % (self.user, self.hostname, " ".join(paths)) def _make_rsync_cmd(self, sources, dest, delete_dest, preserve_symlinks): """ Given a list of source paths and a destination path, produces the appropriate rsync command for copying them. Remote paths must be pre-encoded. """ ssh_cmd = make_ssh_command(user=self.user, port=self.port, opts=self.master_ssh_option, hosts_file=self.known_hosts_file) if delete_dest: delete_flag = "--delete" else: delete_flag = "" if preserve_symlinks: symlink_flag = "" else: symlink_flag = "-L" command = "rsync %s %s --timeout=1800 --rsh='%s' -az %s %s" return command % (symlink_flag, delete_flag, ssh_cmd, " ".join(sources), dest) def _make_ssh_cmd(self, cmd): """ Create a base ssh command string for the host which can be used to run commands directly on the machine """ base_cmd = make_ssh_command(user=self.user, port=self.port, opts=self.master_ssh_option, hosts_file=self.known_hosts_file) return '%s %s "%s"' % (base_cmd, self.hostname, utils.sh_escape(cmd)) def _make_scp_cmd(self, sources, dest): """ Given a list of source paths and a destination path, produces the appropriate scp command for encoding it. Remote paths must be pre-encoded. """ command = ("scp -rq %s -o StrictHostKeyChecking=no " "-o UserKnownHostsFile=%s -P %d %s '%s'") return command % (self.master_ssh_option, self.known_hosts_file, self.port, " ".join(sources), dest) def _make_rsync_compatible_globs(self, path, is_local): """ Given an rsync-style path, returns a list of globbed paths that will hopefully provide equivalent behaviour for scp. Does not support the full range of rsync pattern matching behaviour, only that exposed in the get/send_file interface (trailing slashes). The is_local param is flag indicating if the paths should be interpreted as local or remote paths. """ # non-trailing slash paths should just work if len(path) == 0 or path[-1] != "/": return [path] # make a function to test if a pattern matches any files if is_local: def glob_matches_files(path, pattern): return len(glob.glob(path + pattern)) > 0 else: def glob_matches_files(path, pattern): result = self.run("ls \"%s\"%s" % (utils.sh_escape(path), pattern), stdout_tee=None, ignore_status=True) return result.exit_status == 0 # take a set of globs that cover all files, and see which are needed patterns = ["*", ".[!.]*"] patterns = [p for p in patterns if glob_matches_files(path, p)] # convert them into a set of paths suitable for the commandline if is_local: return [ "\"%s\"%s" % (utils.sh_escape(path), pattern) for pattern in patterns ] else: return [ utils.scp_remote_escape(path) + pattern for pattern in patterns ] def _make_rsync_compatible_source(self, source, is_local): """ Applies the same logic as _make_rsync_compatible_globs, but applies it to an entire list of sources, producing a new list of sources, properly quoted. """ return sum((self._make_rsync_compatible_globs(path, is_local) for path in source), []) def _set_umask_perms(self, dest): """ Given a destination file/dir (recursively) set the permissions on all the files and directories to the max allowed by running umask. """ # now this looks strange but I haven't found a way in Python to _just_ # get the umask, apparently the only option is to try to set it umask = os.umask(0) os.umask(umask) max_privs = 0777 & ~umask def set_file_privs(filename): file_stat = os.stat(filename) file_privs = max_privs # if the original file permissions do not have at least one # executable bit then do not set it anywhere if not file_stat.st_mode & 0111: file_privs &= ~0111 os.chmod(filename, file_privs) # try a bottom-up walk so changes on directory permissions won't cut # our access to the files/directories inside it for root, dirs, files in os.walk(dest, topdown=False): # when setting the privileges we emulate the chmod "X" behaviour # that sets to execute only if it is a directory or any of the # owner/group/other already has execute right for dirname in dirs: os.chmod(os.path.join(root, dirname), max_privs) for filename in files: set_file_privs(os.path.join(root, filename)) # now set privs for the dest itself if os.path.isdir(dest): os.chmod(dest, max_privs) else: set_file_privs(dest) def get_file(self, source, dest, delete_dest=False, preserve_perm=True, preserve_symlinks=False): """ Copy files from the remote host to a local path. Directories will be copied recursively. If a source component is a directory with a trailing slash, the content of the directory will be copied, otherwise, the directory itself and its content will be copied. This behavior is similar to that of the program 'rsync'. Args: source: either 1) a single file or directory, as a string 2) a list of one or more (possibly mixed) files or directories dest: a file or a directory (if source contains a directory or more than one element, you must supply a directory dest) delete_dest: if this is true, the command will also clear out any old files at dest that are not in the source preserve_perm: tells get_file() to try to preserve the sources permissions on files and dirs preserve_symlinks: try to preserve symlinks instead of transforming them into files/dirs on copy Raises: AutoservRunError: the scp command failed """ # Start a master SSH connection if necessary. self.start_master_ssh() if isinstance(source, basestring): source = [source] dest = os.path.abspath(dest) # If rsync is disabled or fails, try scp. try_scp = True if self.use_rsync(): try: remote_source = self._encode_remote_paths(source) local_dest = utils.sh_escape(dest) rsync = self._make_rsync_cmd([remote_source], local_dest, delete_dest, preserve_symlinks) utils.run(rsync) try_scp = False except error.CmdError, e: logging.warn("trying scp, rsync failed: %s" % e) if try_scp: # scp has no equivalent to --delete, just drop the entire dest dir if delete_dest and os.path.isdir(dest): shutil.rmtree(dest) os.mkdir(dest) remote_source = self._make_rsync_compatible_source(source, False) if remote_source: # _make_rsync_compatible_source() already did the escaping remote_source = self._encode_remote_paths(remote_source, escape=False) local_dest = utils.sh_escape(dest) scp = self._make_scp_cmd([remote_source], local_dest) try: utils.run(scp) except error.CmdError, e: raise error.AutoservRunError(e.args[0], e.args[1])
if dest_exists and dest_is_dir: cmd = "rm -rf %s && mkdir %s" % (dest, dest) self.run(cmd) elif not dest_exists and dest_is_dir: cmd = "mkdir %s" % dest self.run(cmd) local_sources = self._make_rsync_compatible_source(source, True) if local_sources: scp = self._make_scp_cmd(local_sources, remote_dest) try: utils.run(scp) except error.CmdError, e: raise error.AutoservRunError(e.args[0], e.args[1]) def ssh_ping(self, timeout=60): try: self.run("true", timeout=timeout, connect_timeout=timeout) except error.AutoservSSHTimeout: msg = "Host (ssh) verify timed out (timeout = %d)" % timeout raise error.AutoservSSHTimeout(msg) except error.AutoservSshPermissionDeniedError: #let AutoservSshPermissionDeniedError be visible to the callers raise except error.AutoservRunError, e: # convert the generic AutoservRunError into something more # specific for this context raise error.AutoservSshPingHostError(e.description + '\n' + repr(e.result_obj))
def run_grep(self, command, timeout=30, ignore_status=False, stdout_ok_regexp=None, stdout_err_regexp=None, stderr_ok_regexp=None, stderr_err_regexp=None, connect_timeout=30): """ Run a command on the remote host and look for regexp in stdout or stderr to determine if the command was successul or not. Args: command: the command line string timeout: time limit in seconds before attempting to kill the running process. The run() function will take a few seconds longer than 'timeout' to complete if it has to kill the process. ignore_status: do not raise an exception, no matter what the exit code of the command is. stdout_ok_regexp: regexp that should be in stdout if the command was successul. stdout_err_regexp: regexp that should be in stdout if the command failed. stderr_ok_regexp: regexp that should be in stderr if the command was successul. stderr_err_regexp: regexp that should be in stderr if the command failed. Returns: if the command was successul, raises an exception otherwise. Raises: AutoservRunError: - the exit code of the command execution was not 0. - If stderr_err_regexp is found in stderr, - If stdout_err_regexp is found in stdout, - If stderr_ok_regexp is not found in stderr. - If stdout_ok_regexp is not found in stdout, """ # We ignore the status, because we will handle it at the end. result = self.run(command, timeout, ignore_status=True, connect_timeout=connect_timeout) # Look for the patterns, in order for (regexp, stream) in ((stderr_err_regexp, result.stderr), (stdout_err_regexp, result.stdout)): if regexp and stream: err_re = re.compile(regexp) if err_re.search(stream): raise error.AutoservRunError( '%s failed, found error pattern: "%s"' % (command, regexp), result) for (regexp, stream) in ((stderr_ok_regexp, result.stderr), (stdout_ok_regexp, result.stdout)): if regexp and stream: ok_re = re.compile(regexp) if ok_re.search(stream): if ok_re.search(stream): return if not ignore_status and result.exit_status > 0: raise error.AutoservRunError("command execution error", result)
def run(self, command, timeout=3600, ignore_status=False, stdout_tee=utils.TEE_TO_LOGS, stderr_tee=utils.TEE_TO_LOGS, connect_timeout=30, stdin=None, verbose=True, args=()): """ Run a command on the remote host. @see shared.hosts.host.run() :param connect_timeout: connection timeout (in seconds) :param options: string with additional ssh command options :param verbose: log the commands :raise AutoservRunError: if the command failed :raise AutoservSSHTimeout: ssh connection has timed out """ stdout = utils.get_stream_tee_file(stdout_tee, utils.DEFAULT_STDOUT_LEVEL, prefix=utils.STDOUT_PREFIX) stderr = utils.get_stream_tee_file( stderr_tee, utils.get_stderr_level(ignore_status), prefix=utils.STDERR_PREFIX) for arg in args: command += ' "%s"' % utils.sh_escape(arg) if verbose: logging.debug("Running (ssh-paramiko) '%s'" % command) # start up the command start_time = time.time() try: channel = self._open_channel(timeout) channel.exec_command(command) except (socket.error, paramiko.SSHException, EOFError) as e: # This has to match the string from paramiko *exactly*. if str(e) != 'Channel closed.': raise error.AutoservSSHTimeout("ssh failed: %s" % e) # pull in all the stdout, stderr until the command terminates raw_stdout, raw_stderr = [], [] timed_out = False while not channel.exit_status_ready(): if channel.recv_ready(): raw_stdout.append(channel.recv(self.BUFFSIZE)) stdout.write(raw_stdout[-1]) if channel.recv_stderr_ready(): raw_stderr.append(channel.recv_stderr(self.BUFFSIZE)) stderr.write(raw_stderr[-1]) if timeout and time.time() - start_time > timeout: timed_out = True break stdin = self.__send_stdin(channel, stdin) time.sleep(1) if timed_out: exit_status = -signal.SIGTERM else: exit_status = channel.recv_exit_status() channel.settimeout(10) self._exhaust_stream(stdout, raw_stdout, channel.recv) self._exhaust_stream(stderr, raw_stderr, channel.recv_stderr) channel.close() duration = time.time() - start_time # create the appropriate results stdout = "".join(raw_stdout) stderr = "".join(raw_stderr) result = utils.CmdResult(command, stdout, stderr, exit_status, duration) if exit_status == -signal.SIGHUP: msg = "ssh connection unexpectedly terminated" raise error.AutoservRunError(msg, result) if timed_out: logging.warn('Paramiko command timed out after %s sec: %s', timeout, command) raise error.AutoservRunError("command timed out", result) if not ignore_status and exit_status: raise error.AutoservRunError(command, result) return result
def send_file(self, source, dest, delete_dest=False, preserve_symlinks=False): """ Copy files from a local path to the remote host. Directories will be copied recursively. If a source component is a directory with a trailing slash, the content of the directory will be copied, otherwise, the directory itself and its content will be copied. This behavior is similar to that of the program 'rsync'. Args: source: either 1) a single file or directory, as a string 2) a list of one or more (possibly mixed) files or directories dest: a file or a directory (if source contains a directory or more than one element, you must supply a directory dest) delete_dest: if this is true, the command will also clear out any old files at dest that are not in the source preserve_symlinks: controls if symlinks on the source will be copied as such on the destination or transformed into the referenced file/directory Raises: AutoservRunError: the scp command failed """ # Start a master SSH connection if necessary. self.start_master_ssh() if isinstance(source, str): source_is_dir = os.path.isdir(source) source = [source] remote_dest = self._encode_remote_paths([dest]) # If rsync is disabled or fails, try scp. try_scp = True if self.use_rsync(): try: local_sources = [utils.sh_escape(path) for path in source] rsync = self._make_rsync_cmd(local_sources, remote_dest, delete_dest, preserve_symlinks) utils.run(rsync) try_scp = False except error.CmdError as e: logging.warn("trying scp, rsync failed: %s" % e) if try_scp: # scp has no equivalent to --delete, just drop the entire dest dir if delete_dest: dest_exists = False try: self.run("test -x %s" % dest) dest_exists = True except error.AutoservRunError: pass dest_is_dir = False if dest_exists: try: self.run("test -d %s" % dest) dest_is_dir = True except error.AutoservRunError: pass # If there is a list of more than one path, destination *has* # to be a dir. If there's a single path being transferred and # it is a dir, the destination also has to be a dir. Therefore # it has to be created on the remote machine in case it doesn't # exist, otherwise we will have an scp failure. if len(source) > 1 or source_is_dir: dest_is_dir = True if dest_exists and dest_is_dir: cmd = "rm -rf %s && mkdir %s" % (dest, dest) self.run(cmd) elif not dest_exists and dest_is_dir: cmd = "mkdir %s" % dest self.run(cmd) local_sources = self._make_rsync_compatible_source(source, True) if local_sources: scp = self._make_scp_cmd(local_sources, remote_dest) try: utils.run(scp) except error.CmdError as e: raise error.AutoservRunError(e.args[0], e.args[1])
def get_file(self, source, dest, delete_dest=False, preserve_perm=True, preserve_symlinks=False): """ Copy files from the remote host to a local path. Directories will be copied recursively. If a source component is a directory with a trailing slash, the content of the directory will be copied, otherwise, the directory itself and its content will be copied. This behavior is similar to that of the program 'rsync'. Args: source: either 1) a single file or directory, as a string 2) a list of one or more (possibly mixed) files or directories dest: a file or a directory (if source contains a directory or more than one element, you must supply a directory dest) delete_dest: if this is true, the command will also clear out any old files at dest that are not in the source preserve_perm: tells get_file() to try to preserve the sources permissions on files and dirs preserve_symlinks: try to preserve symlinks instead of transforming them into files/dirs on copy Raises: AutoservRunError: the scp command failed """ # Start a master SSH connection if necessary. self.start_master_ssh() if isinstance(source, str): source = [source] dest = os.path.abspath(dest) # If rsync is disabled or fails, try scp. try_scp = True if self.use_rsync(): try: remote_source = self._encode_remote_paths(source) local_dest = utils.sh_escape(dest) rsync = self._make_rsync_cmd([remote_source], local_dest, delete_dest, preserve_symlinks) utils.run(rsync) try_scp = False except error.CmdError as e: logging.warn("trying scp, rsync failed: %s" % e) if try_scp: # scp has no equivalent to --delete, just drop the entire dest dir if delete_dest and os.path.isdir(dest): shutil.rmtree(dest) os.mkdir(dest) remote_source = self._make_rsync_compatible_source(source, False) if remote_source: # _make_rsync_compatible_source() already did the escaping remote_source = self._encode_remote_paths(remote_source, escape=False) local_dest = utils.sh_escape(dest) scp = self._make_scp_cmd([remote_source], local_dest) try: utils.run(scp) except error.CmdError as e: raise error.AutoservRunError(e.args[0], e.args[1]) if not preserve_perm: # we have no way to tell scp to not try to preserve the # permissions so set them after copy instead. # for rsync we could use "--no-p --chmod=ugo=rwX" but those # options are only in very recent rsync versions self._set_umask_perms(dest)