def _connect_transport(self, pkey): for _ in xrange(self.CONNECT_TIMEOUT_RETRIES): transport = paramiko.Transport(self._connect_socket()) completed = threading.Event() transport.start_client(completed) completed.wait(self.CONNECT_TIMEOUT_SECONDS) if completed.isSet(): self._check_transport_error(transport) completed.clear() transport.auth_publickey(self.user, pkey, completed) completed.wait(self.CONNECT_TIMEOUT_SECONDS) if completed.isSet(): self._check_transport_error(transport) if not transport.is_authenticated(): transport.close() raise paramiko.AuthenticationException() return transport logging.warn("SSH negotiation (%s:%d) timed out, retrying", self.hostname, self.port) # HACK: we can't count on transport.join not hanging now, either transport.join = lambda: None transport.close() logging.error( "SSH negotiation (%s:%d) has timed out %s times, " "giving up", self.hostname, self.port, self.CONNECT_TIMEOUT_RETRIES) raise error.AutoservSSHTimeout("SSH negotiation timed out")
def _run(self, command, timeout, ignore_status, stdout, stderr, connect_timeout, env, options, stdin, args): """Helper function for run().""" ssh_cmd = self.ssh_command(connect_timeout, options) if not env.strip(): env = "" else: env = "export %s;" % env for arg in args: command += ' "%s"' % utils.sh_escape(arg) full_cmd = '%s "%s %s"' % (ssh_cmd, env, utils.sh_escape(command)) result = utils.run(full_cmd, timeout, True, stdout, stderr, verbose=False, stdin=stdin, stderr_is_expected=ignore_status) # The error messages will show up in band (indistinguishable # from stuff sent through the SSH connection), so we have the # remote computer echo the message "Connected." before running # any command. Since the following 2 errors have to do with # connecting, it's safe to do these checks. if result.exit_status == 255: if re.search(r'^ssh: connect to host .* port .*: ' r'Connection timed out\r$', result.stderr): raise error.AutoservSSHTimeout("ssh timed out", result) if "Permission denied." in result.stderr: msg = "ssh permission denied" raise error.AutoservSshPermissionDeniedError(msg, result) if not ignore_status and result.exit_status > 0: raise error.AutoservRunError("command execution error", result) return result
def _open_channel(self, timeout): start_time = time.time() if os.getpid() != self.pid: if self.pid is not None: # HACK: paramiko tries to join() on its worker thread # and this just hangs on linux after a fork() self.transport.join = lambda: None self.transport.atfork() def join_hook(cmd): return self._close_transport() subcommand.subcommand.register_join_hook(join_hook) logging.debug("Reopening SSH connection after a process fork") self._init_transport() channel = None try: channel = self.transport.open_session() except (socket.error, paramiko.SSHException, EOFError) as e: logging.warn("Exception occurred while opening session: %s", e) if time.time() - start_time >= timeout: raise error.AutoservSSHTimeout("ssh failed: %s" % e) if not channel: # we couldn't get a channel; re-initing transport should fix that try: self.transport.close() except Exception as e: logging.debug("paramiko.Transport.close failed with %s", e) self._init_transport() return self.transport.open_session() else: return channel
def ssh_ping(self, timeout=60): try: self.run("true", timeout=timeout, connect_timeout=timeout) except error.AutoservSSHTimeout: msg = "Host (ssh) verify timed out (timeout = %d)" % timeout raise error.AutoservSSHTimeout(msg) except error.AutoservSshPermissionDeniedError: #let AutoservSshPermissionDeniedError be visible to the callers raise except error.AutoservRunError, e: # convert the generic AutoservRunError into something more # specific for this context raise error.AutoservSshPingHostError(e.description + '\n' + repr(e.result_obj))
def _execute_daemon(self, section, timeout, stderr_redirector, client_disconnect_timeout): monitor_dir = self.host.get_tmp_dir() daemon_cmd = self.get_daemon_cmd(section, monitor_dir) # grab the location for the server-side client log file client_log_prefix = self.get_client_log() client_log_path = os.path.join(self.results_dir, 'debug', client_log_prefix + '.log') client_log = open(client_log_path, 'w', 0) self.copy_client_config_file(client_log_prefix) stdout_read = stderr_read = 0 self.host.job.push_execution_context(self.results_dir) try: self.host.run(daemon_cmd, ignore_status=True, timeout=timeout) disconnect_warnings = [] while True: monitor_cmd = self.get_monitor_cmd(monitor_dir, stdout_read, stderr_read) try: result = self.host.run(monitor_cmd, ignore_status=True, timeout=timeout, stdout_tee=client_log, stderr_tee=stderr_redirector) except error.AutoservRunError, e: result = e.result_obj result.exit_status = None disconnect_warnings.append(e.description) stderr_redirector.log_warning( "Autotest client was disconnected: %s" % e.description, "NETWORK") except error.AutoservSSHTimeout: result = utils.CmdResult(monitor_cmd, "", "", None, 0) stderr_redirector.log_warning( "Attempt to connect to Autotest client timed out", "NETWORK") stdout_read += len(result.stdout) stderr_read += len(self._strip_stderr_prologue(result.stderr)) if result.exit_status is not None: return result elif not self.host.wait_up(client_disconnect_timeout): raise error.AutoservSSHTimeout( "client was disconnected, reconnect timed out")
def run(self, command, timeout=3600, ignore_status=False, stdout_tee=utils.TEE_TO_LOGS, stderr_tee=utils.TEE_TO_LOGS, connect_timeout=30, stdin=None, verbose=True, args=()): """ Run a command on the remote host. @see shared.hosts.host.run() :param connect_timeout: connection timeout (in seconds) :param options: string with additional ssh command options :param verbose: log the commands :raise AutoservRunError: if the command failed :raise AutoservSSHTimeout: ssh connection has timed out """ stdout = utils.get_stream_tee_file(stdout_tee, utils.DEFAULT_STDOUT_LEVEL, prefix=utils.STDOUT_PREFIX) stderr = utils.get_stream_tee_file( stderr_tee, utils.get_stderr_level(ignore_status), prefix=utils.STDERR_PREFIX) for arg in args: command += ' "%s"' % utils.sh_escape(arg) if verbose: logging.debug("Running (ssh-paramiko) '%s'" % command) # start up the command start_time = time.time() try: channel = self._open_channel(timeout) channel.exec_command(command) except (socket.error, paramiko.SSHException, EOFError), e: # This has to match the string from paramiko *exactly*. if str(e) != 'Channel closed.': raise error.AutoservSSHTimeout("ssh failed: %s" % e)
def _open_channel(self, timeout): start_time = time.time() if os.getpid() != self.pid: if self.pid is not None: # HACK: paramiko tries to join() on its worker thread # and this just hangs on linux after a fork() self.transport.join = lambda: None self.transport.atfork() join_hook = lambda cmd: self._close_transport() subcommand.subcommand.register_join_hook(join_hook) logging.debug("Reopening SSH connection after a process fork") self._init_transport() channel = None try: channel = self.transport.open_session() except (socket.error, paramiko.SSHException, EOFError), e: logging.warn("Exception occured while opening session: %s", e) if time.time() - start_time >= timeout: raise error.AutoservSSHTimeout("ssh failed: %s" % e)
def run(self, command, timeout=3600, ignore_status=False, stdout_tee=utils.TEE_TO_LOGS, stderr_tee=utils.TEE_TO_LOGS, connect_timeout=30, stdin=None, verbose=True, args=()): """ Run a command on the remote host. @see shared.hosts.host.run() :param connect_timeout: connection timeout (in seconds) :param options: string with additional ssh command options :param verbose: log the commands :raise AutoservRunError: if the command failed :raise AutoservSSHTimeout: ssh connection has timed out """ stdout = utils.get_stream_tee_file(stdout_tee, utils.DEFAULT_STDOUT_LEVEL, prefix=utils.STDOUT_PREFIX) stderr = utils.get_stream_tee_file( stderr_tee, utils.get_stderr_level(ignore_status), prefix=utils.STDERR_PREFIX) for arg in args: command += ' "%s"' % utils.sh_escape(arg) if verbose: logging.debug("Running (ssh-paramiko) '%s'" % command) # start up the command start_time = time.time() try: channel = self._open_channel(timeout) channel.exec_command(command) except (socket.error, paramiko.SSHException, EOFError) as e: # This has to match the string from paramiko *exactly*. if str(e) != 'Channel closed.': raise error.AutoservSSHTimeout("ssh failed: %s" % e) # pull in all the stdout, stderr until the command terminates raw_stdout, raw_stderr = [], [] timed_out = False while not channel.exit_status_ready(): if channel.recv_ready(): raw_stdout.append(channel.recv(self.BUFFSIZE)) stdout.write(raw_stdout[-1]) if channel.recv_stderr_ready(): raw_stderr.append(channel.recv_stderr(self.BUFFSIZE)) stderr.write(raw_stderr[-1]) if timeout and time.time() - start_time > timeout: timed_out = True break stdin = self.__send_stdin(channel, stdin) time.sleep(1) if timed_out: exit_status = -signal.SIGTERM else: exit_status = channel.recv_exit_status() channel.settimeout(10) self._exhaust_stream(stdout, raw_stdout, channel.recv) self._exhaust_stream(stderr, raw_stderr, channel.recv_stderr) channel.close() duration = time.time() - start_time # create the appropriate results stdout = "".join(raw_stdout) stderr = "".join(raw_stderr) result = utils.CmdResult(command, stdout, stderr, exit_status, duration) if exit_status == -signal.SIGHUP: msg = "ssh connection unexpectedly terminated" raise error.AutoservRunError(msg, result) if timed_out: logging.warn('Paramiko command timed out after %s sec: %s', timeout, command) raise error.AutoservRunError("command timed out", result) if not ignore_status and exit_status: raise error.AutoservRunError(command, result) return result