def _connect_transport(self, pkey):
for _ in xrange(self.CONNECT_TIMEOUT_RETRIES):
transport = paramiko.Transport(self._connect_socket())
completed = threading.Event()
transport.start_client(completed)
completed.wait(self.CONNECT_TIMEOUT_SECONDS)
if completed.isSet():
self._check_transport_error(transport)
completed.clear()
transport.auth_publickey(self.user, pkey, completed)
completed.wait(self.CONNECT_TIMEOUT_SECONDS)
if completed.isSet():
self._check_transport_error(transport)
if not transport.is_authenticated():
transport.close()
raise paramiko.AuthenticationException()
return transport
logging.warn("SSH negotiation (%s:%d) timed out, retrying",
self.hostname, self.port)
# HACK: we can't count on transport.join not hanging now, either
transport.join = lambda: None
transport.close()
logging.error(
"SSH negotiation (%s:%d) has timed out %s times, "
"giving up", self.hostname, self.port,
self.CONNECT_TIMEOUT_RETRIES)
raise error.AutoservSSHTimeout("SSH negotiation timed out")
def _run(self, command, timeout, ignore_status, stdout, stderr,
connect_timeout, env, options, stdin, args):
"""Helper function for run()."""
ssh_cmd = self.ssh_command(connect_timeout, options)
if not env.strip():
env = ""
else:
env = "export %s;" % env
for arg in args:
command += ' "%s"' % utils.sh_escape(arg)
full_cmd = '%s "%s %s"' % (ssh_cmd, env, utils.sh_escape(command))
result = utils.run(full_cmd, timeout, True, stdout, stderr,
verbose=False, stdin=stdin,
stderr_is_expected=ignore_status)
# The error messages will show up in band (indistinguishable
# from stuff sent through the SSH connection), so we have the
# remote computer echo the message "Connected." before running
# any command. Since the following 2 errors have to do with
# connecting, it's safe to do these checks.
if result.exit_status == 255:
if re.search(r'^ssh: connect to host .* port .*: '
r'Connection timed out\r$', result.stderr):
raise error.AutoservSSHTimeout("ssh timed out", result)
if "Permission denied." in result.stderr:
msg = "ssh permission denied"
raise error.AutoservSshPermissionDeniedError(msg, result)
if not ignore_status and result.exit_status > 0:
raise error.AutoservRunError("command execution error", result)
return result
def _open_channel(self, timeout):
start_time = time.time()
if os.getpid() != self.pid:
if self.pid is not None:
# HACK: paramiko tries to join() on its worker thread
# and this just hangs on linux after a fork()
self.transport.join = lambda: None
self.transport.atfork()
def join_hook(cmd):
return self._close_transport()
subcommand.subcommand.register_join_hook(join_hook)
logging.debug("Reopening SSH connection after a process fork")
self._init_transport()
channel = None
try:
channel = self.transport.open_session()
except (socket.error, paramiko.SSHException, EOFError) as e:
logging.warn("Exception occurred while opening session: %s", e)
if time.time() - start_time >= timeout:
raise error.AutoservSSHTimeout("ssh failed: %s" % e)
if not channel:
# we couldn't get a channel; re-initing transport should fix that
try:
self.transport.close()
except Exception as e:
logging.debug("paramiko.Transport.close failed with %s", e)
self._init_transport()
return self.transport.open_session()
else:
return channel
def ssh_ping(self, timeout=60):
try:
self.run("true", timeout=timeout, connect_timeout=timeout)
except error.AutoservSSHTimeout:
msg = "Host (ssh) verify timed out (timeout = %d)" % timeout
raise error.AutoservSSHTimeout(msg)
except error.AutoservSshPermissionDeniedError:
#let AutoservSshPermissionDeniedError be visible to the callers
raise
except error.AutoservRunError, e:
# convert the generic AutoservRunError into something more
# specific for this context
raise error.AutoservSshPingHostError(e.description + '\n' +
repr(e.result_obj))
def _execute_daemon(self, section, timeout, stderr_redirector,
client_disconnect_timeout):
monitor_dir = self.host.get_tmp_dir()
daemon_cmd = self.get_daemon_cmd(section, monitor_dir)
# grab the location for the server-side client log file
client_log_prefix = self.get_client_log()
client_log_path = os.path.join(self.results_dir, 'debug',
client_log_prefix + '.log')
client_log = open(client_log_path, 'w', 0)
self.copy_client_config_file(client_log_prefix)
stdout_read = stderr_read = 0
self.host.job.push_execution_context(self.results_dir)
try:
self.host.run(daemon_cmd, ignore_status=True, timeout=timeout)
disconnect_warnings = []
while True:
monitor_cmd = self.get_monitor_cmd(monitor_dir, stdout_read,
stderr_read)
try:
result = self.host.run(monitor_cmd,
ignore_status=True,
timeout=timeout,
stdout_tee=client_log,
stderr_tee=stderr_redirector)
except error.AutoservRunError, e:
result = e.result_obj
result.exit_status = None
disconnect_warnings.append(e.description)
stderr_redirector.log_warning(
"Autotest client was disconnected: %s" % e.description,
"NETWORK")
except error.AutoservSSHTimeout:
result = utils.CmdResult(monitor_cmd, "", "", None, 0)
stderr_redirector.log_warning(
"Attempt to connect to Autotest client timed out",
"NETWORK")
stdout_read += len(result.stdout)
stderr_read += len(self._strip_stderr_prologue(result.stderr))
if result.exit_status is not None:
return result
elif not self.host.wait_up(client_disconnect_timeout):
raise error.AutoservSSHTimeout(
"client was disconnected, reconnect timed out")
def run(self,
command,
timeout=3600,
ignore_status=False,
stdout_tee=utils.TEE_TO_LOGS,
stderr_tee=utils.TEE_TO_LOGS,
connect_timeout=30,
stdin=None,
verbose=True,
args=()):
"""
Run a command on the remote host.
@see shared.hosts.host.run()
:param connect_timeout: connection timeout (in seconds)
:param options: string with additional ssh command options
:param verbose: log the commands
:raise AutoservRunError: if the command failed
:raise AutoservSSHTimeout: ssh connection has timed out
"""
stdout = utils.get_stream_tee_file(stdout_tee,
utils.DEFAULT_STDOUT_LEVEL,
prefix=utils.STDOUT_PREFIX)
stderr = utils.get_stream_tee_file(
stderr_tee,
utils.get_stderr_level(ignore_status),
prefix=utils.STDERR_PREFIX)
for arg in args:
command += ' "%s"' % utils.sh_escape(arg)
if verbose:
logging.debug("Running (ssh-paramiko) '%s'" % command)
# start up the command
start_time = time.time()
try:
channel = self._open_channel(timeout)
channel.exec_command(command)
except (socket.error, paramiko.SSHException, EOFError), e:
# This has to match the string from paramiko *exactly*.
if str(e) != 'Channel closed.':
raise error.AutoservSSHTimeout("ssh failed: %s" % e)
def _open_channel(self, timeout):
start_time = time.time()
if os.getpid() != self.pid:
if self.pid is not None:
# HACK: paramiko tries to join() on its worker thread
# and this just hangs on linux after a fork()
self.transport.join = lambda: None
self.transport.atfork()
join_hook = lambda cmd: self._close_transport()
subcommand.subcommand.register_join_hook(join_hook)
logging.debug("Reopening SSH connection after a process fork")
self._init_transport()
channel = None
try:
channel = self.transport.open_session()
except (socket.error, paramiko.SSHException, EOFError), e:
logging.warn("Exception occured while opening session: %s", e)
if time.time() - start_time >= timeout:
raise error.AutoservSSHTimeout("ssh failed: %s" % e)
def run(self,
command,
timeout=3600,
ignore_status=False,
stdout_tee=utils.TEE_TO_LOGS,
stderr_tee=utils.TEE_TO_LOGS,
connect_timeout=30,
stdin=None,
verbose=True,
args=()):
"""
Run a command on the remote host.
@see shared.hosts.host.run()
:param connect_timeout: connection timeout (in seconds)
:param options: string with additional ssh command options
:param verbose: log the commands
:raise AutoservRunError: if the command failed
:raise AutoservSSHTimeout: ssh connection has timed out
"""
stdout = utils.get_stream_tee_file(stdout_tee,
utils.DEFAULT_STDOUT_LEVEL,
prefix=utils.STDOUT_PREFIX)
stderr = utils.get_stream_tee_file(
stderr_tee,
utils.get_stderr_level(ignore_status),
prefix=utils.STDERR_PREFIX)
for arg in args:
command += ' "%s"' % utils.sh_escape(arg)
if verbose:
logging.debug("Running (ssh-paramiko) '%s'" % command)
# start up the command
start_time = time.time()
try:
channel = self._open_channel(timeout)
channel.exec_command(command)
except (socket.error, paramiko.SSHException, EOFError) as e:
# This has to match the string from paramiko *exactly*.
if str(e) != 'Channel closed.':
raise error.AutoservSSHTimeout("ssh failed: %s" % e)
# pull in all the stdout, stderr until the command terminates
raw_stdout, raw_stderr = [], []
timed_out = False
while not channel.exit_status_ready():
if channel.recv_ready():
raw_stdout.append(channel.recv(self.BUFFSIZE))
stdout.write(raw_stdout[-1])
if channel.recv_stderr_ready():
raw_stderr.append(channel.recv_stderr(self.BUFFSIZE))
stderr.write(raw_stderr[-1])
if timeout and time.time() - start_time > timeout:
timed_out = True
break
stdin = self.__send_stdin(channel, stdin)
time.sleep(1)
if timed_out:
exit_status = -signal.SIGTERM
else:
exit_status = channel.recv_exit_status()
channel.settimeout(10)
self._exhaust_stream(stdout, raw_stdout, channel.recv)
self._exhaust_stream(stderr, raw_stderr, channel.recv_stderr)
channel.close()
duration = time.time() - start_time
# create the appropriate results
stdout = "".join(raw_stdout)
stderr = "".join(raw_stderr)
result = utils.CmdResult(command, stdout, stderr, exit_status,
duration)
if exit_status == -signal.SIGHUP:
msg = "ssh connection unexpectedly terminated"
raise error.AutoservRunError(msg, result)
if timed_out:
logging.warn('Paramiko command timed out after %s sec: %s',
timeout, command)
raise error.AutoservRunError("command timed out", result)
if not ignore_status and exit_status:
raise error.AutoservRunError(command, result)
return result
