def timed_out(self,
signal_number,
frame):
r"""
Handle a SIGUSR1 generated by the child process after the time_out has
expired.
signal_number The signal_number of the signal causing
this method to get invoked. This should
always be 10 (SIGUSR1).
frame The stack frame associated with the
function that times out.
"""
gp.lprint_executing()
self.cleanup()
# Compose an error message.
err_msg = "The " + self.__func.__name__
err_msg += " function timed out after " + str(self.__time_out)
err_msg += " seconds.\n"
if not gp.robot_env:
err_msg += gp.sprint_call_stack()
raise ValueError(err_msg)
def stop_boot_test(signal_number=0,
frame=None):
r"""
Handle SIGUSR1 by aborting the boot test that is running.
Description of argument(s):
signal_number The signal number (should always be 10 for SIGUSR1).
frame The frame data.
"""
gp.qprintn()
gp.qprint_executing()
gp.lprint_executing()
# Restore original sigusr1 handler.
set_default_siguser1()
message = "The caller has asked that the boot test be stopped and marked"
message += " as a failure."
function_stack = gm.get_function_stack()
if "wait_state" in function_stack:
st.set_exit_wait_early_message(message)
else:
BuiltIn().fail(gp.sprint_error(message))
def set_default_siguser1():
r"""
Set the default_sigusr1 function to be the SIGUSR1 handler.
"""
gp.printn()
gp.print_executing()
gp.lprint_executing()
signal.signal(signal.SIGUSR1, default_sigusr1)
def login_ssh(login_args={},
max_login_attempts=5):
r"""
Login on the latest open SSH connection. Retry on failure up to
max_login_attempts.
The caller is responsible for making sure there is an open SSH connection.
Description of argument(s):
login_args A dictionary containing the key/value
pairs which are acceptable to the
SSHLibrary login function as parms/args.
At a minimum, this should contain a
'username' and a 'password' entry.
max_login_attempts The max number of times to try logging in
(in the event of login failures).
"""
gp.lprint_executing()
global sshlib
# Get connection data for debug output.
connection = sshlib.get_connection()
gp.lprintn(sprint_connection(connection))
for login_attempt_num in range(1, max_login_attempts + 1):
gp.lprint_timen("Logging in to " + connection.host + ".")
gp.lprint_var(login_attempt_num)
try:
out_buf = sshlib.login(**login_args)
except Exception as login_exception:
# Login will sometimes fail if the connection is new.
except_type, except_value, except_traceback = sys.exc_info()
gp.lprint_var(except_type)
gp.lprint_varx("except_value", str(except_value))
if except_type is paramiko.ssh_exception.SSHException and\
re.match(r"No existing session", str(except_value)):
continue
else:
# We don't tolerate any other error so break from loop and
# re-raise exception.
break
# If we get to this point, the login has worked and we can return.
gp.lpvar(out_buf)
return
# If we get to this point, the login has failed on all attempts so the
# exception will be raised again.
raise(login_exception)
def login_ssh(login_args={},
max_login_attempts=5):
r"""
Login on the latest open SSH connection. Retry on failure up to
max_login_attempts.
The caller is responsible for making sure there is an open SSH connection.
Description of argument(s):
login_args A dictionary containing the key/value
pairs which are acceptable to the
SSHLibrary login function as parms/args.
At a minimum, this should contain a
'username' and a 'password' entry.
max_login_attempts The max number of times to try logging in
(in the event of login failures).
"""
gp.lprint_executing()
global sshlib
# Get connection data for debug output.
connection = sshlib.get_connection()
gp.lprintn(sprint_connection(connection))
for login_attempt_num in range(1, max_login_attempts + 1):
gp.lprint_timen("Logging in to " + connection.host + ".")
gp.lprint_var(login_attempt_num)
try:
out_buf = sshlib.login(**login_args)
except Exception as login_exception:
# Login will sometimes fail if the connection is new.
except_type, except_value, except_traceback = sys.exc_info()
gp.lprint_var(except_type)
gp.lprint_varx("except_value", str(except_value))
if except_type is paramiko.ssh_exception.SSHException and\
re.match(r"No existing session", str(except_value)):
continue
else:
# We don't tolerate any other error so break from loop and
# re-raise exception.
break
# If we get to this point, the login has worked and we can return.
gp.lpvar(out_buf)
return
# If we get to this point, the login has failed on all attempts so the
# exception will be raised again.
raise(login_exception)
def default_sigusr1(signal_number=0, frame=None):
r"""
Handle SIGUSR1 by doing nothing.
This function assists in debugging SIGUSR1 processing by printing messages
to stdout and to the log.html file.
Description of argument(s):
signal_number The signal number (should always be 10 for SIGUSR1).
frame The frame data.
"""
gp.printn()
gp.print_executing()
gp.lprint_executing()
def cleanup(self):
r"""
Cleanup after the run method.
"""
try:
gp.lprint_executing()
gp.lprint_var(self.__child_pid)
except (AttributeError, KeyError, TypeError):
# NOTE: In python 3, this code fails with "KeyError:
# ('__main__',)" when calling functions like lprint_executing that
# use inspect.stack() during object destruction. No fixes found
# so tolerating the error. In python 2.x, it may fail with
# TypeError. This seems to happen when cleaning up after an
# exception was raised.
pass
# If self.__child_pid is 0, then we are either running as the child
# or we've already cleaned up.
# If self.__time_out is None, then no child process would have been
# spawned.
if self.__child_pid == 0 or self.__time_out is None:
return
# Restore the original SIGUSR1 handler.
if self.__original_SIGUSR1_handler != 0:
signal.signal(signal.SIGUSR1, self.__original_SIGUSR1_handler)
try:
gp.lprint_timen("Killing child pid " + str(self.__child_pid) + ".")
os.kill(self.__child_pid, signal.SIGKILL)
except OSError:
gp.lprint_timen("Tolerated kill failure.")
try:
gp.lprint_timen("os.waitpid(" + str(self.__child_pid) + ")")
os.waitpid(self.__child_pid, 0)
except OSError:
gp.lprint_timen("Tolerated waitpid failure.")
self.__child_pid = 0
# For debug purposes, prove that the child pid was killed.
children = gm.get_child_pids()
gp.lprint_var(children)
def cleanup(self):
r"""
Cleanup after the run method.
"""
try:
gp.lprint_executing()
gp.lprint_var(self.__child_pid)
except AttributeError:
pass
# If self.__child_pid is 0, then we are either running as the child
# or we've already cleaned up.
# If self.__time_out is None, then no child process would have been
# spawned.
if self.__child_pid == 0 or self.__time_out is None:
return
# Restore the original SIGUSR1 handler.
if self.__original_SIGUSR1_handler != 0:
signal.signal(signal.SIGUSR1, self.__original_SIGUSR1_handler)
try:
gp.lprint_timen("Killing child pid " + str(self.__child_pid)
+ ".")
os.kill(self.__child_pid, signal.SIGKILL)
except OSError:
gp.lprint_timen("Tolerated kill failure.")
try:
gp.lprint_timen("os.waitpid(" + str(self.__child_pid) + ")")
os.waitpid(self.__child_pid, 0)
except OSError:
gp.lprint_timen("Tolerated waitpid failure.")
self.__child_pid = 0
# For debug purposes, prove that the child pid was killed.
children = gm.get_child_pids()
gp.lprint_var(children)
def execute_ssh_command(cmd_buf,
open_connection_args={},
login_args={},
print_out=0,
print_err=0,
ignore_err=1,
fork=0,
quiet=None,
test_mode=None):
r"""
Run the given command in an SSH session and return the stdout, stderr and
the return code.
If there is no open SSH connection, this function will connect and login.
Likewise, if the caller has not yet logged in to the connection, this
function will do the login.
NOTE: There is special handling when open_connection_args['alias'] equals
"device_connection".
- A write, rather than an execute_command, is done.
- Only stdout is returned (no stderr or rc).
- print_err, ignore_err and fork are not supported.
Description of arguments:
cmd_buf The command string to be run in an SSH
session.
open_connection_args A dictionary of arg names and values which
are legal to pass to the SSHLibrary
open_connection function as parms/args.
At a minimum, this should contain a 'host'
entry.
login_args A dictionary containing the key/value
pairs which are acceptable to the
SSHLibrary login function as parms/args.
At a minimum, this should contain a
'username' and a 'password' entry.
print_out If this is set, this function will print
the stdout/stderr generated by the shell
command.
print_err If show_err is set, this function will
print a standardized error report if the
shell command returns non-zero.
ignore_err Indicates that errors encountered on the
sshlib.execute_command are to be ignored.
fork Indicates that sshlib.start is to be used
rather than sshlib.execute_command.
quiet Indicates whether this function should run
the pissuing() function which prints an
"Issuing: <cmd string>" to stdout. This
defaults to the global quiet value.
test_mode If test_mode is set, this function will
not actually run the command. This
defaults to the global test_mode value.
"""
gp.lprint_executing()
# Obtain default values.
quiet = int(gp.get_var_value(quiet, 0))
test_mode = int(gp.get_var_value(test_mode, 0))
if not quiet:
gp.pissuing(cmd_buf, test_mode)
gp.lpissuing(cmd_buf, test_mode)
if test_mode:
return "", "", 0
global sshlib
max_exec_cmd_attempts = 2
# Look for existing SSH connection.
# Prepare a search connection dictionary.
search_connection_args = open_connection_args.copy()
# Remove keys that don't work well for searches.
search_connection_args.pop("timeout", None)
connection = find_connection(search_connection_args)
if connection:
gp.lprint_timen("Found the following existing connection:")
gp.lprintn(sprint_connection(connection))
if connection.alias == "":
index_or_alias = connection.index
else:
index_or_alias = connection.alias
gp.lprint_timen("Switching to existing connection: \""
+ str(index_or_alias) + "\".")
sshlib.switch_connection(index_or_alias)
else:
gp.lprint_timen("Connecting to " + open_connection_args['host'] + ".")
cix = sshlib.open_connection(**open_connection_args)
try:
login_ssh(login_args)
except Exception as login_exception:
except_type, except_value, except_traceback = sys.exc_info()
rc = 1
stderr = str(except_value)
stdout = ""
max_exec_cmd_attempts = 0
for exec_cmd_attempt_num in range(1, max_exec_cmd_attempts + 1):
gp.lprint_var(exec_cmd_attempt_num)
try:
if fork:
sshlib.start_command(cmd_buf)
else:
if open_connection_args['alias'] == "device_connection":
stdout = sshlib.write(cmd_buf)
stderr = ""
rc = 0
else:
stdout, stderr, rc = \
sshlib.execute_command(cmd_buf,
return_stdout=True,
return_stderr=True,
return_rc=True)
except Exception as execute_exception:
except_type, except_value, except_traceback = sys.exc_info()
gp.lprint_var(except_type)
gp.lprint_varx("except_value", str(except_value))
if except_type is exceptions.AssertionError and\
re.match(r"Connection not open", str(except_value)):
login_ssh(login_args)
# Now we must continue to next loop iteration to retry the
# execute_command.
continue
if (except_type is paramiko.ssh_exception.SSHException
and re.match(r"SSH session not active", str(except_value))) or\
(except_type is socket.error
and re.match(r"\[Errno 104\] Connection reset by peer",
str(except_value))):
# Close and re-open a connection.
# Note: close_connection() doesn't appear to get rid of the
# connection. It merely closes it. Since there is a concern
# about over-consumption of resources, we use
# close_all_connections() which also gets rid of all
# connections.
gp.lprint_timen("Closing all connections.")
sshlib.close_all_connections()
gp.lprint_timen("Connecting to "
+ open_connection_args['host'] + ".")
cix = sshlib.open_connection(**open_connection_args)
login_ssh(login_args)
continue
# We do not handle any other RuntimeErrors so we will raise the
# exception again.
sshlib.close_all_connections()
raise(execute_exception)
# If we get to this point, the command was executed.
break
if fork:
return
if rc != 0 and print_err:
gp.print_var(rc, 1)
if not print_out:
gp.print_var(stderr)
gp.print_var(stdout)
if print_out:
gp.printn(stderr + stdout)
if not ignore_err:
message = gp.sprint_error("The prior SSH"
+ " command returned a non-zero return"
+ " code:\n" + gp.sprint_var(rc, 1) + stderr
+ "\n")
BuiltIn().should_be_equal(rc, 0, message)
if open_connection_args['alias'] == "device_connection":
return stdout
return stdout, stderr, rc
def run(self, func, *args, **kwargs):
r"""
Run the indicated function with the given args and kwargs and return
the value that the function returns. If the time_out value expires,
raise a ValueError exception with a detailed error message.
This method passes all of the args and kwargs directly to the child
function with the following important exception: If kwargs contains a
'time_out' value, it will be used to set the func timer object's
time_out value and then the kwargs['time_out'] entry will be removed.
If the time-out expires before the function finishes running, this
method will raise a ValueError.
Example:
func_timer = func_timer_class()
func_timer.run(run_key, "sleep 3", time_out=2)
Example:
try:
result = func_timer.run(func1, "parm1", time_out=2)
print_var(result)
except ValueError:
print("The func timed out but we're handling it.")
Description of argument(s):
func The function object which is to be called.
args The arguments which are to be passed to
the function object.
kwargs The keyword arguments which are to be
passed to the function object. As noted
above, kwargs['time_out'] will get special
treatment.
"""
gp.lprint_executing()
# Store method parms as object parms.
self.__func = func
# Get self.__time_out value from kwargs. If kwargs['time_out'] is
# not present, self.__time_out will default to None.
self.__time_out = None
if 'time_out' in kwargs:
self.__time_out = kwargs['time_out']
del kwargs['time_out']
# Convert "none" string to None.
try:
if self.__time_out.lower() == "none":
self.__time_out = None
except AttributeError:
pass
if self.__time_out is not None:
self.__time_out = int(self.__time_out)
# Ensure that time_out is non-negative.
message = gv.svalid_range(self.__time_out, [0], "time_out")
if message != "":
raise ValueError("\n"
+ gp.sprint_error_report(message,
format='long'))
gp.lprint_varx("time_out", self.__time_out)
self.__child_pid = 0
if self.__time_out is not None:
# Save the original SIGUSR1 handler for later restoration by this
# class' methods.
self.__original_SIGUSR1_handler = signal.getsignal(signal.SIGUSR1)
# Designate a SIGUSR1 handling function.
signal.signal(signal.SIGUSR1, self.timed_out)
parent_pid = os.getpid()
self.__child_pid = os.fork()
if self.__child_pid == 0:
gp.dprint_timen("Child timer pid " + str(os.getpid())
+ ": Sleeping for " + str(self.__time_out)
+ " seconds.")
time.sleep(self.__time_out)
gp.dprint_timen("Child timer pid " + str(os.getpid())
+ ": Sending SIGUSR1 to parent pid "
+ str(parent_pid) + ".")
os.kill(parent_pid, signal.SIGUSR1)
os._exit(0)
# Call the user's function with the user's arguments.
children = gm.get_child_pids()
gp.lprint_var(children)
gp.lprint_timen("Calling the user's function.")
gp.lprint_varx("func_name", func.__name__)
gp.lprint_vars(args, kwargs)
try:
result = func(*args, **kwargs)
except Exception as func_exception:
# We must handle all exceptions so that we have the chance to
# cleanup before re-raising the exception.
gp.lprint_timen("Encountered exception in user's function.")
self.cleanup()
raise(func_exception)
gp.lprint_timen("Returned from the user's function.")
self.cleanup()
return result
def execute_ssh_command(cmd_buf,
open_connection_args={},
login_args={},
print_out=0,
print_err=0,
ignore_err=1,
fork=0,
quiet=None,
test_mode=None,
time_out=None):
r"""
Run the given command in an SSH session and return the stdout, stderr and
the return code.
If there is no open SSH connection, this function will connect and login.
Likewise, if the caller has not yet logged in to the connection, this
function will do the login.
NOTE: There is special handling when open_connection_args['alias'] equals
"device_connection".
- A write, rather than an execute_command, is done.
- Only stdout is returned (no stderr or rc).
- print_err, ignore_err and fork are not supported.
Description of arguments:
cmd_buf The command string to be run in an SSH
session.
open_connection_args A dictionary of arg names and values which
are legal to pass to the SSHLibrary
open_connection function as parms/args.
At a minimum, this should contain a 'host'
entry.
login_args A dictionary containing the key/value
pairs which are acceptable to the
SSHLibrary login function as parms/args.
At a minimum, this should contain a
'username' and a 'password' entry.
print_out If this is set, this function will print
the stdout/stderr generated by the shell
command.
print_err If show_err is set, this function will
print a standardized error report if the
shell command returns non-zero.
ignore_err Indicates that errors encountered on the
sshlib.execute_command are to be ignored.
fork Indicates that sshlib.start is to be used
rather than sshlib.execute_command.
quiet Indicates whether this function should run
the pissuing() function which prints an
"Issuing: <cmd string>" to stdout. This
defaults to the global quiet value.
test_mode If test_mode is set, this function will
not actually run the command. This
defaults to the global test_mode value.
time_out The amount of time to allow for the
execution of cmd_buf. A value of None
means that there is no limit to how long
the command may take.
"""
gp.lprint_executing()
# Obtain default values.
quiet = int(gp.get_var_value(quiet, 0))
test_mode = int(gp.get_var_value(test_mode, 0))
if not quiet:
gp.pissuing(cmd_buf, test_mode)
gp.lpissuing(cmd_buf, test_mode)
if test_mode:
return "", "", 0
global sshlib
max_exec_cmd_attempts = 2
# Look for existing SSH connection.
# Prepare a search connection dictionary.
search_connection_args = open_connection_args.copy()
# Remove keys that don't work well for searches.
search_connection_args.pop("timeout", None)
connection = find_connection(search_connection_args)
if connection:
gp.lprint_timen("Found the following existing connection:")
gp.lprintn(sprint_connection(connection))
if connection.alias == "":
index_or_alias = connection.index
else:
index_or_alias = connection.alias
gp.lprint_timen("Switching to existing connection: \"" +
str(index_or_alias) + "\".")
sshlib.switch_connection(index_or_alias)
else:
gp.lprint_timen("Connecting to " + open_connection_args['host'] + ".")
cix = sshlib.open_connection(**open_connection_args)
try:
login_ssh(login_args)
except Exception:
except_type, except_value, except_traceback = sys.exc_info()
rc = 1
stderr = str(except_value)
stdout = ""
max_exec_cmd_attempts = 0
for exec_cmd_attempt_num in range(1, max_exec_cmd_attempts + 1):
gp.lprint_var(exec_cmd_attempt_num)
try:
if fork:
sshlib.start_command(cmd_buf)
else:
if open_connection_args['alias'] == "device_connection":
stdout = sshlib.write(cmd_buf)
stderr = ""
rc = 0
else:
stdout, stderr, rc = \
func_timer.run(sshlib.execute_command,
cmd_buf,
return_stdout=True,
return_stderr=True,
return_rc=True,
time_out=time_out)
except Exception:
except_type, except_value, except_traceback = sys.exc_info()
gp.lprint_var(except_type)
gp.lprint_varx("except_value", str(except_value))
# This may be our last time through the retry loop, so setting
# return variables.
rc = 1
stderr = str(except_value)
stdout = ""
if except_type is exceptions.AssertionError and\
re.match(r"Connection not open", str(except_value)):
try:
login_ssh(login_args)
# Now we must continue to next loop iteration to retry the
# execute_command.
continue
except Exception:
except_type, except_value, except_traceback =\
sys.exc_info()
rc = 1
stderr = str(except_value)
stdout = ""
break
if (except_type is paramiko.ssh_exception.SSHException
and re.match(r"SSH session not active", str(except_value))) or\
(except_type is socket.error
and re.match(r"\[Errno 104\] Connection reset by peer",
str(except_value))) or\
(except_type is paramiko.ssh_exception.SSHException
and re.match(r"Timeout opening channel\.",
str(except_value))):
# Close and re-open a connection.
# Note: close_connection() doesn't appear to get rid of the
# connection. It merely closes it. Since there is a concern
# about over-consumption of resources, we use
# close_all_connections() which also gets rid of all
# connections.
gp.lprint_timen("Closing all connections.")
sshlib.close_all_connections()
gp.lprint_timen("Connecting to " +
open_connection_args['host'] + ".")
cix = sshlib.open_connection(**open_connection_args)
login_ssh(login_args)
continue
# We do not handle any other RuntimeErrors so we will raise the
# exception again.
sshlib.close_all_connections()
gp.lprintn(traceback.format_exc())
raise (except_value)
# If we get to this point, the command was executed.
break
if fork:
return
if rc != 0 and print_err:
gp.print_var(rc, gp.hexa())
if not print_out:
gp.print_var(stderr)
gp.print_var(stdout)
if print_out:
gp.printn(stderr + stdout)
if not ignore_err:
message = gp.sprint_error("The prior SSH" +
" command returned a non-zero return" +
" code:\n" + gp.sprint_var(rc, gp.hexa()) +
stderr + "\n")
BuiltIn().should_be_equal(rc, 0, message)
if open_connection_args['alias'] == "device_connection":
return stdout
return stdout, stderr, rc
