def atomic_read(self, timeout=None, do_expect=True):
"""Atomically read command output without waiting for certain timeout,
this method will return instantly when commands execution complete."""
data_rd = ''
read_completed = True
time_interval = 0.03
size_interval = 1024
if timeout is None: timeout = self.command_timeout
if timeout > 0:
data_rd = self.read_leftover
self.read_leftover = ''
t_end_read = time.time() + timeout
read_completed = False
linesep_complemented = False
while time.time() <= t_end_read:
chunk = self._str(self.pty.read_nonblocking(size_interval))
data_rd = data_rd + chunk
if do_expect and not chunk and data_rd:
# strip all ANSI escape characters first
data_rd = utils.strip_ansi_escape(data_rd)
# match shell prompt to check if command execution ends
s = data_rd[-(len(self.prompt) + prompt_offset_range):]
spos = in_search(self.prompt, s, do_find=True)
if spos >= 0:
epos = utils.reversed_find_term(spos, self.prompt, s)
if epos < 0:
self.read_leftover = data_rd[epos:]
data_rd = data_rd[:epos]
read_completed = True
break
elif not linesep_complemented and not data_rd and time.time(
) > (t_end_read - timeout * 0.6):
# only two exceptional cases will make process reach here as below,
# 1. invisible text isn't correctly sent, eg, passphrases
# 2. the final linesep of command text isn't successfully sent
self._send_all(self.pty_linesep)
linesep_complemented = True
time.sleep(time_interval)
self.log(data_rd)
# in very occasional cases, the Pty connection dies unnaturally when performing reading,
# in this case, nothing will be returned and also shell prompt won't be reached.
if do_expect and not read_completed and not self.running_locally:
raise TimeoutError('Command exceeded time limit: %r sec' %
(timeout),
prompt=self.prompt,
output=data_rd)
return data_rd
def _escape(self, out, escape):
"""Escape each item in escape list, return False if any of items is found."""
if not escape or not out: return False
escapes = escape if isinstance(escape, type([])) else [
escape,
]
do_raise = False
for esc in escapes:
if in_search(esc, out):
do_raise = True
break
return do_raise
def _expect(self, out, expect):
"""Expect each item in expect list, return False only if all items are found."""
if not expect or not out: return False
expects = expect if isinstance(expect, type([])) else [
expect,
]
do_raise = False
for exp in expects:
pos = in_search(exp, out, do_find=True)
if pos < 0:
do_raise = True
break
out = out[pos + len(exp):]
return do_raise
def _connect(self, **seqcmdargs):
"""Connect to new pty and update everything related with post validation."""
cmd = seqcmdargs['command']
cmd_argv = utils.split_command_args(cmd)
cmd_word = cmd_argv[0]
fixed_cmd = ' '.join(cmd_argv)
cmd_args = [
x for x in cmd_argv[1:] if not (x.startswith('-') or '=' in x)
]
login_user = seqcmdargs.get('user')
login_password = seqcmdargs.get('password')
do_boot_check = seqcmdargs.get('boot_expect') or seqcmdargs.get(
'boot_escape')
# initialize host information
target_host = self.host
connect_session = cmd
is_serial_port_mode = False
do_remote_connect = True
# initialize connect timeout
if 'ssh' in fixed_cmd:
connect_timeout = seqcmdargs['timeout'] if seqcmdargs.get(
'timeout') else ssh_timeout
elif 'telnet' in fixed_cmd:
connect_timeout = seqcmdargs['timeout'] if seqcmdargs.get(
'timeout') else telnet_timeout
else:
# establish local connection
do_remote_connect = False
connect_timeout = default_connect_timeout
# fix host information based on arguments
if do_remote_connect:
target_host = cmd_args[0][cmd_args[0].find('@') + 1:]
telnet_port = int(
cmd_args[1]
) if 'telnet' in fixed_cmd and len(cmd_args) > 1 else -1
connect_session = '%s %s' % (cmd_word, target_host) + (
' %d' % (telnet_port) if telnet_port > 0 else '')
if telnet_port >= base_serial_port: is_serial_port_mode = True
# do session connecting with retry
session_connected = False
connect_retry = session_connect_retry
while connect_retry > 0 and not session_connected:
if self.running_locally:
self.close_pty()
self.log('%s %s' % (self.prompt, cmd) + newline)
self.pty = ptyprocess.PtyProcess.spawn(argv=cmd_argv)
else:
if do_remote_connect and not self.pty_ping_host(target_host):
raise ConnectionError('Host [%s] unaccessible from: %s' %
(target_host, self.host))
self._ensure_send_line(fixed_cmd)
# handle login process
# only remote connections will require login info
if is_serial_port_mode or not do_remote_connect:
nexts = PROMPT_WAIT_INPUT
self._send_all('\r\n')
else:
nexts = PROMPT_WAIT_LOGIN
while True:
out = self.read_until(nexts,
connect_timeout,
ignore_error=True)
# to avoid cases when remote system doesn't need password because of RSA key
#if 'ssh' in fixed_cmd and not login_password_sent:
# append_out = self.read_until(PROMPT_WAIT_INPUT, 0.3, ignore_error=True)
# if append_out.strip(): out = append_out
# match out read with cases
if in_search(LoginCases.INPUT_WAIT_TIMEOUT.value, out):
self._ensure_send_line()
elif in_search(LoginCases.RSA_KEY_CORRUPTED.value,
out.lower()):
self.pty_rm_known_hosts()
break
elif in_search(LoginCases.INPUT_WAIT_YES_NO.value, out):
self._ensure_send_line('yes')
elif in_search(LoginCases.INPUT_WAIT_USERNAME.value, out):
if login_user:
self._ensure_send_line(login_user)
elif login_password:
self._ensure_send_line(login_password)
login_user, login_password = login_password, login_user
else:
raise ConnectionError('Need login info to %s: %s' %
(cmd_word, target_host))
nexts = PROMPT_WAIT_INPUT + PROMPT_WAIT_LOGIN
elif in_search(LoginCases.INPUT_WAIT_PASSWORD.value, out):
if login_password:
self._ensure_send_line(login_password,
text_visible=False)
elif login_user:
self._ensure_send_line(login_user, text_visible=False)
login_user, login_password = login_password, login_user
else:
raise ConnectionError('Need login info to %s: %s' %
(cmd_word, target_host))
nexts = PROMPT_WAIT_INPUT
else:
if out and not in_search(
LoginCases.CONNECTION_REFUSED.value, out.lower()):
is_cisco_sol_mode = True if in_search(
LoginCases.CISCO_SOL_ENTERED.value, out) else False
prompt_info = utils.get_prompt_line(out)
serial_connect = is_serial_port_mode or is_cisco_sol_mode
if self._s_verify_prompt(prompt_info, login_user,
serial_connect):
session_connected = True
elif serial_connect:
# watch system booting, only serially connected system will be watched.
t_end_watch = time.time() + bootup_watch_timeout
while time.time() <= t_end_watch:
self._send_all('\r\n')
boot_stream = self.read_until(
PROMPT_WAIT_INPUT,
bootup_watch_period,
ignore_error=True)
if do_boot_check: out = out + boot_stream
if boot_stream and boot_stream.count('\n') in (
1, 2
) and self._s_verify_termchar(boot_stream):
session_connected = True
break
if not session_connected:
connect_retry -= 1 # one login attempt failed
break # one login attempt completed
if session_connected:
# Connected Successfully
# Set pty linesep for new session
if do_boot_check:
exp_raise = self._expect(out, seqcmdargs.get('boot_expect'))
esc_raise = self._escape(out, seqcmdargs.get('boot_escape'))
if exp_raise or esc_raise:
raise ExpectError('Expect failure found while booting: %r' \
%(seqcmdargs.get('boot_expect') if exp_raise else seqcmdargs.get('boot_escape')))
prompt_read = prompt_read_prev = None
retry = session_prompt_retry
while retry > 0:
self.flush()
self._send_all('\r\n')
s = self.read_until(PROMPT_WAIT_INPUT,
session_prompt_retry_timeout,
ignore_error=True)
if s and s.count('\n') in (1,
2) and self._s_verify_termchar(s):
if s.count('\n') == 2: self.pty_linesep = '\n'
else: self.pty_linesep = '\r\n'
self.flush(delaybeforeflush=0.1)
self._ensure_send_line()
prompt_line = self.read_until(PROMPT_WAIT_INPUT,
session_prompt_retry_timeout,
ignore_error=True)
if prompt_line and prompt_line.count(
'\n') == 1: # do postly verify
prompt_read_prev = utils.get_prompt_line(prompt_line)
if 'telnet' in fixed_cmd:
prompt_read_prev = utils.prompt_strip_date(
prompt_read_prev)
break
retry -= 1
if retry == 0:
raise ConnectionError(
'Pty set linesep failed in new session: %s, [%s, %s]' %
(connect_session, s, prompt_read_prev))
# Set pty prompt for new session
retry = session_prompt_retry
while retry > 0:
self.flush(delaybeforeflush=delay_before_prompt_flush)
self._ensure_send_line()
s = self.read_until(PROMPT_WAIT_INPUT,
session_prompt_retry_timeout,
ignore_error=True)
prompt_info = utils.get_prompt_line(s)
# Strip dynamic datetime part of prompt for telnet session
if 'telnet' in fixed_cmd:
prompt_info = utils.prompt_strip_date(prompt_info)
if self._s_verify_prompt(prompt_info, login_user,
serial_connect):
if not prompt_read_prev:
prompt_read_prev = prompt_info
continue
if not prompt_read:
prompt_read = prompt_info
if prompt_read == prompt_read_prev: break
# do postly verify
prompt_read_prev = prompt_read
prompt_read = None
retry -= 1
if retry == 0:
raise ConnectionError(
'Pty set prompt failed in new session: %s, [%s, %s]' %
(connect_session, s, prompt_read))
# Update agent
self.host = target_host
self.current_session = connect_session
self.user = login_user
self.password = login_password
self.serial_port_mode = is_serial_port_mode
self.cisco_sol_mode = is_cisco_sol_mode
self.prompt = prompt_read
self.command_timeout = seqcmdargs[
'command_timeout'] if seqcmdargs.get(
'command_timeout') else remote_command_timeout
session_info = {
"target_host": self.host,
"session": self.current_session,
"user": self.user,
"password": self.password,
"prompt": self.prompt,
"serial_port_mode": self.serial_port_mode,
"cisco_sol_mode": self.cisco_sol_mode,
"pty_linesep": self.pty_linesep,
"command_timeout": self.command_timeout
}
self.session_info_chain.append(session_info)
return True
# Connect Failed
raise ConnectionError('%s to %s failed with 3 retry' %
(cmd_word, target_host))
def _s_verify_termchar(self, s):
return any(
in_search(p, s[-prompt_offset_range:]) for p in PROMPT_WAIT_INPUT)
def run_all(self):
total = len(self.test_sequence)
last_recover_loop = 0
test_recover_retry = session_recover_retry
self.complt_loops = 0
while self.complt_loops < self.test_loops:
loop_result = Messages.LOOP_RESULT_PASS
loop_failure_messages = []
current = 0
self.spawned_workers = []
while current < total:
command = self.test_sequence[current]
if command.builtin:
if command.action == 'INTR':
try:
self.agent.send_control('c')
except Exception as err:
self.error_logging(
self.format_error_message('INTR', err) +
newline + newline)
loop_result = Messages.LOOP_RESULT_UNKNOWN
loop_failure_messages = [repr(err)]
self.agent.flush()
elif command.action == 'QUIT':
try:
self.agent.quit()
except Exception as err:
self.error_logging(
self.format_error_message('QUIT', err) +
newline + newline)
loop_result = Messages.LOOP_RESULT_UNKNOWN
loop_failure_messages = [repr(err)]
self.agent.flush()
elif command.action == 'CLOSE':
self.agent.close_pty()
elif command.action == 'PULSE':
try:
self.agent.pty_pulse_session()
except Exception as err:
self.error_logging(
self.format_error_message('PULSE', err) +
newline + newline)
loop_result = Messages.LOOP_RESULT_UNKNOWN
loop_failure_messages = [repr(err)]
elif command.action == 'WAIT':
seconds = utils.parse_time_to_sec(command.argv[1])
time.sleep(seconds)
elif command.action == 'SET_PROMPT':
try:
self.agent.set_pty_prompt(command.argv[1])
except Exception as err:
self.error_logging(
self.format_error_message('SET PROMPT', err) +
newline + newline)
loop_result = Messages.LOOP_RESULT_UNKNOWN
loop_failure_messages = [repr(err)]
elif command.action == 'ENTER':
command.command = ''
self.run_sequence_command(command)
elif command.action == 'FIND':
target_found = False
outputs = []
for d in command.find_dir:
if 'cd' in d or re.search(r"^FS\d+:$", d.strip()):
command.command = d
else:
command.command = 'cd ' + d
self.run_sequence_command(command)
result, message, output = self.run_sequence_command(
BuiltinCommand(action='SEND', command='ls'))
outputs.append(output)
if utils.in_search(command.target_file, output):
target_found = True
break
if not target_found:
ferr = FileError('File not found: %s' %
(command.target_file),
outputs=outputs)
self.error_logging(repr(ferr) + newline)
loop_result = Messages.LOOP_RESULT_UNKNOWN
loop_failure_messages = [
repr(ferr),
]
if self.errordump:
loop_failure_messages.append(
repr(self.errordump))
elif command.action == 'NEW_WORKER':
new_worker = Process(target=run_sequence_worker,
args=(
self.display_control,
command.sequence_file,
command.loops,
))
#if command.wait: new_worker.daemon = True
new_worker.start() # Start worker
ipc_message = {
'MSG': Messages.SEQUENCE_RUNNING_START.value,
'NAME': command.sequence_file.split('.')[0],
'LOOPS': command.loops
}
self.send_ipc_msg(ipc_message)
#print('Spawned new worker [pid : %r] for sequence: %s' %(worker.pid, command.argv[1]))
# wait for derived sequence worker to complete if wait flag is set
if command.wait: new_worker.join()
self.spawned_workers.append(new_worker)
elif command.action == 'MONITOR':
cont = True
while cont:
result, message, output = self.run_sequence_command(
command)
for w in command.watch:
if w in output:
cont = False
break
if cont:
time.sleep(command.interval
if command.interval > 0 else 0)
elif command.action == 'LOOP':
start = sequence.SUBSEQUENCES[
command.subsequence_name]['start']
end = sequence.SUBSEQUENCES[
command.subsequence_name]['end']
test_loops_save = self.test_loops
complt_loops_save = self.complt_loops
test_sequence_save = self.test_sequence
sequence_file_save = self.sequence_file
self.sequence_file = command.subsequence_name
self.test_loops = command.loops
self.test_sequence = self.test_sequence[start:end]
ipc_message = {
'MSG': Messages.SEQUENCE_RUNNING_START.value,
'NAME': self.sequence_file.split('.')[0],
'LOOPS': command.loops
}
self.send_ipc_msg(ipc_message)
self.run_all()
ipc_message = {
'MSG': Messages.SEQUENCE_RUNNING_COMPLETE.value,
'NAME': self.sequence_file.split('.')[0]
}
self.send_ipc_msg(ipc_message)
self.test_loops = test_loops_save
self.test_sequence = test_sequence_save
self.complt_loops = complt_loops_save
self.sequence_file = sequence_file_save
else:
result, message, output = self.run_sequence_command(
command)
if result == Messages.ITEM_RESULT_UNKNOWN:
loop_result = Messages.LOOP_RESULT_UNKNOWN
loop_failure_messages = [repr(self.errordump)]
elif result == Messages.ITEM_RESULT_FAIL:
loop_result = Messages.LOOP_RESULT_FAIL
loop_failure_messages.append(message)
# reset loop environments, restart current loop from sequence begining
if loop_result == Messages.LOOP_RESULT_UNKNOWN:
# DO LOOP RECOVERY
if test_recover_retry == 0:
err = RecoveryError(
'Recovery failed after %d retry at loop %d' %
(session_recover_retry, self.complt_loops + 1))
self.error_logging(
newline +
'****************ERROR DUMP END****************' +
newline)
err_msg = newline + repr(err) + newline
self.error_logging(err_msg + newline)
self.stop()
time.sleep(5)
return
if (self.complt_loops + 1) == last_recover_loop:
test_recover_retry -= 1
else:
last_recover_loop = self.complt_loops + 1
test_recover_retry = session_recover_retry
ipc_message = {
'MSG': loop_result.value,
'NAME': self.sequence_file.split('.')[0],
'LOOP': self.complt_loops + 1,
'MSG_Q': loop_failure_messages
}
self.send_ipc_msg(ipc_message)
# reset loop scope variables
for worker in self.spawned_workers:
worker.kill()
time.sleep(0.1)
self.agent.close_pty()
loop_result = Messages.LOOP_RESULT_PASS
loop_failure_messages = []
self.spawned_workers = []
current = 0
else:
current += 1
ipc_message = {
'MSG': loop_result.value,
'NAME': self.sequence_file.split('.')[0],
'LOOP': self.complt_loops + 1,
'MSG_Q': loop_failure_messages
}
self.send_ipc_msg(ipc_message)
# move on to next loop
#self.agent.close_pty()
self.complt_loops += 1