def is_bash_type(bash: pexpect.spawn, cmd: str) -> bool:
typecmd = "type %s &>/dev/null && echo -n 0 || echo -n 1" % cmd
bash.sendline(typecmd)
bash.expect_exact(typecmd + "\r\n")
result = bash.expect_exact(["0", "1"]) == 0
bash.expect_exact(PS1)
return result
def _run_expect(self, child: pexpect.spawn) -> Optional[str]:
if self._database.has_password():
child.expect('Enter password to unlock {}: '.format(
self._database.get_path()))
child.sendline(self._database.get_password())
child.expect(pexpect.EOF)
return child.before
def run_cheribsd_command_or_die(qemu: pexpect.spawn, cmd: str, timeout=600):
qemu.sendline(
test_command +
" ;if test $? -eq 0; then echo 'COMMAND' 'SUCCESSFUL'; else echo 'COMMAND' 'FAILED'; fi"
)
i = qemu.expect([
pexpect.TIMEOUT, "COMMAND SUCCESSFUL", "COMMAND FAILED", PANIC,
CHERI_TRAP, STOPPED
],
timeout=timeout)
testtime = datetime.datetime.now() - run_tests_starttime
if i == 0: # Timeout
return failure("timeout after",
testtime,
"waiting for tests: ",
str(qemu),
exit=False)
elif i == 1:
success("===> Tests completed!")
success("Running tests took ", testtime)
return True
else:
return failure("error after ",
testtime,
"while running tests : ",
str(qemu),
exit=False)
def assert_bash_exec(
bash: pexpect.spawn, cmd: str, want_output: bool = False) -> str:
# Send command
bash.sendline(cmd)
bash.expect_exact(cmd)
# Find prompt, output is before it
bash.expect_exact("\r\n" + PS1)
output = bash.before
# Retrieve exit status
echo = "echo $?"
bash.sendline(echo)
got = bash.expect([
r"^%s\r\n(\d+)\r\n%s" % (re.escape(echo), re.escape(PS1)),
PS1,
pexpect.EOF,
pexpect.TIMEOUT,
])
status = bash.match.group(1) if got == 0 else "unknown"
assert status == "0", \
'Error running "%s": exit status=%s, output="%s"' % \
(cmd, status, output)
if output:
assert want_output, \
'Unexpected output from "%s": exit status=%s, output="%s"' % \
(cmd, status, output)
else:
assert not want_output, \
'Expected output from "%s": exit status=%s, output="%s"' % \
(cmd, status, output)
return output
def is_bash_type(bash: pexpect.spawn, cmd: str) -> bool:
typecmd = "type %s &>/dev/null && echo -n 0 || echo -n 1" % cmd
bash.sendline(typecmd)
bash.expect_exact(typecmd + "\r\n")
result = bash.expect_exact(["0", "1"]) == 0
bash.expect_exact(PS1)
return result
def run_cheribsd_command(qemu: pexpect.spawn,
cmd: str,
expected_output=None,
error_output=None,
timeout=60):
qemu.sendline(cmd)
if expected_output:
qemu.expect(expected_output)
results = [
pexpect.TIMEOUT, PROMPT, "/bin/sh: [\\w\\d_-]+: not found", CHERI_TRAP
]
if error_output:
results.append(error_output)
i = qemu.expect(results, timeout=timeout)
if i == 0:
failure("timeout running ", cmd)
elif i == 2:
failure("Command not found!")
elif i == 3:
# wait up to 20 seconds for a prompt to ensure the dump output has been printed
qemu.expect([pexpect.TIMEOUT, PROMPT], timeout=20)
qemu.flush()
failure("Got CHERI TRAP!")
elif i == 4:
# wait up to 5 seconds for a prompt to ensure the full output has been printed
qemu.expect([pexpect.TIMEOUT, PROMPT], timeout=5)
qemu.flush()
failure("Matched error output ", error_output)
def debug_kernel_panic(qemu: pexpect.spawn):
# wait up to 10 seconds for a db prompt
i = qemu.expect([pexpect.TIMEOUT, "db> "], timeout=10)
if i == 1:
qemu.sendline("bt")
# wait for the backtrace
qemu.expect([pexpect.TIMEOUT, "db> "], timeout=30)
failure("GOT KERNEL PANIC!", exit=False)
def assert_bash_exec(
bash: pexpect.spawn,
cmd: str,
want_output: Optional[bool] = False,
want_newline=True,
) -> str:
"""
:param want_output: if None, don't care if got output or not
"""
# Send command
bash.sendline(cmd)
bash.expect_exact(cmd)
# Find prompt, output is before it
bash.expect_exact("%s%s" % ("\r\n" if want_newline else "", PS1))
output = bash.before
# Retrieve exit status
echo = "echo $?"
bash.sendline(echo)
got = bash.expect(
[
r"^%s\r\n(\d+)\r\n%s" % (re.escape(echo), re.escape(PS1)),
PS1,
pexpect.EOF,
pexpect.TIMEOUT,
]
)
status = bash.match.group(1) if got == 0 else "unknown"
assert status == "0", 'Error running "%s": exit status=%s, output="%s"' % (
cmd,
status,
output,
)
if want_output is not None:
if output:
assert (
want_output
), 'Unexpected output from "%s": exit status=%s, output="%s"' % (
cmd,
status,
output,
)
else:
assert (
not want_output
), 'Expected output from "%s": exit status=%s, output="%s"' % (
cmd,
status,
output,
)
return output
def _run_expect(self, child: pexpect.spawn) -> Optional[str]:
if self._database.has_password():
child.expect('Enter password to unlock {}: '.format(
self._database.get_path()))
child.sendline(self._database.get_password())
if self._database_from.has_password():
child.expect('Enter password to unlock {}: '.format(
self._database_from.get_path()))
child.sendline(self._database_from.get_password())
child.expect('Database was not modified by merge operation.')
return child.before
def run_noop_test(qemu: pexpect.spawn, args: argparse.Namespace):
import boot_cheribsd
boot_cheribsd.success("Booted successfully")
boot_cheribsd.run_cheribsd_command(qemu, "mount_smbfs --help", cheri_trap_fatal=False)
boot_cheribsd.run_cheribsd_command(qemu, "/libexec/ld-cheri-elf.so.1 --help")
poweroff_start = datetime.datetime.now()
qemu.sendline("poweroff")
i = qemu.expect(["Uptime:", pexpect.TIMEOUT, pexpect.EOF] + boot_cheribsd.FATAL_ERROR_MESSAGES, timeout=20)
if i != 0:
boot_cheribsd.failure("Poweroff " + ("timed out" if i == 1 else "failed"))
return False
i = qemu.expect([pexpect.TIMEOUT, pexpect.EOF], timeout=20)
if i == 0:
boot_cheribsd.failure("QEMU didn't exit after shutdown!")
return False
boot_cheribsd.success("Poweroff took: ", datetime.datetime.now() - poweroff_start)
return True
def wait_for(child: pexpect.spawn, pattern: str, answer: str):
countdown = 1024
while True:
hit = child.expect([pattern, pexpect.TIMEOUT, pexpect.EOF], timeout=15)
if hit == 0:
if answer and len(answer):
child.sendline(answer)
return 0
if hit == 1:
print('timeout on pattern : ', pattern)
sigterm(signal.SIGTERM, None)
return 1
if hit == 2:
countdown -= 1
if countdown == 0:
print('too much EOF on pattern : ', pattern)
return 2
def assert_bash_exec(bash: pexpect.spawn,
cmd: str,
want_output: bool = False) -> str:
# Send command
bash.sendline(cmd)
bash.expect_exact(cmd)
# Find prompt, output is before it
bash.expect_exact("\r\n" + PS1)
output = bash.before
# Retrieve exit status
echo = "echo $?"
bash.sendline(echo)
got = bash.expect([
r"^%s\r\n(\d+)\r\n%s" % (re.escape(echo), re.escape(PS1)),
PS1,
pexpect.EOF,
pexpect.TIMEOUT,
])
status = bash.match.group(1) if got == 0 else "unknown"
assert status == "0", 'Error running "%s": exit status=%s, output="%s"' % (
cmd,
status,
output,
)
if output:
assert want_output, (
'Unexpected output from "%s": exit status=%s, output="%s"' %
(cmd, status, output))
else:
assert not want_output, (
'Expected output from "%s": exit status=%s, output="%s"' %
(cmd, status, output))
return output
def setup_ssh(qemu: pexpect.spawn, pubkey: Path):
run_cheribsd_command(qemu, "mkdir -p /root/.ssh")
contents = pubkey.read_text(encoding="utf-8").strip()
run_cheribsd_command(
qemu,
"echo " + shlex.quote(contents) + " >> /root/.ssh/authorized_keys")
run_cheribsd_command(qemu, "chmod 600 /root/.ssh/authorized_keys")
run_cheribsd_command(
qemu,
"echo 'PermitRootLogin without-password' >> /etc/ssh/sshd_config")
# TODO: check for bluehive images without /sbin/service
run_cheribsd_command(qemu,
"cat /root/.ssh/authorized_keys",
expected_output="ssh-")
run_cheribsd_command(qemu, "grep -n PermitRootLogin /etc/ssh/sshd_config")
qemu.sendline("service sshd restart")
i = qemu.expect([pexpect.TIMEOUT, "service: not found", "Starting sshd."],
timeout=120)
if i == 0:
failure("Timed out setting up SSH keys")
qemu.expect(PROMPT)
time.sleep(2) # sleep for two seconds to avoid a rejection
success("===> SSH authorized_keys set up")
def worker_run_task_in_subprocess(sp: pexpect.spawn, *args, **kwargs):
# TODO check the buffer size rules for stdin and stdout and directly
# stream bytes instead of saving them to the file system. See 'encoding='
# argument in pexpect.spawn constructor
# Serialize args and kwargs to a file
path_to_args = os.path.join(tempfile.gettempdir(), f'{uuid.uuid4()}.pkl')
with open(path_to_args, 'wb') as args_file:
cloudpickle.dump([args, kwargs], args_file)
# Prepare result and error files
path_to_result = os.path.join(tempfile.gettempdir(), f'{uuid.uuid4()}.pkl')
path_to_error = os.path.join(tempfile.gettempdir(), f'{uuid.uuid4()}.pkl')
# Communicate to the subprocess
sp.expect('Ready for next task.') # Printed in the subprocess loop
sp.sendline(path_to_args)
sp.sendline(path_to_result)
sp.sendline(path_to_error)
# Wait for the subprocess to complete
sp.expect('Task complete.') # Printed in the subprocess loop
# Check for an exception to be reraised
error: SerializableException = None
try:
with open(path_to_error, 'rb') as error_file:
error: SerializableException = cloudpickle.load(error_file)
except FileNotFoundError:
pass
# Check for a result to be returned
result = None
try:
with open(path_to_result, 'rb') as result_file:
result = cloudpickle.load(result_file)
except FileNotFoundError as e:
pass
# Remove the args, result and error files
try:
os.remove(path_to_args)
except (FileNotFoundError, NameError):
pass
try:
os.remove(path_to_result)
except (FileNotFoundError, NameError):
pass
try:
os.remove(path_to_error)
except (FileNotFoundError, NameError):
pass
# Finish up (reraise error or return result)
if error is not None:
error.reraise()
return result
def connect(self, connection=None):
"""
See :meth:`BaseShell.connect` for more information.
"""
connection = connection or self._default_connection or '0'
if connection in self._connections and self.is_connected(connection):
raise AlreadyConnectedError(connection)
# Create a child process
spawn = Spawn(
self._get_connect_command().strip(),
**self._spawn_args
)
self._connections[connection] = spawn
try:
# If connection is via user
if self._user is not None:
spawn.expect(
[self._user_match], timeout=self._timeout
)
spawn.sendline(self._user)
# If connection is via password
if self._password is not None:
spawn.expect(
[self._password_match], timeout=self._timeout
)
spawn.sendline(self._password)
# Setup shell before using it
self._setup_shell(connection)
# Execute initial command if required
if self._initial_command is not None:
spawn.expect(
self._prompt, timeout=self._timeout
)
spawn.sendline(self._initial_command)
# Wait for command response to match the prompt
spawn.expect(
self._prompt, timeout=self._timeout
)
except:
# Always remove bad connections if it failed
del self._connections[connection]
raise
# Set connection as default connection if required
if self.default_connection is None:
self.default_connection = connection
def connect(self, connection=None):
"""
See :meth:`BaseShell.connect` for more information.
"""
connection = connection or self._default_connection or '0'
if connection in self._connections and self.is_connected(connection):
raise AlreadyConnectedError(connection)
# Create a child process
spawn = Spawn(self._get_connect_command().strip(), **self._spawn_args)
self._connections[connection] = spawn
try:
# If connection is via user
if self._user is not None:
spawn.expect([self._user_match], timeout=self._timeout)
spawn.sendline(self._user)
# If connection is via password
if self._password is not None:
spawn.expect([self._password_match], timeout=self._timeout)
spawn.sendline(self._password)
# Setup shell before using it
self._setup_shell(connection)
# Execute initial command if required
if self._initial_command is not None:
spawn.expect(self._prompt, timeout=self._timeout)
spawn.sendline(self._initial_command)
# Wait for command response to match the prompt
spawn.expect(self._prompt, timeout=self._timeout)
except:
# Always remove bad connections if it failed
del self._connections[connection]
raise
# Set connection as default connection if required
if self.default_connection is None:
self.default_connection = connection
def send_cmd(child: pexpect.spawn, cmd):
prompt = ['# ', '>>> ', '> ', '\$ ']
child.sendline(cmd)
child.expect(prompt)
print(child.before.decode())
def spawnAction(self, sp: spawn):
print("sendline {}".format(self))
yield sp.sendline(self)
def runtests(
qemu: pexpect.spawn,
test_archives: list,
test_command: str,
smb_dir: typing.Optional[Path],
ssh_keyfile: typing.Optional[str],
ssh_port: typing.Optional[int],
timeout: int,
test_function: "typing.Callable[[pexpect.spawn, ...], bool]" = None
) -> bool:
setup_tests_starttime = datetime.datetime.now()
# disable coredumps, otherwise we get no space left on device errors
run_cheribsd_command(qemu, "sysctl kern.coredump=0")
# create tmpfs on opt
run_cheribsd_command(
qemu, "mkdir -p /opt && mount -t tmpfs -o size=500m tmpfs /opt")
# ensure that /usr/local exists and if not create it as a tmpfs (happens in the minimal image)
run_cheribsd_command(
qemu,
"mkdir -p /usr/local && mount -t tmpfs -o size=300m tmpfs /usr/local")
run_cheribsd_command(qemu, "df -h", expected_output="/opt")
info("\nWill transfer the following archives: ", test_archives)
# strip the .pub from the key file
for archive in test_archives:
if smb_dir:
run_host_command(["tar", "xJf", str(archive), "-C", str(smb_dir)])
else:
# Extract to temporary directory and scp over
with tempfile.TemporaryDirectory(dir=os.getcwd(),
prefix="test_files_") as tmp:
run_host_command(["tar", "xJf", str(archive), "-C", tmp])
private_key = str(Path(ssh_keyfile).with_suffix(""))
scp_cmd = [
"scp", "-B", "-r", "-P",
str(ssh_port), "-o", "StrictHostKeyChecking=no", "-o",
"UserKnownHostsFile=/dev/null", "-i",
shlex.quote(private_key), ".", "root@localhost:/"
]
# use script for a fake tty to get progress output from scp
if sys.platform.startswith("linux"):
scp_cmd = [
"script", "--quiet", "--return", "--command",
" ".join(scp_cmd), "/dev/null"
]
run_host_command(["ls", "-la"], cwd=tmp)
run_host_command(scp_cmd, cwd=tmp)
if test_archives:
time.sleep(5) # wait 5 seconds to make sure the disks have synced
# See how much space we have after running scp
run_cheribsd_command(qemu, "df -h", expected_output="/opt")
success("Preparing test enviroment took ",
datetime.datetime.now() - setup_tests_starttime)
run_tests_starttime = datetime.datetime.now()
# Run the tests (allowing custom test functions)
if test_function:
return test_function(qemu, ssh_keyfile=ssh_keyfile, ssh_port=ssh_port)
qemu.sendline(
test_command +
" ;if test $? -eq 0; then echo 'TESTS' 'COMPLETED'; else echo 'TESTS' 'FAILED'; fi"
)
i = qemu.expect(
[pexpect.TIMEOUT, "TESTS COMPLETED", "TESTS FAILED", PANIC, STOPPED],
timeout=timeout)
testtime = datetime.datetime.now() - run_tests_starttime
if i == 0: # Timeout
return failure("timeout after",
testtime,
"waiting for tests: ",
str(qemu),
exit=False)
elif i == 1:
success("===> Tests completed!")
success("Running tests took ", testtime)
run_cheribsd_command(
qemu, "df -h",
expected_output="/opt") # see how much space we have now
return True
else:
return failure("error after ",
testtime,
"while running tests : ",
str(qemu),
exit=False)
def connect(self, connection=None):
"""
See :meth:`BaseShell.connect` for more information.
"""
connection = connection or self._default_connection or '0'
if connection in self._connections and self.is_connected(connection):
raise AlreadyConnectedError(connection)
# Inject framework logger to the spawn object
spawn_args = {
'logfile': get_logger(
OrderedDict([
('node_identifier', self._node_identifier),
('shell_name', self._shell_name),
('connection', connection)
]),
category='pexpect'
),
}
# Create a child process
spawn_args.update(self._spawn_args)
spawn = Spawn(
self._get_connect_command().strip(),
**spawn_args
)
# Add a connection logger
# Note: self._node and self._name were added to this shell in the
# node's call to its _register_shell method.
spawn._connection_logger = get_logger(
OrderedDict([
('node_identifier', self._node_identifier),
('shell_name', self._shell_name),
('connection', connection)
]),
category='connection'
)
self._connections[connection] = spawn
try:
# If connection is via user
if self._user is not None:
spawn.expect(
[self._user_match], timeout=self._timeout
)
spawn.sendline(self._user)
# If connection is via password
if self._password is not None:
spawn.expect(
[self._password_match], timeout=self._timeout
)
spawn.sendline(self._password)
# Setup shell before using it
self._setup_shell(connection)
# Execute initial command if required
if self._initial_command is not None:
spawn.expect(
self._prompt, timeout=self._timeout
)
spawn.sendline(self._initial_command)
# Wait for command response to match the prompt
spawn.expect(
self._prompt, timeout=self._timeout
)
except:
# Always remove bad connections if it failed
del self._connections[connection]
raise
# Set connection as default connection if required
if self.default_connection is None:
self.default_connection = connection