def _build_kmod(self, kernel_dir, kmod):
kernel_build_dir = kernel_dir / "build"
# External modules can't do out-of-tree builds for some reason, so copy
# the source files to a temporary directory and build the module there,
# then move it to the final location.
kmod_source_dir = Path("tests/linux_kernel/kmod")
source_files = ("drgn_test.c", "Makefile")
if out_of_date(
kmod,
*[kmod_source_dir / filename for filename in source_files]):
with tempfile.TemporaryDirectory(dir=kmod.parent) as tmp_name:
tmp_dir = Path(tmp_name)
# Make sure that header files have the same paths as in the
# original kernel build.
debug_prefix_map = [
f"{kernel_build_dir.resolve()}=.",
f"{tmp_dir.resolve()}=./drgn_test",
]
cflags = " ".join(
["-fdebug-prefix-map=" + map for map in debug_prefix_map])
for filename in source_files:
copy_file(kmod_source_dir / filename, tmp_dir / filename)
if (subprocess.call([
"make",
"-C",
kernel_build_dir,
f"M={tmp_dir.resolve()}",
"KAFLAGS=" + cflags,
"KCFLAGS=" + cflags,
"-j",
str(nproc()),
]) != 0):
return False
(tmp_dir / "drgn_test.ko").rename(kmod)
return True
async def build_kernel(commit: str, build_dir: str,
log_file: TextIO) -> Tuple[str, str]:
"""
Returns built kernel release (i.e., `uname -r`) and image name (e.g.,
`arch/x86/boot/bzImage`).
"""
await check_call("git",
"checkout",
commit,
stdout=log_file,
stderr=asyncio.subprocess.STDOUT)
shutil.copy(
os.path.join(os.path.dirname(__file__), "config"),
os.path.join(build_dir, ".config"),
)
logger.info("building %s", commit)
start = time.monotonic()
cflags = f"-fdebug-prefix-map={os.path.join(getpwd(), build_dir)}="
kbuild_args = [
"KBUILD_BUILD_USER=drgn",
"KBUILD_BUILD_HOST=drgn",
"KAFLAGS=" + cflags,
"KCFLAGS=" + cflags,
"O=" + build_dir,
"-j",
str(nproc()),
]
await check_call(
"make",
*kbuild_args,
"olddefconfig",
"all",
stdout=log_file,
stderr=asyncio.subprocess.STDOUT,
)
elapsed = time.monotonic() - start
logger.info("built %s in %s", commit, humanize_duration(elapsed))
vmlinux = os.path.join(build_dir, "vmlinux")
release, image_name = (await asyncio.gather(
post_process_vmlinux(vmlinux,
stdout=log_file,
stderr=asyncio.subprocess.STDOUT),
check_output("make",
*kbuild_args,
"-s",
"kernelrelease",
stderr=log_file),
check_output("make", *kbuild_args, "-s", "image_name",
stderr=log_file),
))[1:]
return release.decode().strip(), image_name.decode().strip()
async def _prepare_make(self) -> Tuple[str, ...]:
if self._cached_make_args is None:
self._build_dir.mkdir(parents=True, exist_ok=True)
debug_prefix_map = []
# GCC uses the "logical" working directory, i.e., the PWD
# environment variable, when it can. See
# https://gcc.gnu.org/git/?p=gcc.git;a=blob;f=libiberty/getpwd.c;hb=HEAD.
# Map both the canonical and logical paths.
build_dir_real = self._build_dir.resolve()
debug_prefix_map.append(str(build_dir_real) + "=.")
build_dir_logical = (await check_output_shell(
f"cd {shlex.quote(str(self._build_dir))}; pwd -L", )
).decode()[:-1]
if build_dir_logical != str(build_dir_real):
debug_prefix_map.append(build_dir_logical + "=.")
# Before Linux kernel commit 25b146c5b8ce ("kbuild: allow Kbuild to
# start from any directory") (in v5.2), O= forces the source
# directory to be absolute. Since Linux kernel commit 95fd3f87bfbe
# ("kbuild: add a flag to force absolute path for srctree") (in
# v5.3), KBUILD_ABS_SRCTREE=1 does the same. This means that except
# for v5.2, which we don't support, the source directory will
# always be absolute, and we don't need to worry about mapping it
# from a relative path.
kernel_dir_real = self._kernel_dir.resolve()
if kernel_dir_real != build_dir_real:
debug_prefix_map.append(str(kernel_dir_real) + "/=./")
cflags = " ".join(
["-fdebug-prefix-map=" + map for map in debug_prefix_map])
self._cached_make_args = (
"-C",
str(self._kernel_dir),
"ARCH=" + str(self._arch),
"O=" + str(build_dir_real),
"KBUILD_ABS_SRCTREE=1",
"KBUILD_BUILD_USER=drgn",
"KBUILD_BUILD_HOST=drgn",
"HOSTCFLAGS=" + cflags,
"KAFLAGS=" + cflags,
"KCFLAGS=" + cflags,
"-j",
str(nproc()),
)
return self._cached_make_args
def finalize_options(self):
super().finalize_options()
if self.parallel is None:
self.parallel = nproc() + 1
def run_in_vm(command: str, kernel_dir: Path, build_dir: Path) -> int:
match = re.search(
"QEMU emulator version ([0-9]+(?:\.[0-9]+)*)",
subprocess.check_output(["qemu-system-x86_64", "-version"],
universal_newlines=True),
)
if not match:
raise Exception("could not determine QEMU version")
qemu_version = tuple(int(x) for x in match.group(1).split("."))
# multidevs was added in QEMU 4.2.0.
multidevs = ",multidevs=remap" if qemu_version >= (4, 2) else ""
# QEMU's 9pfs O_NOATIME handling was fixed in 5.1.0. The fix was backported
# to 5.0.1.
env = os.environ.copy()
if qemu_version < (5, 0, 1):
onoatimehack_so = _build_onoatimehack(build_dir)
env["LD_PRELOAD"] = f"{str(onoatimehack_so)}:{env.get('LD_PRELOAD', '')}"
with tempfile.TemporaryDirectory(
prefix="drgn-vmtest-") as temp_dir, socket.socket(
socket.AF_UNIX) as server_sock:
temp_path = Path(temp_dir)
socket_path = temp_path / "socket"
server_sock.bind(str(socket_path))
server_sock.listen()
busybox = shutil.which("busybox")
if busybox is None:
raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT),
"busybox")
init = (temp_path / "init").resolve()
with open(init, "w") as init_file:
init_file.write(
_INIT_TEMPLATE.format(busybox=shlex.quote(busybox),
command=shlex.quote(command)))
os.chmod(init, 0o755)
with subprocess.Popen(
[
# fmt: off
"qemu-system-x86_64",
"-cpu",
"host",
"-enable-kvm",
"-smp",
str(nproc()),
"-m",
"2G",
"-nodefaults",
"-display",
"none",
"-serial",
"mon:stdio",
# This along with -append panic=-1 ensures that we exit on a
# panic instead of hanging.
"-no-reboot",
"-virtfs",
f"local,id=root,path=/,mount_tag=/dev/root,security_model=none,readonly{multidevs}",
"-virtfs",
f"local,path={kernel_dir},mount_tag=modules,security_model=none,readonly",
"-device",
"virtio-serial",
"-chardev",
f"socket,id=vmtest,path={socket_path}",
"-device",
"virtserialport,chardev=vmtest,name=com.osandov.vmtest.0",
"-kernel",
str(kernel_dir / "vmlinuz"),
"-append",
f"rootfstype=9p rootflags=trans=virtio,cache=loose ro console=0,115200 panic=-1 init={init}",
# fmt: on
],
env=env,
) as qemu:
server_sock.settimeout(5)
try:
sock = server_sock.accept()[0]
except socket.timeout:
raise LostVMError(
f"QEMU did not connect within {server_sock.gettimeout()} seconds"
)
try:
status_buf = bytearray()
while True:
try:
buf = sock.recv(4)
except ConnectionResetError:
buf = b""
if not buf:
break
status_buf.extend(buf)
finally:
sock.close()
if not status_buf:
raise LostVMError("VM did not return status")
if status_buf[-1] != ord("\n") or not status_buf[:-1].isdigit():
raise LostVMError(
f"VM returned invalid status: {repr(status_buf)[11:-1]}")
return int(status_buf)
async def build_kernel(
commit: str, build_dir: Path, log_file: TextIO
) -> Tuple[str, Path]:
"""
Returns built kernel release (i.e., `uname -r`) and image name (e.g.,
`arch/x86/boot/bzImage`).
"""
await check_call(
"git", "checkout", commit, stdout=log_file, stderr=asyncio.subprocess.STDOUT
)
shutil.copy(KERNEL_CONFIG_PATH, build_dir / ".config")
logger.info("building %s", commit)
start = time.monotonic()
cflags = f"-fdebug-prefix-map={getpwd() / build_dir}="
kbuild_args = [
"KBUILD_BUILD_USER=drgn",
"KBUILD_BUILD_HOST=drgn",
"KAFLAGS=" + cflags,
"KCFLAGS=" + cflags,
"O=" + str(build_dir),
"-j",
str(nproc()),
]
await check_call(
"make",
*kbuild_args,
"olddefconfig",
"all",
stdout=log_file,
stderr=asyncio.subprocess.STDOUT,
)
elapsed = time.monotonic() - start
logger.info("built %s in %s", commit, humanize_duration(elapsed))
logger.info("packaging %s", commit)
start = time.monotonic()
release = (
(
await check_output(
"make", *kbuild_args, "-s", "kernelrelease", stderr=log_file
)
)
.decode()
.strip()
)
image_name = (
(await check_output("make", *kbuild_args, "-s", "image_name", stderr=log_file))
.decode()
.strip()
)
install_dir = build_dir / "install"
modules_dir = install_dir / "lib" / "modules" / release
await check_call(
"make",
*kbuild_args,
"INSTALL_MOD_PATH=install",
"modules_install",
stdout=log_file,
stderr=asyncio.subprocess.STDOUT,
)
# Don't want these symlinks.
(modules_dir / "build").unlink()
(modules_dir / "source").unlink()
vmlinux = modules_dir / "vmlinux"
await check_call(
"objcopy",
"--remove-relocations=*",
str(build_dir / "vmlinux"),
str(vmlinux),
stdout=log_file,
stderr=asyncio.subprocess.STDOUT,
)
vmlinux.chmod(0o644)
vmlinuz = modules_dir / "vmlinuz"
shutil.copy(build_dir / image_name, vmlinuz)
vmlinuz.chmod(0o644)
tarball = build_dir / "kernel.tar.zst"
tar_command = ("tar", "-C", str(modules_dir), "-c", ".")
zstd_command = ("zstd", "-T0", "-19", "-q", "-", "-o", str(tarball))
pipe_r, pipe_w = os.pipe()
try:
tar_proc, zstd_proc = await asyncio.gather(
asyncio.create_subprocess_exec(
*tar_command, stdout=pipe_w, stderr=log_file
),
asyncio.create_subprocess_exec(
*zstd_command,
stdin=pipe_r,
stdout=log_file,
stderr=asyncio.subprocess.STDOUT,
),
)
finally:
os.close(pipe_r)
os.close(pipe_w)
tar_returncode, zstd_returncode = await asyncio.gather(
tar_proc.wait(), zstd_proc.wait()
)
if tar_returncode != 0:
raise CalledProcessError(tar_returncode, tar_command)
if zstd_returncode != 0:
raise CalledProcessError(zstd_returncode, zstd_command)
shutil.rmtree(install_dir)
elapsed = time.monotonic() - start
logger.info("packaged %s in %s", commit, humanize_duration(elapsed))
return release, tarball
def __init__(
self,
*,
init: str,
onoatimehack: str,
vmlinux: str,
vmlinuz: str,
) -> None:
self._temp_dir = tempfile.TemporaryDirectory("drgn-vmtest-")
self._server_sock = socket.socket(socket.AF_UNIX)
socket_path = os.path.join(self._temp_dir.name, "socket")
self._server_sock.bind(socket_path)
self._server_sock.listen()
init = os.path.abspath(init)
if " " in init:
init = '"' + init + '"'
vmlinux = os.path.abspath(vmlinux)
if " " in vmlinux:
vmlinux = '"' + vmlinux + '"'
# This was added in QEMU 4.2.0.
if ("multidevs" in subprocess.run(
["qemu-system-x86_64", "-help"],
stdout=subprocess.PIPE,
universal_newlines=True,
).stdout):
multidevs = ",multidevs=remap"
else:
multidevs = ""
self._qemu = subprocess.Popen(
[
# fmt: off
"qemu-system-x86_64",
"-cpu",
"kvm64",
"-enable-kvm",
"-smp",
str(nproc()),
"-m",
"2G",
"-nodefaults",
"-display",
"none",
"-serial",
"mon:stdio",
# This along with -append panic=-1 ensures that we exit on a
# panic instead of hanging.
"-no-reboot",
"-virtfs",
f"local,id=root,path=/,mount_tag=/dev/root,security_model=none,readonly{multidevs}",
"-device",
"virtio-serial",
"-chardev",
f"socket,id=vmtest,path={socket_path}",
"-device",
"virtserialport,chardev=vmtest,name=com.osandov.vmtest.0",
"-kernel",
vmlinuz,
"-append",
f"rootfstype=9p rootflags=trans=virtio,cache=loose ro console=0,115200 panic=-1 init={init} VMLINUX={vmlinux}",
# fmt: on
],
env={
**os.environ,
"LD_PRELOAD":
f"{onoatimehack}:{os.getenv('LD_PRELOAD', '')}",
},
)
self._server_sock.settimeout(5)
try:
self._sock = self._server_sock.accept()[0]
except socket.timeout:
raise LostVMError(
f"QEMU did not connect within {self._server_sock.gettimeout()} seconds"
)
def run_in_vm(command: str, *, vmlinuz: str, build_dir: str) -> int:
# multidevs was added in QEMU 4.2.0.
if ("multidevs" in subprocess.run(
["qemu-system-x86_64", "-help"],
stdout=subprocess.PIPE,
universal_newlines=True,
).stdout):
multidevs = ",multidevs=remap"
else:
multidevs = ""
onoatimehack = _build_onoatimehack(build_dir)
with tempfile.TemporaryDirectory(
prefix="drgn-vmtest-") as temp_dir, socket.socket(
socket.AF_UNIX) as server_sock:
socket_path = os.path.join(temp_dir, "socket")
server_sock.bind(socket_path)
server_sock.listen()
busybox = shutil.which("busybox")
if busybox is None:
raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT),
"busybox")
init = os.path.abspath(os.path.join(temp_dir, "init"))
with open(init, "w") as init_file:
init_file.write(
_INIT_TEMPLATE.format(busybox=shlex.quote(busybox),
command=shlex.quote(command)))
os.chmod(init, 0o755)
with subprocess.Popen(
[
# fmt: off
"qemu-system-x86_64",
"-cpu",
"host",
"-enable-kvm",
"-smp",
str(nproc()),
"-m",
"2G",
"-nodefaults",
"-display",
"none",
"-serial",
"mon:stdio",
# This along with -append panic=-1 ensures that we exit on a
# panic instead of hanging.
"-no-reboot",
"-virtfs",
f"local,id=root,path=/,mount_tag=/dev/root,security_model=none,readonly{multidevs}",
"-device",
"virtio-serial",
"-chardev",
f"socket,id=vmtest,path={socket_path}",
"-device",
"virtserialport,chardev=vmtest,name=com.osandov.vmtest.0",
"-kernel",
vmlinuz,
"-append",
f"rootfstype=9p rootflags=trans=virtio,cache=loose ro console=0,115200 panic=-1 init={init}",
# fmt: on
],
env={
**os.environ,
"LD_PRELOAD":
f"{onoatimehack}:{os.getenv('LD_PRELOAD', '')}",
},
) as qemu:
server_sock.settimeout(5)
try:
sock = server_sock.accept()[0]
except socket.timeout:
raise LostVMError(
f"QEMU did not connect within {server_sock.gettimeout()} seconds"
)
try:
status_buf = bytearray()
while True:
try:
buf = sock.recv(4)
except ConnectionResetError:
buf = b""
if not buf:
break
status_buf.extend(buf)
finally:
sock.close()
if not status_buf:
raise LostVMError("VM did not return status")
if status_buf[-1] != ord("\n") or not status_buf[:-1].isdigit():
raise LostVMError(
f"VM returned invalid status: {repr(status_buf)[11:-1]}")
return int(status_buf)
