def _build_kmod(self, kernel_dir, kmod): kernel_build_dir = kernel_dir / "build" # External modules can't do out-of-tree builds for some reason, so copy # the source files to a temporary directory and build the module there, # then move it to the final location. kmod_source_dir = Path("tests/linux_kernel/kmod") source_files = ("drgn_test.c", "Makefile") if out_of_date( kmod, *[kmod_source_dir / filename for filename in source_files]): with tempfile.TemporaryDirectory(dir=kmod.parent) as tmp_name: tmp_dir = Path(tmp_name) # Make sure that header files have the same paths as in the # original kernel build. debug_prefix_map = [ f"{kernel_build_dir.resolve()}=.", f"{tmp_dir.resolve()}=./drgn_test", ] cflags = " ".join( ["-fdebug-prefix-map=" + map for map in debug_prefix_map]) for filename in source_files: copy_file(kmod_source_dir / filename, tmp_dir / filename) if (subprocess.call([ "make", "-C", kernel_build_dir, f"M={tmp_dir.resolve()}", "KAFLAGS=" + cflags, "KCFLAGS=" + cflags, "-j", str(nproc()), ]) != 0): return False (tmp_dir / "drgn_test.ko").rename(kmod) return True
async def build_kernel(commit: str, build_dir: str, log_file: TextIO) -> Tuple[str, str]: """ Returns built kernel release (i.e., `uname -r`) and image name (e.g., `arch/x86/boot/bzImage`). """ await check_call("git", "checkout", commit, stdout=log_file, stderr=asyncio.subprocess.STDOUT) shutil.copy( os.path.join(os.path.dirname(__file__), "config"), os.path.join(build_dir, ".config"), ) logger.info("building %s", commit) start = time.monotonic() cflags = f"-fdebug-prefix-map={os.path.join(getpwd(), build_dir)}=" kbuild_args = [ "KBUILD_BUILD_USER=drgn", "KBUILD_BUILD_HOST=drgn", "KAFLAGS=" + cflags, "KCFLAGS=" + cflags, "O=" + build_dir, "-j", str(nproc()), ] await check_call( "make", *kbuild_args, "olddefconfig", "all", stdout=log_file, stderr=asyncio.subprocess.STDOUT, ) elapsed = time.monotonic() - start logger.info("built %s in %s", commit, humanize_duration(elapsed)) vmlinux = os.path.join(build_dir, "vmlinux") release, image_name = (await asyncio.gather( post_process_vmlinux(vmlinux, stdout=log_file, stderr=asyncio.subprocess.STDOUT), check_output("make", *kbuild_args, "-s", "kernelrelease", stderr=log_file), check_output("make", *kbuild_args, "-s", "image_name", stderr=log_file), ))[1:] return release.decode().strip(), image_name.decode().strip()
async def _prepare_make(self) -> Tuple[str, ...]: if self._cached_make_args is None: self._build_dir.mkdir(parents=True, exist_ok=True) debug_prefix_map = [] # GCC uses the "logical" working directory, i.e., the PWD # environment variable, when it can. See # https://gcc.gnu.org/git/?p=gcc.git;a=blob;f=libiberty/getpwd.c;hb=HEAD. # Map both the canonical and logical paths. build_dir_real = self._build_dir.resolve() debug_prefix_map.append(str(build_dir_real) + "=.") build_dir_logical = (await check_output_shell( f"cd {shlex.quote(str(self._build_dir))}; pwd -L", ) ).decode()[:-1] if build_dir_logical != str(build_dir_real): debug_prefix_map.append(build_dir_logical + "=.") # Before Linux kernel commit 25b146c5b8ce ("kbuild: allow Kbuild to # start from any directory") (in v5.2), O= forces the source # directory to be absolute. Since Linux kernel commit 95fd3f87bfbe # ("kbuild: add a flag to force absolute path for srctree") (in # v5.3), KBUILD_ABS_SRCTREE=1 does the same. This means that except # for v5.2, which we don't support, the source directory will # always be absolute, and we don't need to worry about mapping it # from a relative path. kernel_dir_real = self._kernel_dir.resolve() if kernel_dir_real != build_dir_real: debug_prefix_map.append(str(kernel_dir_real) + "/=./") cflags = " ".join( ["-fdebug-prefix-map=" + map for map in debug_prefix_map]) self._cached_make_args = ( "-C", str(self._kernel_dir), "ARCH=" + str(self._arch), "O=" + str(build_dir_real), "KBUILD_ABS_SRCTREE=1", "KBUILD_BUILD_USER=drgn", "KBUILD_BUILD_HOST=drgn", "HOSTCFLAGS=" + cflags, "KAFLAGS=" + cflags, "KCFLAGS=" + cflags, "-j", str(nproc()), ) return self._cached_make_args
def finalize_options(self): super().finalize_options() if self.parallel is None: self.parallel = nproc() + 1
def run_in_vm(command: str, kernel_dir: Path, build_dir: Path) -> int: match = re.search( "QEMU emulator version ([0-9]+(?:\.[0-9]+)*)", subprocess.check_output(["qemu-system-x86_64", "-version"], universal_newlines=True), ) if not match: raise Exception("could not determine QEMU version") qemu_version = tuple(int(x) for x in match.group(1).split(".")) # multidevs was added in QEMU 4.2.0. multidevs = ",multidevs=remap" if qemu_version >= (4, 2) else "" # QEMU's 9pfs O_NOATIME handling was fixed in 5.1.0. The fix was backported # to 5.0.1. env = os.environ.copy() if qemu_version < (5, 0, 1): onoatimehack_so = _build_onoatimehack(build_dir) env["LD_PRELOAD"] = f"{str(onoatimehack_so)}:{env.get('LD_PRELOAD', '')}" with tempfile.TemporaryDirectory( prefix="drgn-vmtest-") as temp_dir, socket.socket( socket.AF_UNIX) as server_sock: temp_path = Path(temp_dir) socket_path = temp_path / "socket" server_sock.bind(str(socket_path)) server_sock.listen() busybox = shutil.which("busybox") if busybox is None: raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), "busybox") init = (temp_path / "init").resolve() with open(init, "w") as init_file: init_file.write( _INIT_TEMPLATE.format(busybox=shlex.quote(busybox), command=shlex.quote(command))) os.chmod(init, 0o755) with subprocess.Popen( [ # fmt: off "qemu-system-x86_64", "-cpu", "host", "-enable-kvm", "-smp", str(nproc()), "-m", "2G", "-nodefaults", "-display", "none", "-serial", "mon:stdio", # This along with -append panic=-1 ensures that we exit on a # panic instead of hanging. "-no-reboot", "-virtfs", f"local,id=root,path=/,mount_tag=/dev/root,security_model=none,readonly{multidevs}", "-virtfs", f"local,path={kernel_dir},mount_tag=modules,security_model=none,readonly", "-device", "virtio-serial", "-chardev", f"socket,id=vmtest,path={socket_path}", "-device", "virtserialport,chardev=vmtest,name=com.osandov.vmtest.0", "-kernel", str(kernel_dir / "vmlinuz"), "-append", f"rootfstype=9p rootflags=trans=virtio,cache=loose ro console=0,115200 panic=-1 init={init}", # fmt: on ], env=env, ) as qemu: server_sock.settimeout(5) try: sock = server_sock.accept()[0] except socket.timeout: raise LostVMError( f"QEMU did not connect within {server_sock.gettimeout()} seconds" ) try: status_buf = bytearray() while True: try: buf = sock.recv(4) except ConnectionResetError: buf = b"" if not buf: break status_buf.extend(buf) finally: sock.close() if not status_buf: raise LostVMError("VM did not return status") if status_buf[-1] != ord("\n") or not status_buf[:-1].isdigit(): raise LostVMError( f"VM returned invalid status: {repr(status_buf)[11:-1]}") return int(status_buf)
async def build_kernel( commit: str, build_dir: Path, log_file: TextIO ) -> Tuple[str, Path]: """ Returns built kernel release (i.e., `uname -r`) and image name (e.g., `arch/x86/boot/bzImage`). """ await check_call( "git", "checkout", commit, stdout=log_file, stderr=asyncio.subprocess.STDOUT ) shutil.copy(KERNEL_CONFIG_PATH, build_dir / ".config") logger.info("building %s", commit) start = time.monotonic() cflags = f"-fdebug-prefix-map={getpwd() / build_dir}=" kbuild_args = [ "KBUILD_BUILD_USER=drgn", "KBUILD_BUILD_HOST=drgn", "KAFLAGS=" + cflags, "KCFLAGS=" + cflags, "O=" + str(build_dir), "-j", str(nproc()), ] await check_call( "make", *kbuild_args, "olddefconfig", "all", stdout=log_file, stderr=asyncio.subprocess.STDOUT, ) elapsed = time.monotonic() - start logger.info("built %s in %s", commit, humanize_duration(elapsed)) logger.info("packaging %s", commit) start = time.monotonic() release = ( ( await check_output( "make", *kbuild_args, "-s", "kernelrelease", stderr=log_file ) ) .decode() .strip() ) image_name = ( (await check_output("make", *kbuild_args, "-s", "image_name", stderr=log_file)) .decode() .strip() ) install_dir = build_dir / "install" modules_dir = install_dir / "lib" / "modules" / release await check_call( "make", *kbuild_args, "INSTALL_MOD_PATH=install", "modules_install", stdout=log_file, stderr=asyncio.subprocess.STDOUT, ) # Don't want these symlinks. (modules_dir / "build").unlink() (modules_dir / "source").unlink() vmlinux = modules_dir / "vmlinux" await check_call( "objcopy", "--remove-relocations=*", str(build_dir / "vmlinux"), str(vmlinux), stdout=log_file, stderr=asyncio.subprocess.STDOUT, ) vmlinux.chmod(0o644) vmlinuz = modules_dir / "vmlinuz" shutil.copy(build_dir / image_name, vmlinuz) vmlinuz.chmod(0o644) tarball = build_dir / "kernel.tar.zst" tar_command = ("tar", "-C", str(modules_dir), "-c", ".") zstd_command = ("zstd", "-T0", "-19", "-q", "-", "-o", str(tarball)) pipe_r, pipe_w = os.pipe() try: tar_proc, zstd_proc = await asyncio.gather( asyncio.create_subprocess_exec( *tar_command, stdout=pipe_w, stderr=log_file ), asyncio.create_subprocess_exec( *zstd_command, stdin=pipe_r, stdout=log_file, stderr=asyncio.subprocess.STDOUT, ), ) finally: os.close(pipe_r) os.close(pipe_w) tar_returncode, zstd_returncode = await asyncio.gather( tar_proc.wait(), zstd_proc.wait() ) if tar_returncode != 0: raise CalledProcessError(tar_returncode, tar_command) if zstd_returncode != 0: raise CalledProcessError(zstd_returncode, zstd_command) shutil.rmtree(install_dir) elapsed = time.monotonic() - start logger.info("packaged %s in %s", commit, humanize_duration(elapsed)) return release, tarball
def __init__( self, *, init: str, onoatimehack: str, vmlinux: str, vmlinuz: str, ) -> None: self._temp_dir = tempfile.TemporaryDirectory("drgn-vmtest-") self._server_sock = socket.socket(socket.AF_UNIX) socket_path = os.path.join(self._temp_dir.name, "socket") self._server_sock.bind(socket_path) self._server_sock.listen() init = os.path.abspath(init) if " " in init: init = '"' + init + '"' vmlinux = os.path.abspath(vmlinux) if " " in vmlinux: vmlinux = '"' + vmlinux + '"' # This was added in QEMU 4.2.0. if ("multidevs" in subprocess.run( ["qemu-system-x86_64", "-help"], stdout=subprocess.PIPE, universal_newlines=True, ).stdout): multidevs = ",multidevs=remap" else: multidevs = "" self._qemu = subprocess.Popen( [ # fmt: off "qemu-system-x86_64", "-cpu", "kvm64", "-enable-kvm", "-smp", str(nproc()), "-m", "2G", "-nodefaults", "-display", "none", "-serial", "mon:stdio", # This along with -append panic=-1 ensures that we exit on a # panic instead of hanging. "-no-reboot", "-virtfs", f"local,id=root,path=/,mount_tag=/dev/root,security_model=none,readonly{multidevs}", "-device", "virtio-serial", "-chardev", f"socket,id=vmtest,path={socket_path}", "-device", "virtserialport,chardev=vmtest,name=com.osandov.vmtest.0", "-kernel", vmlinuz, "-append", f"rootfstype=9p rootflags=trans=virtio,cache=loose ro console=0,115200 panic=-1 init={init} VMLINUX={vmlinux}", # fmt: on ], env={ **os.environ, "LD_PRELOAD": f"{onoatimehack}:{os.getenv('LD_PRELOAD', '')}", }, ) self._server_sock.settimeout(5) try: self._sock = self._server_sock.accept()[0] except socket.timeout: raise LostVMError( f"QEMU did not connect within {self._server_sock.gettimeout()} seconds" )
def run_in_vm(command: str, *, vmlinuz: str, build_dir: str) -> int: # multidevs was added in QEMU 4.2.0. if ("multidevs" in subprocess.run( ["qemu-system-x86_64", "-help"], stdout=subprocess.PIPE, universal_newlines=True, ).stdout): multidevs = ",multidevs=remap" else: multidevs = "" onoatimehack = _build_onoatimehack(build_dir) with tempfile.TemporaryDirectory( prefix="drgn-vmtest-") as temp_dir, socket.socket( socket.AF_UNIX) as server_sock: socket_path = os.path.join(temp_dir, "socket") server_sock.bind(socket_path) server_sock.listen() busybox = shutil.which("busybox") if busybox is None: raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), "busybox") init = os.path.abspath(os.path.join(temp_dir, "init")) with open(init, "w") as init_file: init_file.write( _INIT_TEMPLATE.format(busybox=shlex.quote(busybox), command=shlex.quote(command))) os.chmod(init, 0o755) with subprocess.Popen( [ # fmt: off "qemu-system-x86_64", "-cpu", "host", "-enable-kvm", "-smp", str(nproc()), "-m", "2G", "-nodefaults", "-display", "none", "-serial", "mon:stdio", # This along with -append panic=-1 ensures that we exit on a # panic instead of hanging. "-no-reboot", "-virtfs", f"local,id=root,path=/,mount_tag=/dev/root,security_model=none,readonly{multidevs}", "-device", "virtio-serial", "-chardev", f"socket,id=vmtest,path={socket_path}", "-device", "virtserialport,chardev=vmtest,name=com.osandov.vmtest.0", "-kernel", vmlinuz, "-append", f"rootfstype=9p rootflags=trans=virtio,cache=loose ro console=0,115200 panic=-1 init={init}", # fmt: on ], env={ **os.environ, "LD_PRELOAD": f"{onoatimehack}:{os.getenv('LD_PRELOAD', '')}", }, ) as qemu: server_sock.settimeout(5) try: sock = server_sock.accept()[0] except socket.timeout: raise LostVMError( f"QEMU did not connect within {server_sock.gettimeout()} seconds" ) try: status_buf = bytearray() while True: try: buf = sock.recv(4) except ConnectionResetError: buf = b"" if not buf: break status_buf.extend(buf) finally: sock.close() if not status_buf: raise LostVMError("VM did not return status") if status_buf[-1] != ord("\n") or not status_buf[:-1].isdigit(): raise LostVMError( f"VM returned invalid status: {repr(status_buf)[11:-1]}") return int(status_buf)