def copy_efi_partition(*, image_file: str, efi_device, tempdir: str, kernel_only: bool = True): verbose('Copying EFI configuration out of image file.') with disk.NbdDevice(image_file, disk_format='raw') \ as internal_device: internal_device.wait_for_device_node(partition=1) with mount.Mount(internal_device.device(1), os.path.join(tempdir, '_efi')) \ as int_efi: with mount.Mount(efi_device, os.path.join(tempdir, 'efi'), fs_type='vfat') \ as efi: if kernel_only: img_dir = os.path.join(int_efi, 'EFI', 'Linux') efi_dir = os.path.join(efi, 'EFI', 'Linux') assert os.path.isdir(img_dir) if not os.path.isdir(efi_dir): os.makedirs(efi_dir) for f in [f for f in os.listdir(img_dir) if os.path.isfile(os.path.join(img_dir, f))]: trace('Copying EFI kernel {}.'.format(f)) copyfile(os.path.join(img_dir, f), os.path.join(efi_dir, f)) else: trace('Copying EFI folder into system.') copy_tree(int_efi, efi)
def execute_with_system_mounted(to_execute: typing.Callable[[str, str], int], *, image_file: str, tmp_dir: str) -> int: assert os.path.isfile(image_file) with disk.NbdDevice(image_file, disk_format="raw", read_only=True) as device: verbose("Mounting EFI...") device.wait_for_device_node(partition=1) with mount.Mount( device.device(1), os.path.join(tmp_dir, "EFI"), fs_type="vfat", options="ro", ) as efi: verbose("Mounting root filesystem...") with mount.Mount( device.device(2), os.path.join(tmp_dir, "root"), fs_type="squashfs", options="ro", ) as root: trace(f'Executing with EFI "{efi}" and root "{root}".') result = to_execute(efi, root) return result
def __call__(self, location: Location, system_context: SystemContext, *args: typing.Any, **kwargs: typing.Any) -> None: """Execute command.""" self._setup(*args, **kwargs) h2('Exporting system "{}".'.format(system_context.system_name)) debug('Running Hooks.') self._run_all_exportcommand_hooks(system_context) verbose('Preparing system for export.') self.prepare_for_export(location, system_context) info('Validating installation for export.') if not self._skip_validation: _validate_installation(location.next_line(), system_context) export_directory \ = self.create_export_directory(system_context) assert export_directory system_context.set_substitution('EXPORT_DIRECTORY', export_directory) verbose('Exporting all data in {}.'.format(export_directory)) self._execute(location.next_line(), system_context, '_export_directory', export_directory, compression=self._repository_compression, compression_level=self._repository_compression_level, repository=self._repository) info('Cleaning up export location.') self.delete_export_directory(export_directory)
def _populate_with_efi_emulator(staging_area: str, efi_emulator: str): verbose("Installing Clover binaries into EFI partition") assert efi_emulator, "Missing EFI emulator path" shutil.copy( os.path.join(efi_emulator, "Bootloaders/x64/boot7"), os.path.join(staging_area, "boot"), follow_symlinks=False, ) efi_dir = os.path.join(staging_area, "EFI") if not os.path.isdir(efi_dir): os.makedirs(efi_dir) shutil.copytree(os.path.join(efi_emulator, "EFI/CLOVER"), os.path.join(efi_dir, "CLOVER")) config_file = os.path.join(efi_dir, "CLOVER/config.plist") with open(config_file, "wb") as config: config.write("""\ <?xml version="1.0" encoding="UTF-8"?> <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> <plist version="1.0"> <dict> <key>Boot</key> <dict> <key>DefaultVolume</key> <string>EFI</string> <key>DefaultLoader</key> <string>\\EFI\\systemd\\systemd-bootx64.efi</string> <key>Fast</key> <true/> </dict> <key>GUI</key> <dict> <key>Custom</key> <dict> <key>Entries</key> <array> <dict> <key>Hidden</key> <false/> <key>Disabled</key> <false/> <key>Image</key> <string>os_arch</string> <key>Volume</key> <string>EFI</string> <key>Path</key> <string>\\EFI\\systemd\\systemd-bootx64.efi</string> <key>Title</key> <string>Cleanroom Linux</string> <key>Type</key> <string>Linux</string> </dict> </array> </dict> </dict> </dict> </plist> """.encode("utf-8"))
def __call__( self, location: Location, system_context: SystemContext, *args: typing.Any, **kwargs: typing.Any, ) -> None: """Execute command.""" ## validate package type: if system_context.substitution("CLRM_PACKAGE_TYPE", ""): raise GenerateError( "Trying to run swupd_init on a system that already has a CLRM_PACKAGE_TYPE defined." ) system_context.set_substitution("CLRM_PACKAGE_TYPE", "swupd") run( self._binary(Binaries.SWUPD), "autoupdate", f"--path={system_context.fs_directory}", "--disable", "--no-progress", returncode=28, ) # Setup update-helper so that swupd os-install will actually work: os.makedirs(system_context.file_name("/usr/bin")) with open(system_context.file_name("/usr/bin/update-helper"), "wb") as fd: fd.write( dedent( """\ #!/usr/bin/sh exit 0 """ ).encode("utf-8") ) os.chmod(system_context.file_name("/usr/bin/update-helper"), 0o755) run( self._binary(Binaries.SWUPD), "os-install", f"--path={system_context.fs_directory}", "--skip-optional", "--no-progress", ) location.set_description("Move systemd files into /usr") self._add_hook(location, system_context, "_teardown", "systemd_cleanup") with open(system_context.file_name("/usr/lib/os-release"), "r") as osr: for l in osr: l = l.strip() if l.startswith("BUILD_ID="): build_id = l[9:] verbose(f"Installed {build_id}.") system_context.set_substitution("DISTRO_VERSION_ID", build_id) system_context.set_substitution("DISTRO_VERSION", build_id) self._execute(location.next_line(), system_context, "create_os_release")
def execute_with_system_mounted(to_execute: typing.Callable[[str, str], None], *, repository: str, system_name: str, system_version: str = "") -> None: with TemporaryDirectory(prefix="clrm_qemu_") as tempdir: verbose("Extracting image") image_path = export_into_directory(system_name, tempdir, repository=repository, version=system_version) assert os.path.isfile(image_path) with disk.NbdDevice(image_path, disk_format="raw") as device: verbose("Mounting EFI...") device.wait_for_device_node(partition=1) with mount.Mount( device.device(1), os.path.join(tempdir, "EFI"), fs_type="vfat", options="ro", ) as efi: verbose("Mounting root filesystem...") with mount.Mount( device.device(2), os.path.join(tempdir, "root"), fs_type="squashfs", options="ro", ) as root: verbose('Executing with EFI "{}" and root "{}".'.format( efi, root)) to_execute(efi, root)
def execute_with_system_mounted(to_execute: typing.Callable[[str, str], None], *, repository: str, system_name: str, system_version: str = '') -> None: with TemporaryDirectory(prefix='clrm_qemu_') as tempdir: verbose('Extracting image') image_path \ = export_into_directory(system_name, tempdir, repository=repository, version=system_version) assert os.path.isfile(image_path) with disk.NbdDevice(image_path, disk_format='raw') as device: verbose('Mounting EFI...') device.wait_for_device_node(partition=1) with mount.Mount(device.device(1), os.path.join(tempdir, 'EFI'), fs_type='vfat', options='ro') as efi: verbose('Mounting root filesystem...') with mount.Mount(device.device(2), os.path.join(tempdir, 'root'), fs_type='squashfs', options='ro') as root: verbose('Executing with EFI "{}" and root "{}".' .format(efi, root)) to_execute(efi, root)
def _execution(efi: str, rootfs: str, *, command: str) -> int: to_exec = command or '/usr/bin/bash -c "read -n1 -s"' prompt = "" if command else "<<< Press any key to continue >>>" env = os.environ env["EFI_MOUNT"] = efi env["ROOT_MOUNT"] = rootfs verbose(f"Running {command}.") print(f'EFI partition is mounted at "{efi}".') print(f'Root partition is mounted at "{rootfs}".') if prompt: print(prompt) return tool.run(*split(to_exec), env=env).returncode
def _create_hdd_image(device): verbose('hdd.img created.') partitioner = disk.Partitioner(device) partitioner.repartition([ disk.Partitioner.efi_partition(size='512M'), disk.Partitioner.swap_partition(size='1G', name='swap'), disk.Partitioner.data_partition(name='data')]) verbose('hdd.img repartitioned.') debug('Format EFI partitition.') run('/usr/bin/mkfs.vfat', device.device(1)) debug('Set up swap partitition.') run('/usr/bin/mkswap', device.device(2)) debug('Format data partitition.') run('/usr/bin/mkfs.btrfs', '-L', 'fs_btrfs', device.device(3))
def __call__(self, location: Location, system_context: SystemContext, *args: typing.Any, **kwargs: typing.Any) -> None: """Execute command.""" base_system = args[0] if base_system == 'scratch': assert system_context.base_context is None verbose('Building from scratch!') self._add_hook(location, system_context, 'testing', '_test') self._execute(location, system_context, '_setup') else: assert system_context.base_context.system_name == base_system verbose('Building on top of {}.'.format(base_system)) self._execute(location, system_context, '_restore', base_system) self._run_hooks(system_context, "_setup")
def create_qemu_image(image_path: str, *, image_size: str, image_format: str = "qcow2", system_name: str, system_version: str = "", repository: str, tempdir: str) -> str: trace("Creating image file {}.".format(image_path)) with disk.NbdDevice.new_image_file(image_path, image_size, disk_format=image_format) as device: _create_hdd_image(device) debug("mounting data partition for further setup.") with mount.Mount( device.device(3), os.path.join(tempdir, "data"), fs_type="btrfs", options="subvolid=0", fallback_cwd=os.getcwd(), ) as data_dir: _setup_btrfs(data_dir) extract_location = os.path.join(data_dir, ".images") verbose("Extracting system image to {}.".format(extract_location)) extracted_version = tool.write_image( system_name, extract_location, repository=repository, version=system_version, ) extracted_image = os.path.join(data_dir, ".images", "clrm_{}".format(extracted_version)) assert os.path.isfile(extracted_image) tool.copy_efi_partition( image_file=extracted_image, efi_device=device.device(1), tempdir=tempdir, kernel_only=False, ) return image_path
def _create_hdd_image(device): verbose("hdd.img created.") partitioner = disk.Partitioner(device) partitioner.repartition([ disk.Partitioner.efi_partition(size="512M"), disk.Partitioner.swap_partition(size="1G", name="swap"), disk.Partitioner.data_partition(name="data"), ]) verbose("hdd.img repartitioned.") debug("Format EFI partitition.") run("/usr/bin/mkfs.vfat", device.device(1)) debug("Set up swap partitition.") run("/usr/bin/mkswap", device.device(2)) debug("Format data partitition.") run("/usr/bin/mkfs.btrfs", "-L", "fs_btrfs", device.device(3))
def __call__( self, location: Location, system_context: SystemContext, *args: typing.Any, **kwargs: typing.Any, ) -> None: """Execute command.""" base_system = args[0] if base_system == "scratch": assert system_context.base_context is None verbose("Building from scratch!") self._add_hook(location, system_context, "testing", "_test") self._execute(location, system_context, "_setup") else: assert (system_context.base_context and system_context.base_context.system_name == base_system) verbose(f"Building on top of {base_system}.") self._execute(location, system_context, "_restore", base_system) self._run_hooks(system_context, "_setup")
def create_qemu_image(image_path: str, *, image_size: str, image_format: str = 'qcow2', system_name: str, system_version: str = '', repository: str, tempdir: str) -> str: trace('Creating image file {}.'.format(image_path)) with disk.NbdDevice.new_image_file(image_path, image_size, disk_format=image_format) as device: _create_hdd_image(device) debug('mounting data partition for further setup.') with mount.Mount(device.device(3), os.path.join(tempdir, 'data'), fs_type='btrfs', options='subvolid=0', fallback_cwd=os.getcwd()) as data_dir: _setup_btrfs(data_dir) extract_location = os.path.join(data_dir, '.images') verbose('Extracting system image to {}.' .format(extract_location)) extracted_version \ = tool.write_image(system_name, extract_location, repository=repository, version=system_version) extracted_image \ = os.path.join(data_dir, '.images', 'clrm_{}'.format(extracted_version)) assert os.path.isfile(extracted_image) tool.copy_efi_partition(image_file=extracted_image, efi_device=device.device(1), tempdir=tempdir, kernel_only=False) return image_path
def _execution(efi: str, rootfs: str, *, command: str) -> None: to_exec = command or '/usr/bin/bash -c "read -n1 -s"' prompt = "" if command else "<<< Press any key to continue >>>" env = os.environ env["EFI_MOUNT"] = efi env["ROOT_MOUNT"] = rootfs verbose("Running {}.".format(command)) verbose('EFI_MOUNT env var set to : "{}".'.format(efi)) verbose('ROOT_MOUNT env var set to: "{}".'.format(rootfs)) print('EFI partition is mounted at "{}".'.format(efi)) print('Root partition is mounted at "{}".'.format(rootfs)) if prompt: print(prompt) tool.run(*split(to_exec), env=env)
def __call__( self, location: Location, system_context: SystemContext, *args: typing.Any, **kwargs: typing.Any, ) -> None: """Execute command.""" cert = kwargs.get("efi_cert", "") efi_emulator = kwargs.get("efi_emulator", "") key = kwargs.get("efi_key", "") skip_validation = kwargs.get("skip_validation", False) repository = args[0] repository_compression = kwargs.get("repository_compression", "zstd") repository_compression_level = kwargs.get( "repository_compression_level", 5) usr_only = kwargs.get("usr_only", True) debug_initrd = kwargs.get("debug_initrd", False) h2(f'Exporting system "{system_context.system_name}".') debug("Running Hooks.") self._run_all_exportcommand_hooks(system_context) verbose("Preparing system for export.") self._execute(location.next_line(), system_context, "_write_deploy_info") # Create some extra data: self._create_root_tarball(location, system_context) root_partition = self._create_root_fsimage(location, system_context, usr_only=usr_only) assert root_partition (verity_partition, root_hash) = self._create_rootverity_fsimage( location, system_context, rootfs=root_partition, ) assert root_hash has_kernel = os.path.exists( os.path.join(system_context.boot_directory, "vmlinuz")) if has_kernel: self._create_initrd(location, system_context) self._create_clrm_config_initrd(location, system_context, root_hash, debug=debug_initrd) cmdline = system_context.set_or_append_substitution( "KERNEL_CMDLINE", "systemd.volatile=true rootfstype=squashfs") cmdline = _setup_kernel_commandline(cmdline, root_hash) kernel_file = "" if has_kernel: trace( f'KERNEL_FILENAME: {system_context.substitution("KERNEL_FILENAME", "")}' ) kernel_file = os.path.join( system_context.boot_directory, system_context.substitution_expanded("KERNEL_FILENAME", ""), ) assert kernel_file self._create_complete_kernel( location, system_context, cmdline, kernel_file=kernel_file, efi_key=key, efi_cert=cert, ) efi_partition = os.path.join(system_context.cache_directory, "efi_partition.img") self._create_efi_partition( location, system_context, efi_partition=efi_partition, kernel_file=kernel_file, efi_emulator=efi_emulator, root_hash=root_hash, ) info("Validating installation for export.") if not skip_validation: _validate_installation(location.next_line(), system_context) export_directory = self.create_export_directory(system_context) assert export_directory self.create_image( location, system_context, export_directory, efi_partition=efi_partition, root_partition=root_partition, verity_partition=verity_partition, root_hash=root_hash, ) system_context.set_substitution("EXPORT_DIRECTORY", export_directory) verbose(f"Exporting all data in {export_directory}.") self._execute( location.next_line(), system_context, "_export_directory", export_directory, compression=repository_compression, compression_level=repository_compression_level, repository=repository, ) info("Cleaning up export location.") self.delete_export_directory(export_directory) system_context.set_substitution("EXPORT_DIRECTORY", "")
def __call__( self, location: Location, system_context: SystemContext, *args: typing.Any, **kwargs: typing.Any, ) -> None: """Execute command.""" ## validate package type: if system_context.substitution("CLRM_PACKAGE_TYPE", ""): raise GenerateError( "Trying to run swupd_init on a system that already has a CLRM_PACKAGE_TYPE defined." ) system_context.set_substitution("CLRM_PACKAGE_TYPE", "swupd") system_context.set_substitution("DISTRO_PRETTY_NAME", "Cleanroom - CLR") run( self._binary(Binaries.SWUPD), "autoupdate", f"--path={system_context.fs_directory}", "--disable", "--no-progress", returncode=28, ) # Setup update-helper so that swupd os-install will actually work: os.makedirs(system_context.file_name("/usr/bin")) with open(system_context.file_name("/usr/bin/update-helper"), "wb") as fd: fd.write( dedent("""\ #!/usr/bin/sh exit 0 """).encode("utf-8")) os.chmod(system_context.file_name("/usr/bin/update-helper"), 0o755) run( self._binary(Binaries.SWUPD), "os-install", f"--path={system_context.fs_directory}", "--skip-optional", "--no-progress", ) system_context.set_substitution("INITRD_GENERATOR", "clr") location.set_description("Move systemd files into /usr") self._add_hook(location, system_context, "_teardown", "systemd_cleanup") location.set_description("Setup Clearlinux triggers as hooks") self._add_hook( location, system_context, "export", "run", "/usr/bin/systemd-tmpfiles", "--create", inside=True, ) self._add_hook( location, system_context, "export", "_clr_catalog_trigger", ) self._add_hook( location, system_context, "export", "_clr_fontconfig_trigger", ) self._add_hook( location, system_context, "export", "_clr_glib_schemas_trigger", ) self._add_hook( location, system_context, "export", "_clr_graphviz_dot_trigger", ) self._add_hook( location, system_context, "export", "_clr_hwdb_update_trigger", ) self._add_hook( location, system_context, "export", "_clr_icon_cache_update_trigger", ) self._add_hook( location, system_context, "export", "_clr_ldconfig_trigger", ) self._add_hook( location, system_context, "export", "_clr_locale_archive_trigger", ) self._add_hook( location, system_context, "export", "_clr_mandb_trigger", ) self._add_hook( location, system_context, "export", "_clr_sysusers_trigger", ) self._add_hook( location, system_context, "export", "_clr_dynamic_trust_store_trigger", ) self._add_hook( location, system_context, "export", "_clr_mime_update_trigger", ) with open(system_context.file_name("/usr/lib/os-release"), "r") as osr: for l in osr: l = l.strip() if l.startswith("BUILD_ID="): build_id = l[9:] verbose(f"Installed {build_id}.") system_context.set_substitution("DISTRO_VERSION_ID", build_id) system_context.set_substitution("DISTRO_VERSION", build_id) system_context.set_substitution("DISTRO_ID_LIKE", "clearlinux") system_context.set_substitution( "ROOTFS_PARTLABEL", "root_${TIMESTAMP}-${DISTRO_VERSION_ID}") system_context.set_substitution( "VRTYFS_PARTLABEL", "vrty_${TIMESTAMP}-${DISTRO_VERSION_ID}") system_context.set_substitution( "KERNEL_FILENAME", "${PRETTY_SYSTEM_NAME}_${TIMESTAMP}-${DISTRO_VERSION_ID}.efi", ) system_context.set_substitution( "CLRM_IMAGE_FILENAME", "${PRETTY_SYSTEM_NAME}_${TIMESTAMP}-${DISTRO_VERSION_ID}", ) self._execute(location.next_line(), system_context, "create_os_release")