def __call__(self, location: Location, system_context: SystemContext, *args: typing.Any, **kwargs: typing.Any) -> None: """Execute command.""" self._setup(*args, **kwargs) h2('Exporting system "{}".'.format(system_context.system_name)) debug('Running Hooks.') self._run_all_exportcommand_hooks(system_context) verbose('Preparing system for export.') self.prepare_for_export(location, system_context) info('Validating installation for export.') if not self._skip_validation: _validate_installation(location.next_line(), system_context) export_directory \ = self.create_export_directory(system_context) assert export_directory system_context.set_substitution('EXPORT_DIRECTORY', export_directory) verbose('Exporting all data in {}.'.format(export_directory)) self._execute(location.next_line(), system_context, '_export_directory', export_directory, compression=self._repository_compression, compression_level=self._repository_compression_level, repository=self._repository) info('Cleaning up export location.') self.delete_export_directory(export_directory)
def __call__( self, location: Location, system_context: SystemContext, *args: typing.Any, **kwargs: typing.Any, ) -> None: """Execute command.""" if not os.path.exists( os.path.join(system_context.boot_directory, "vmlinuz")): info("Skipping initrd generation: No vmlinuz in boot directory.") return initrd = args[0] copy( system_context, os.path.join(system_context.boot_directory, "vmlinuz"), "/boot/vmlinuz", from_outside=True, ) # Use pre-created initrs if possible! pre_created = os.path.join(system_context.initrd_parts_directory, "50-clr") assert os.path.exists(pre_created) if pre_created != initrd: shutil.copyfile(pre_created, initrd) assert os.path.isfile(initrd)
def __call__(self, location: Location, system_context: SystemContext, *args: typing.Any, **kwargs: typing.Any) -> None: """Execute command.""" to_sign = args[0] keep_unsigned = kwargs.get("keep_unsigned", False) if not kwargs.get("outside", False): to_sign = system_context.file_name(to_sign) systems_directory = system_context.systems_definition_directory key = os.path.join(systems_directory, kwargs.get("key", "config/efi/sign.key")) cert = os.path.join(systems_directory, kwargs.get("cert", "config/efi/sign.crt")) info("Signing EFI binary {} using key {} and cert {}.".format( input, key, cert)) output = to_sign + ".signed" assert os.path.isfile(key) assert os.path.isfile(cert) assert os.path.isfile(to_sign) assert not os.path.exists(output) run( self._binary(Binaries.SBSIGN), "--key", key, "--cert", cert, "--output", output, to_sign, ) if not keep_unsigned: os.remove(to_sign) shutil.move(output, to_sign)
def __call__(self, location: Location, system_context: SystemContext, *args: typing.Any, **kwargs: typing.Any) -> None: """Execute command.""" if not os.path.exists(system_context.file_name("usr/bin/mkinitcpio")): info("Skipping initrd generation: No mkinitcpio binary.") return if not os.path.exists( os.path.join(system_context.boot_directory, "vmlinuz")): info("Skipping initrd generation: No vmlinuz in boot directory.") return self._vg = system_context.substitution("DEFAULT_VG", None) if not self._vg: self._vg = None self._image_fs = system_context.substitution("IMAGE_FS", "ext2") self._image_device = _deviceify( system_context.substitution("IMAGE_DEVICE", "")) self._image_options = system_context.substitution( "IMAGE_OPTIONS", "rw") name_prefix = system_context.substitution("DISTRO_ID", "clrm") name_version = system_context.substitution("DISTRO_VERSION_ID", system_context.timestamp) self._full_name = "{}_{}".format(name_prefix, name_version) initrd = args[0] to_clean_up = [] # type: typing.List[str] to_clean_up += "/boot/vmlinuz" to_clean_up += self._install_extra_binaries(location, system_context) to_clean_up += self._create_systemd_units(location, system_context) to_clean_up += self._install_mkinitcpio(location, system_context) to_clean_up += self._install_mkinitcpio_hooks(location, system_context) copy( system_context, os.path.join(system_context.boot_directory, "vmlinuz"), "/boot/vmlinuz", from_outside=True, ) run( "/usr/bin/mkinitcpio", "-p", "cleanroom", chroot=system_context.fs_directory, chroot_helper=self._binary(Binaries.CHROOT_HELPER), ) initrd_directory = os.path.dirname(initrd) os.makedirs(initrd_directory, exist_ok=True) move(system_context, "/boot/initramfs.img", initrd, to_outside=True) _cleanup_extra_files(location, system_context, *to_clean_up) self._remove_mkinitcpio(location, system_context) assert os.path.isfile(initrd)
def __call__( self, location: Location, system_context: SystemContext, *args: typing.Any, **kwargs: typing.Any, ) -> None: """Execute command.""" user = args[0] keyfile = args[1] info(f"Adding ssh key to {user}'s authorized_keys file.") data = UserHelper.user_data(user, root_directory=system_context.fs_directory) if data is None: raise GenerateError( f'"{self.name}" could not find user "{user}".', location=location, ) trace(f"{user} mapping: UID {data.uid}, GID {data.gid}, home: {data.home}.") self._check_or_create_directory( location, system_context, data.home, mode=0o750, user=data.uid, group=data.gid, ) ssh_directory = os.path.join(data.home, ".ssh") self._check_or_create_directory( location, system_context, ssh_directory, mode=0o700, user=data.uid, group=data.gid, ) key = read_file(system_context, keyfile, outside=True).decode("utf-8") authorized_file = os.path.join(ssh_directory, "authorized_keys") line = "" options = kwargs.get("options", "") if options: line = options + " " + key + "\n" else: line += key + "\n" self._execute( location.next_line(), system_context, "append", authorized_file, line, force=True, ) chown(system_context, data.uid, data.gid, authorized_file) chmod(system_context, 0o600, authorized_file)
def __call__( self, location: Location, system_context: SystemContext, *args: typing.Any, **kwargs: typing.Any, ) -> None: """Execute command.""" if not os.path.exists( os.path.join(system_context.boot_directory, "vmlinuz")): info("Skipping initrd generation: No vmlinuz in boot directory.") return root_hash = kwargs.get("root_hash", "") vg = system_context.substitution_expanded("DEFAULT_VG", None) image_fs = system_context.substitution_expanded("IMAGE_FS", None) image_device = _device_ify( system_context.substitution_expanded("IMAGE_DEVICE", None)) image_options = system_context.substitution_expanded( "IMAGE_OPTIONS", "") image_name = system_context.substitution_expanded( "CLRM_IMAGE_FILENAME", "") initrd = args[0] staging_area = os.path.join(system_context.cache_directory, "clrm_extra") os.makedirs(staging_area) modules = [ *system_context.substitution_expanded("INITRD_EXTRA_MODULES", "").split(","), "squashfs", *_install_image_file_support(staging_area, image_fs, image_device, image_options, image_name), *_install_lvm_support(staging_area, vg, image_name), *_install_sysroot_setup_support(staging_area), *_install_verity_support(staging_area, system_context, root_hash), *_install_volatile_support(staging_area, system_context), *_install_var_mount_support(staging_area, system_context), *_install_etc_shadow(staging_area, system_context), ] modules = [m for m in modules if m] # Trim empty modules (e.g. added by the substitution) system_context.set_or_append_substitution("INITRD_EXTRA_MODULES", ",".join(modules)) debug( f'INITRD_EXTRA_MODULES is now {system_context.substitution("INITRD_EXTRA_MODULES", "")}.' ) # Create Initrd: run( "/bin/sh", "-c", f'cd "{staging_area}" ; "{self._binary(Binaries.FIND)}" . | "{self._binary(Binaries.CPIO)}" -o -H newc > "{initrd}"', ) assert os.path.exists(initrd)
def __call__(self, location: Location, system_context: SystemContext, *args: typing.Any, **kwargs: typing.Any) -> None: """Execute command.""" if not os.path.exists(system_context.file_name('usr/bin/mkinitcpio')): info('Skipping initrd generation: No mkinitcpio binary.') return if not os.path.exists( os.path.join(system_context.boot_directory, 'vmlinuz')): info('Skipping initrd generation: No vmlinuz in boot directory.') return self._vg = system_context.substitution('DEFAULT_VG', None) if not self._vg: self._vg = None self._image_fs = system_context.substitution('IMAGE_FS', 'ext2') self._image_device = \ _deviceify(system_context.substitution('IMAGE_DEVICE', '')) self._image_options = system_context.substitution( 'IMAGE_OPTIONS', 'rw') name_prefix = system_context.substitution('DISTRO_ID', 'clrm') name_version = system_context.substitution('DISTRO_VERSION_ID', system_context.timestamp) self._full_name = "{}_{}".format(name_prefix, name_version) initrd = args[0] to_clean_up = [] # type: typing.List[str] to_clean_up += '/boot/vmlinuz' to_clean_up += self._install_extra_binaries(location, system_context) to_clean_up += self._create_systemd_units(location, system_context) to_clean_up += self._install_mkinitcpio(location, system_context) to_clean_up += self._install_mkinitcpio_hooks(location, system_context) copy(system_context, os.path.join(system_context.boot_directory, 'vmlinuz'), '/boot/vmlinuz', from_outside=True) run('/usr/bin/mkinitcpio', '-p', 'cleanroom', chroot=system_context.fs_directory, chroot_helper=self._binary(Binaries.CHROOT_HELPER)) initrd_directory = os.path.dirname(initrd) os.makedirs(initrd_directory, exist_ok=True) move(system_context, '/boot/initramfs.img', initrd, to_outside=True) _cleanup_extra_files(location, system_context, *to_clean_up) self._remove_mkinitcpio(location, system_context) assert (os.path.isfile(initrd))
def __call__( self, location: Location, system_context: SystemContext, *args: typing.Any, **kwargs: typing.Any, ) -> None: """Execute command.""" if not os.path.exists(os.path.join(system_context.boot_directory, "vmlinuz")): info("Skipping initrd generation: No vmlinuz in boot directory.") return initrd = args[0] self._install_dracut(location, system_context) copy( system_context, os.path.join(system_context.boot_directory, "vmlinuz"), "/boot/vmlinuz", from_outside=True, ) dracut_args: typing.List[str] = [] kernel_version = system_context.substitution_expanded("KERNEL_VERSION", "") assert kernel_version run( "/usr/bin/dracut", *dracut_args, "--no-early-microcode", "--no-hostonly", "--no-compress", "--reproducible", "--omit", "iscsi nbd network network-legacy nfs qemu qemu-net stratis", "--add", "busybox", "/boot/initramfs.img", kernel_version, chroot=system_context.fs_directory, chroot_helper=self._binary(Binaries.SYSTEMD_NSPAWN), ) initrd_directory = os.path.dirname(initrd) os.makedirs(initrd_directory, exist_ok=True) move(system_context, "/boot/initramfs.img", initrd, to_outside=True) self._remove_dracut(location, system_context) assert os.path.isfile(initrd)
def __call__(self, location: Location, system_context: SystemContext, *args: typing.Any, **kwargs: typing.Any) -> None: """Execute command.""" user = args[0] keyfile = args[1] info('Adding ssh key to {}\'s authorized_keys file.'.format(user)) data = UserHelper.user_data(user, root_directory=system_context.fs_directory) if data is None: raise GenerateError('"{}" could not find user "{}".'.format( self.name, user), location=location) trace('{} mapping: UID {}, GID {}, home: {}.'.format( user, data.uid, data.gid, data.home)) self._check_or_create_directory(location, system_context, data.home, mode=0o750, user=data.uid, group=data.gid) ssh_directory = os.path.join(data.home, '.ssh') self._check_or_create_directory(location, system_context, ssh_directory, mode=0o700, user=data.uid, group=data.gid) key = read_file(system_context, keyfile, outside=True).decode('utf-8') authorized_file = os.path.join(ssh_directory, 'authorized_keys') line = '' options = kwargs.get('options', '') if options: line = options + ' ' + key + '\n' else: line += key + '\n' self._execute(location.next_line(), system_context, 'append', authorized_file, line, force=True) chown(system_context, data.uid, data.gid, authorized_file) chmod(system_context, 0o600, authorized_file)
def __call__( self, location: Location, system_context: SystemContext, *args: typing.Any, **kwargs: typing.Any, ) -> None: """Execute command.""" to_sign = args[0] keep_unsigned = kwargs.get("keep_unsigned", False) if not kwargs.get("outside", False): to_sign = system_context.file_name(to_sign) systems_directory = system_context.systems_definition_directory key = os.path.join(systems_directory, kwargs.get("key", "config/efi/sign.key")) cert = os.path.join(systems_directory, kwargs.get("cert", "config/efi/sign.crt")) info( f"Signing EFI binary {to_sign} using key {key} and cert {cert} (keep unsigned: {keep_unsigned})." ) signed = to_sign + ".signed" assert os.path.isfile(key) assert os.path.isfile(cert) assert os.path.isfile(to_sign) assert not os.path.exists(signed) run( self._binary(Binaries.SBSIGN), "--key", key, "--cert", cert, "--output", signed, to_sign, ) if keep_unsigned: assert os.path.isfile(to_sign) assert os.path.isfile(signed) else: trace(f"Moving {signed} to {to_sign}.") os.remove(to_sign) os.rename(signed, to_sign) assert os.path.isfile(to_sign)
def _copy_modules( system_context: SystemContext, modules: typing.List[str], modules_dir: str, etc_dir: str, *, kernel_version: str, ): src_top = os.path.join(system_context.file_name("/usr/lib/modules"), kernel_version) target_top = os.path.join(modules_dir, kernel_version) known_modules = _kernel_module_map(src_top) to_load: typing.List[str] = [] to_install: typing.Set[str] = set() debug(f'Installing modules from "{src_top}" into "{target_top}"') for module in modules: m = module.replace("_", "-") if m not in known_modules: info(f"Module {m} not found. Was it built into the kernel?") continue p = known_modules[m] if p not in to_install: to_install.add(p) to_load.append(m) _write_modules_load_file(etc_dir, to_load) if not to_install: return # Install dependencies: dependencies = _parse_module_dependencies(src_top) to_test = to_install for t in to_test: to_install = to_install.union(_module_dependencies(t, dependencies)) for i in to_install: os.makedirs(os.path.dirname(os.path.join(target_top, i)), exist_ok=True) trace(f"Copying module {i}.") trace( f' "{os.path.join(src_top, i)}" => "{os.path.join(target_top, i)}".' ) shutil.copy2(os.path.join(src_top, i), os.path.join(target_top, i))
def _find_tests(system_context: SystemContext) -> typing.Generator[str, None, None]: """Find tests to run.""" tests_directory = system_context.system_tests_directory debug('Searching for tests in "{}".'.format(tests_directory)) for f in sorted(os.listdir(tests_directory)): test = os.path.join(tests_directory, f) if not os.path.isfile(test): trace('"{}": Not a file, skipping.'.format(test)) continue if not os.access(test, os.X_OK): trace('"{}": Not executable, skipping.'.format(test)) continue info('Found test: "{}"'.format(test)) yield test
def __call__( self, location: Location, system_context: SystemContext, *args: typing.Any, **kwargs: typing.Any, ) -> None: """Execute command.""" if not os.path.exists( os.path.join(system_context.boot_directory, "vmlinuz")): info("Skipping initrd generation: No vmlinuz in boot directory.") return initrd = args[0] to_clean_up: typing.List[str] = [] to_clean_up += "/boot/vmlinuz" to_clean_up += self._install_mkinitcpio(location, system_context) copy( system_context, os.path.join(system_context.boot_directory, "vmlinuz"), "/boot/vmlinuz", from_outside=True, ) run( "/usr/bin/mkinitcpio", "-p", "cleanroom", chroot=system_context.fs_directory, chroot_helper=self._binary(Binaries.SYSTEMD_NSPAWN), ) initrd_directory = os.path.dirname(initrd) os.makedirs(initrd_directory, exist_ok=True) move(system_context, "/boot/initramfs.img", initrd, to_outside=True) _cleanup_extra_files(location, system_context, *to_clean_up) self._remove_mkinitcpio(location, system_context) assert os.path.isfile(initrd)
def __call__( self, location: Location, system_context: SystemContext, *args: typing.Any, **kwargs: typing.Any, ) -> None: """Execute command.""" cert = kwargs.get("efi_cert", "") efi_emulator = kwargs.get("efi_emulator", "") key = kwargs.get("efi_key", "") skip_validation = kwargs.get("skip_validation", False) repository = args[0] repository_compression = kwargs.get("repository_compression", "zstd") repository_compression_level = kwargs.get( "repository_compression_level", 5) usr_only = kwargs.get("usr_only", True) debug_initrd = kwargs.get("debug_initrd", False) h2(f'Exporting system "{system_context.system_name}".') debug("Running Hooks.") self._run_all_exportcommand_hooks(system_context) verbose("Preparing system for export.") self._execute(location.next_line(), system_context, "_write_deploy_info") # Create some extra data: self._create_root_tarball(location, system_context) root_partition = self._create_root_fsimage(location, system_context, usr_only=usr_only) assert root_partition (verity_partition, root_hash) = self._create_rootverity_fsimage( location, system_context, rootfs=root_partition, ) assert root_hash has_kernel = os.path.exists( os.path.join(system_context.boot_directory, "vmlinuz")) if has_kernel: self._create_initrd(location, system_context) self._create_clrm_config_initrd(location, system_context, root_hash, debug=debug_initrd) cmdline = system_context.set_or_append_substitution( "KERNEL_CMDLINE", "systemd.volatile=true rootfstype=squashfs") cmdline = _setup_kernel_commandline(cmdline, root_hash) kernel_file = "" if has_kernel: trace( f'KERNEL_FILENAME: {system_context.substitution("KERNEL_FILENAME", "")}' ) kernel_file = os.path.join( system_context.boot_directory, system_context.substitution_expanded("KERNEL_FILENAME", ""), ) assert kernel_file self._create_complete_kernel( location, system_context, cmdline, kernel_file=kernel_file, efi_key=key, efi_cert=cert, ) efi_partition = os.path.join(system_context.cache_directory, "efi_partition.img") self._create_efi_partition( location, system_context, efi_partition=efi_partition, kernel_file=kernel_file, efi_emulator=efi_emulator, root_hash=root_hash, ) info("Validating installation for export.") if not skip_validation: _validate_installation(location.next_line(), system_context) export_directory = self.create_export_directory(system_context) assert export_directory self.create_image( location, system_context, export_directory, efi_partition=efi_partition, root_partition=root_partition, verity_partition=verity_partition, root_hash=root_hash, ) system_context.set_substitution("EXPORT_DIRECTORY", export_directory) verbose(f"Exporting all data in {export_directory}.") self._execute( location.next_line(), system_context, "_export_directory", export_directory, compression=repository_compression, compression_level=repository_compression_level, repository=repository, ) info("Cleaning up export location.") self.delete_export_directory(export_directory) system_context.set_substitution("EXPORT_DIRECTORY", "")
def __call__( self, location: Location, system_context: SystemContext, *args: typing.Any, **kwargs: typing.Any, ) -> None: """Execute command.""" if not os.path.exists(os.path.join(system_context.boot_directory, "vmlinuz")): info("Skipping initrd generation: No vmlinuz in boot directory.") return initrd = args[0] self._install_dracut(location, system_context) copy( system_context, os.path.join(system_context.boot_directory, "vmlinuz"), "/boot/vmlinuz", from_outside=True, ) dracut_args: typing.List[str] = [] modules = ( system_context.substitution_expanded("INITRD_EXTRA_MODULES", "") .replace(",", " ") .replace(" ", " ") .split(" ") ) modules = list(set(modules)) modules.sort() if modules: dracut_args += [ "--add-drivers", " ".join(modules), ] run( "/usr/bin/dracut", *dracut_args, "--no-early-microcode", "--no-hostonly", "--no-compress", "--reproducible", "--omit", "iscsi nbd network network-legacy nfs qemu qemu-net stratis", "--add", "busybox", "/boot/initramfs.img", chroot=system_context.fs_directory, chroot_helper=self._binary(Binaries.CHROOT_HELPER), ) initrd_directory = os.path.dirname(initrd) os.makedirs(initrd_directory, exist_ok=True) move(system_context, "/boot/initramfs.img", initrd, to_outside=True) self._remove_dracut(location, system_context) assert os.path.isfile(initrd)
def __call__( self, location: Location, system_context: SystemContext, *args: typing.Any, **kwargs: typing.Any, ) -> None: """Execute command.""" if not os.path.exists( os.path.join(system_context.boot_directory, "vmlinuz")): info("Skipping initrd generation: No vmlinuz in boot directory.") return root_hash = kwargs.get("root_hash", "") debug = kwargs.get("debug", False) vg = system_context.substitution_expanded("DEFAULT_VG", None) image_fs = system_context.substitution_expanded("IMAGE_FS", None) image_device = _device_ify( system_context.substitution_expanded("IMAGE_DEVICE", None)) image_options = system_context.substitution_expanded( "IMAGE_OPTIONS", "") image_name = system_context.substitution_expanded( "CLRM_IMAGE_FILENAME", "") kernel_version = system_context.substitution_expanded( "KERNEL_VERSION", "") assert kernel_version extra_modules = system_context.substitution_expanded( "INITRD_EXTRA_MODULES", "").split() initrd = args[0] assert not os.path.exists(initrd) staging_area = os.path.join(system_context.cache_directory, "clrm_extra") os.makedirs(staging_area) with TemporaryDirectory(prefix="clrm_cpio_") as tmp: _extract_initrds(system_context, tmp, cpio_command=self._binary(Binaries.CPIO)) cpio_files = _hash_directory(tmp) modules: typing.List[str] = [ *extra_modules, *system_context.substitution_expanded("INITRD_EXTRA_MODULES", "").split(","), "squashfs", *_install_image_file_support(tmp, image_fs, image_device, image_options, image_name), *_install_lvm_support(tmp, vg, image_name), *_install_sysroot_setup_support(tmp, system_context), *_install_verity_support(tmp, system_context, root_hash), *_install_volatile_support(tmp, system_context), *_install_var_mount_support(tmp, system_context), ] modules = [m for m in modules if m ] # Trim empty modules (e.g. added by the substitution) _install_extra_modules( system_context, tmp, modules, kernel_version=kernel_version, depmod_command=self._binary(Binaries.DEPMOD), ) if debug: _install_debug_support( system_context, tmp, ) updated_files = _hash_directory(tmp) for t in [ k for k, v in updated_files.items() if k not in cpio_files or cpio_files[k] != v ]: target = os.path.join(staging_area, t) src = os.path.join(tmp, t) trace( f' Installing module-related file: "{src}" => "{target}".' ) os.makedirs(os.path.dirname(target), exist_ok=True) shutil.copy2(src, target, follow_symlinks=False) trace(f"Copied {target} into {staging_area}.") # Create Initrd: run( "/bin/sh", "-c", f'"{self._binary(Binaries.FIND)}" . | "{self._binary(Binaries.CPIO)}" -o -H newc > "{initrd}"', work_directory=staging_area, ) assert os.path.exists(initrd)