def copy_efi_partition(*,
image_file: str,
efi_device, tempdir: str,
kernel_only: bool = True):
verbose('Copying EFI configuration out of image file.')
with disk.NbdDevice(image_file, disk_format='raw') \
as internal_device:
internal_device.wait_for_device_node(partition=1)
with mount.Mount(internal_device.device(1),
os.path.join(tempdir, '_efi')) \
as int_efi:
with mount.Mount(efi_device,
os.path.join(tempdir, 'efi'), fs_type='vfat') \
as efi:
if kernel_only:
img_dir = os.path.join(int_efi, 'EFI', 'Linux')
efi_dir = os.path.join(efi, 'EFI', 'Linux')
assert os.path.isdir(img_dir)
if not os.path.isdir(efi_dir):
os.makedirs(efi_dir)
for f in [f for f in os.listdir(img_dir)
if os.path.isfile(os.path.join(img_dir, f))]:
trace('Copying EFI kernel {}.'.format(f))
copyfile(os.path.join(img_dir, f),
os.path.join(efi_dir, f))
else:
trace('Copying EFI folder into system.')
copy_tree(int_efi, efi)
def execute_with_system_mounted(to_execute: typing.Callable[[str, str], int],
*, image_file: str, tmp_dir: str) -> int:
assert os.path.isfile(image_file)
with disk.NbdDevice(image_file, disk_format="raw",
read_only=True) as device:
verbose("Mounting EFI...")
device.wait_for_device_node(partition=1)
with mount.Mount(
device.device(1),
os.path.join(tmp_dir, "EFI"),
fs_type="vfat",
options="ro",
) as efi:
verbose("Mounting root filesystem...")
with mount.Mount(
device.device(2),
os.path.join(tmp_dir, "root"),
fs_type="squashfs",
options="ro",
) as root:
trace(f'Executing with EFI "{efi}" and root "{root}".')
result = to_execute(efi, root)
return result
def __call__(self, location: Location, system_context: SystemContext,
*args: typing.Any, **kwargs: typing.Any) -> None:
"""Execute command."""
self._setup(*args, **kwargs)
h2('Exporting system "{}".'.format(system_context.system_name))
debug('Running Hooks.')
self._run_all_exportcommand_hooks(system_context)
verbose('Preparing system for export.')
self.prepare_for_export(location, system_context)
info('Validating installation for export.')
if not self._skip_validation:
_validate_installation(location.next_line(), system_context)
export_directory \
= self.create_export_directory(system_context)
assert export_directory
system_context.set_substitution('EXPORT_DIRECTORY', export_directory)
verbose('Exporting all data in {}.'.format(export_directory))
self._execute(location.next_line(),
system_context,
'_export_directory',
export_directory,
compression=self._repository_compression,
compression_level=self._repository_compression_level,
repository=self._repository)
info('Cleaning up export location.')
self.delete_export_directory(export_directory)
def _populate_with_efi_emulator(staging_area: str, efi_emulator: str):
verbose("Installing Clover binaries into EFI partition")
assert efi_emulator, "Missing EFI emulator path"
shutil.copy(
os.path.join(efi_emulator, "Bootloaders/x64/boot7"),
os.path.join(staging_area, "boot"),
follow_symlinks=False,
)
efi_dir = os.path.join(staging_area, "EFI")
if not os.path.isdir(efi_dir):
os.makedirs(efi_dir)
shutil.copytree(os.path.join(efi_emulator, "EFI/CLOVER"),
os.path.join(efi_dir, "CLOVER"))
config_file = os.path.join(efi_dir, "CLOVER/config.plist")
with open(config_file, "wb") as config:
config.write("""\
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>Boot</key>
<dict>
<key>DefaultVolume</key>
<string>EFI</string>
<key>DefaultLoader</key>
<string>\\EFI\\systemd\\systemd-bootx64.efi</string>
<key>Fast</key>
<true/>
</dict>
<key>GUI</key>
<dict>
<key>Custom</key>
<dict>
<key>Entries</key>
<array>
<dict>
<key>Hidden</key>
<false/>
<key>Disabled</key>
<false/>
<key>Image</key>
<string>os_arch</string>
<key>Volume</key>
<string>EFI</string>
<key>Path</key>
<string>\\EFI\\systemd\\systemd-bootx64.efi</string>
<key>Title</key>
<string>Cleanroom Linux</string>
<key>Type</key>
<string>Linux</string>
</dict>
</array>
</dict>
</dict>
</dict>
</plist>
""".encode("utf-8"))
def __call__(
self,
location: Location,
system_context: SystemContext,
*args: typing.Any,
**kwargs: typing.Any,
) -> None:
"""Execute command."""
## validate package type:
if system_context.substitution("CLRM_PACKAGE_TYPE", ""):
raise GenerateError(
"Trying to run swupd_init on a system that already has a CLRM_PACKAGE_TYPE defined."
)
system_context.set_substitution("CLRM_PACKAGE_TYPE", "swupd")
run(
self._binary(Binaries.SWUPD),
"autoupdate",
f"--path={system_context.fs_directory}",
"--disable",
"--no-progress",
returncode=28,
)
# Setup update-helper so that swupd os-install will actually work:
os.makedirs(system_context.file_name("/usr/bin"))
with open(system_context.file_name("/usr/bin/update-helper"), "wb") as fd:
fd.write(
dedent(
"""\
#!/usr/bin/sh
exit 0
"""
).encode("utf-8")
)
os.chmod(system_context.file_name("/usr/bin/update-helper"), 0o755)
run(
self._binary(Binaries.SWUPD),
"os-install",
f"--path={system_context.fs_directory}",
"--skip-optional",
"--no-progress",
)
location.set_description("Move systemd files into /usr")
self._add_hook(location, system_context, "_teardown", "systemd_cleanup")
with open(system_context.file_name("/usr/lib/os-release"), "r") as osr:
for l in osr:
l = l.strip()
if l.startswith("BUILD_ID="):
build_id = l[9:]
verbose(f"Installed {build_id}.")
system_context.set_substitution("DISTRO_VERSION_ID", build_id)
system_context.set_substitution("DISTRO_VERSION", build_id)
self._execute(location.next_line(), system_context, "create_os_release")
def execute_with_system_mounted(to_execute: typing.Callable[[str, str], None],
*,
repository: str,
system_name: str,
system_version: str = "") -> None:
with TemporaryDirectory(prefix="clrm_qemu_") as tempdir:
verbose("Extracting image")
image_path = export_into_directory(system_name,
tempdir,
repository=repository,
version=system_version)
assert os.path.isfile(image_path)
with disk.NbdDevice(image_path, disk_format="raw") as device:
verbose("Mounting EFI...")
device.wait_for_device_node(partition=1)
with mount.Mount(
device.device(1),
os.path.join(tempdir, "EFI"),
fs_type="vfat",
options="ro",
) as efi:
verbose("Mounting root filesystem...")
with mount.Mount(
device.device(2),
os.path.join(tempdir, "root"),
fs_type="squashfs",
options="ro",
) as root:
verbose('Executing with EFI "{}" and root "{}".'.format(
efi, root))
to_execute(efi, root)
def execute_with_system_mounted(to_execute: typing.Callable[[str, str], None],
*,
repository: str, system_name: str,
system_version: str = '') -> None:
with TemporaryDirectory(prefix='clrm_qemu_') as tempdir:
verbose('Extracting image')
image_path \
= export_into_directory(system_name, tempdir,
repository=repository,
version=system_version)
assert os.path.isfile(image_path)
with disk.NbdDevice(image_path, disk_format='raw') as device:
verbose('Mounting EFI...')
device.wait_for_device_node(partition=1)
with mount.Mount(device.device(1), os.path.join(tempdir, 'EFI'),
fs_type='vfat', options='ro') as efi:
verbose('Mounting root filesystem...')
with mount.Mount(device.device(2),
os.path.join(tempdir, 'root'),
fs_type='squashfs', options='ro') as root:
verbose('Executing with EFI "{}" and root "{}".'
.format(efi, root))
to_execute(efi, root)
def _execution(efi: str, rootfs: str, *, command: str) -> int:
to_exec = command or '/usr/bin/bash -c "read -n1 -s"'
prompt = "" if command else "<<< Press any key to continue >>>"
env = os.environ
env["EFI_MOUNT"] = efi
env["ROOT_MOUNT"] = rootfs
verbose(f"Running {command}.")
print(f'EFI partition is mounted at "{efi}".')
print(f'Root partition is mounted at "{rootfs}".')
if prompt:
print(prompt)
return tool.run(*split(to_exec), env=env).returncode
def _create_hdd_image(device):
verbose('hdd.img created.')
partitioner = disk.Partitioner(device)
partitioner.repartition([
disk.Partitioner.efi_partition(size='512M'),
disk.Partitioner.swap_partition(size='1G', name='swap'),
disk.Partitioner.data_partition(name='data')])
verbose('hdd.img repartitioned.')
debug('Format EFI partitition.')
run('/usr/bin/mkfs.vfat', device.device(1))
debug('Set up swap partitition.')
run('/usr/bin/mkswap', device.device(2))
debug('Format data partitition.')
run('/usr/bin/mkfs.btrfs', '-L', 'fs_btrfs', device.device(3))
def __call__(self, location: Location, system_context: SystemContext,
*args: typing.Any, **kwargs: typing.Any) -> None:
"""Execute command."""
base_system = args[0]
if base_system == 'scratch':
assert system_context.base_context is None
verbose('Building from scratch!')
self._add_hook(location, system_context, 'testing', '_test')
self._execute(location, system_context, '_setup')
else:
assert system_context.base_context.system_name == base_system
verbose('Building on top of {}.'.format(base_system))
self._execute(location, system_context, '_restore', base_system)
self._run_hooks(system_context, "_setup")
def create_qemu_image(image_path: str,
*,
image_size: str,
image_format: str = "qcow2",
system_name: str,
system_version: str = "",
repository: str,
tempdir: str) -> str:
trace("Creating image file {}.".format(image_path))
with disk.NbdDevice.new_image_file(image_path,
image_size,
disk_format=image_format) as device:
_create_hdd_image(device)
debug("mounting data partition for further setup.")
with mount.Mount(
device.device(3),
os.path.join(tempdir, "data"),
fs_type="btrfs",
options="subvolid=0",
fallback_cwd=os.getcwd(),
) as data_dir:
_setup_btrfs(data_dir)
extract_location = os.path.join(data_dir, ".images")
verbose("Extracting system image to {}.".format(extract_location))
extracted_version = tool.write_image(
system_name,
extract_location,
repository=repository,
version=system_version,
)
extracted_image = os.path.join(data_dir, ".images",
"clrm_{}".format(extracted_version))
assert os.path.isfile(extracted_image)
tool.copy_efi_partition(
image_file=extracted_image,
efi_device=device.device(1),
tempdir=tempdir,
kernel_only=False,
)
return image_path
def _create_hdd_image(device):
verbose("hdd.img created.")
partitioner = disk.Partitioner(device)
partitioner.repartition([
disk.Partitioner.efi_partition(size="512M"),
disk.Partitioner.swap_partition(size="1G", name="swap"),
disk.Partitioner.data_partition(name="data"),
])
verbose("hdd.img repartitioned.")
debug("Format EFI partitition.")
run("/usr/bin/mkfs.vfat", device.device(1))
debug("Set up swap partitition.")
run("/usr/bin/mkswap", device.device(2))
debug("Format data partitition.")
run("/usr/bin/mkfs.btrfs", "-L", "fs_btrfs", device.device(3))
def __call__(
self,
location: Location,
system_context: SystemContext,
*args: typing.Any,
**kwargs: typing.Any,
) -> None:
"""Execute command."""
base_system = args[0]
if base_system == "scratch":
assert system_context.base_context is None
verbose("Building from scratch!")
self._add_hook(location, system_context, "testing", "_test")
self._execute(location, system_context, "_setup")
else:
assert (system_context.base_context
and system_context.base_context.system_name == base_system)
verbose(f"Building on top of {base_system}.")
self._execute(location, system_context, "_restore", base_system)
self._run_hooks(system_context, "_setup")
def create_qemu_image(image_path: str, *,
image_size: str, image_format: str = 'qcow2',
system_name: str, system_version: str = '',
repository: str,
tempdir: str) -> str:
trace('Creating image file {}.'.format(image_path))
with disk.NbdDevice.new_image_file(image_path, image_size,
disk_format=image_format) as device:
_create_hdd_image(device)
debug('mounting data partition for further setup.')
with mount.Mount(device.device(3), os.path.join(tempdir, 'data'),
fs_type='btrfs',
options='subvolid=0',
fallback_cwd=os.getcwd()) as data_dir:
_setup_btrfs(data_dir)
extract_location = os.path.join(data_dir, '.images')
verbose('Extracting system image to {}.'
.format(extract_location))
extracted_version \
= tool.write_image(system_name, extract_location,
repository=repository,
version=system_version)
extracted_image \
= os.path.join(data_dir, '.images',
'clrm_{}'.format(extracted_version))
assert os.path.isfile(extracted_image)
tool.copy_efi_partition(image_file=extracted_image,
efi_device=device.device(1),
tempdir=tempdir,
kernel_only=False)
return image_path
def _execution(efi: str, rootfs: str, *, command: str) -> None:
to_exec = command or '/usr/bin/bash -c "read -n1 -s"'
prompt = "" if command else "<<< Press any key to continue >>>"
env = os.environ
env["EFI_MOUNT"] = efi
env["ROOT_MOUNT"] = rootfs
verbose("Running {}.".format(command))
verbose('EFI_MOUNT env var set to : "{}".'.format(efi))
verbose('ROOT_MOUNT env var set to: "{}".'.format(rootfs))
print('EFI partition is mounted at "{}".'.format(efi))
print('Root partition is mounted at "{}".'.format(rootfs))
if prompt:
print(prompt)
tool.run(*split(to_exec), env=env)
def __call__(
self,
location: Location,
system_context: SystemContext,
*args: typing.Any,
**kwargs: typing.Any,
) -> None:
"""Execute command."""
cert = kwargs.get("efi_cert", "")
efi_emulator = kwargs.get("efi_emulator", "")
key = kwargs.get("efi_key", "")
skip_validation = kwargs.get("skip_validation", False)
repository = args[0]
repository_compression = kwargs.get("repository_compression", "zstd")
repository_compression_level = kwargs.get(
"repository_compression_level", 5)
usr_only = kwargs.get("usr_only", True)
debug_initrd = kwargs.get("debug_initrd", False)
h2(f'Exporting system "{system_context.system_name}".')
debug("Running Hooks.")
self._run_all_exportcommand_hooks(system_context)
verbose("Preparing system for export.")
self._execute(location.next_line(), system_context,
"_write_deploy_info")
# Create some extra data:
self._create_root_tarball(location, system_context)
root_partition = self._create_root_fsimage(location,
system_context,
usr_only=usr_only)
assert root_partition
(verity_partition, root_hash) = self._create_rootverity_fsimage(
location,
system_context,
rootfs=root_partition,
)
assert root_hash
has_kernel = os.path.exists(
os.path.join(system_context.boot_directory, "vmlinuz"))
if has_kernel:
self._create_initrd(location, system_context)
self._create_clrm_config_initrd(location,
system_context,
root_hash,
debug=debug_initrd)
cmdline = system_context.set_or_append_substitution(
"KERNEL_CMDLINE", "systemd.volatile=true rootfstype=squashfs")
cmdline = _setup_kernel_commandline(cmdline, root_hash)
kernel_file = ""
if has_kernel:
trace(
f'KERNEL_FILENAME: {system_context.substitution("KERNEL_FILENAME", "")}'
)
kernel_file = os.path.join(
system_context.boot_directory,
system_context.substitution_expanded("KERNEL_FILENAME", ""),
)
assert kernel_file
self._create_complete_kernel(
location,
system_context,
cmdline,
kernel_file=kernel_file,
efi_key=key,
efi_cert=cert,
)
efi_partition = os.path.join(system_context.cache_directory,
"efi_partition.img")
self._create_efi_partition(
location,
system_context,
efi_partition=efi_partition,
kernel_file=kernel_file,
efi_emulator=efi_emulator,
root_hash=root_hash,
)
info("Validating installation for export.")
if not skip_validation:
_validate_installation(location.next_line(), system_context)
export_directory = self.create_export_directory(system_context)
assert export_directory
self.create_image(
location,
system_context,
export_directory,
efi_partition=efi_partition,
root_partition=root_partition,
verity_partition=verity_partition,
root_hash=root_hash,
)
system_context.set_substitution("EXPORT_DIRECTORY", export_directory)
verbose(f"Exporting all data in {export_directory}.")
self._execute(
location.next_line(),
system_context,
"_export_directory",
export_directory,
compression=repository_compression,
compression_level=repository_compression_level,
repository=repository,
)
info("Cleaning up export location.")
self.delete_export_directory(export_directory)
system_context.set_substitution("EXPORT_DIRECTORY", "")
def __call__(
self,
location: Location,
system_context: SystemContext,
*args: typing.Any,
**kwargs: typing.Any,
) -> None:
"""Execute command."""
## validate package type:
if system_context.substitution("CLRM_PACKAGE_TYPE", ""):
raise GenerateError(
"Trying to run swupd_init on a system that already has a CLRM_PACKAGE_TYPE defined."
)
system_context.set_substitution("CLRM_PACKAGE_TYPE", "swupd")
system_context.set_substitution("DISTRO_PRETTY_NAME",
"Cleanroom - CLR")
run(
self._binary(Binaries.SWUPD),
"autoupdate",
f"--path={system_context.fs_directory}",
"--disable",
"--no-progress",
returncode=28,
)
# Setup update-helper so that swupd os-install will actually work:
os.makedirs(system_context.file_name("/usr/bin"))
with open(system_context.file_name("/usr/bin/update-helper"),
"wb") as fd:
fd.write(
dedent("""\
#!/usr/bin/sh
exit 0
""").encode("utf-8"))
os.chmod(system_context.file_name("/usr/bin/update-helper"), 0o755)
run(
self._binary(Binaries.SWUPD),
"os-install",
f"--path={system_context.fs_directory}",
"--skip-optional",
"--no-progress",
)
system_context.set_substitution("INITRD_GENERATOR", "clr")
location.set_description("Move systemd files into /usr")
self._add_hook(location, system_context, "_teardown",
"systemd_cleanup")
location.set_description("Setup Clearlinux triggers as hooks")
self._add_hook(
location,
system_context,
"export",
"run",
"/usr/bin/systemd-tmpfiles",
"--create",
inside=True,
)
self._add_hook(
location,
system_context,
"export",
"_clr_catalog_trigger",
)
self._add_hook(
location,
system_context,
"export",
"_clr_fontconfig_trigger",
)
self._add_hook(
location,
system_context,
"export",
"_clr_glib_schemas_trigger",
)
self._add_hook(
location,
system_context,
"export",
"_clr_graphviz_dot_trigger",
)
self._add_hook(
location,
system_context,
"export",
"_clr_hwdb_update_trigger",
)
self._add_hook(
location,
system_context,
"export",
"_clr_icon_cache_update_trigger",
)
self._add_hook(
location,
system_context,
"export",
"_clr_ldconfig_trigger",
)
self._add_hook(
location,
system_context,
"export",
"_clr_locale_archive_trigger",
)
self._add_hook(
location,
system_context,
"export",
"_clr_mandb_trigger",
)
self._add_hook(
location,
system_context,
"export",
"_clr_sysusers_trigger",
)
self._add_hook(
location,
system_context,
"export",
"_clr_dynamic_trust_store_trigger",
)
self._add_hook(
location,
system_context,
"export",
"_clr_mime_update_trigger",
)
with open(system_context.file_name("/usr/lib/os-release"), "r") as osr:
for l in osr:
l = l.strip()
if l.startswith("BUILD_ID="):
build_id = l[9:]
verbose(f"Installed {build_id}.")
system_context.set_substitution("DISTRO_VERSION_ID",
build_id)
system_context.set_substitution("DISTRO_VERSION", build_id)
system_context.set_substitution("DISTRO_ID_LIKE", "clearlinux")
system_context.set_substitution(
"ROOTFS_PARTLABEL", "root_${TIMESTAMP}-${DISTRO_VERSION_ID}")
system_context.set_substitution(
"VRTYFS_PARTLABEL", "vrty_${TIMESTAMP}-${DISTRO_VERSION_ID}")
system_context.set_substitution(
"KERNEL_FILENAME",
"${PRETTY_SYSTEM_NAME}_${TIMESTAMP}-${DISTRO_VERSION_ID}.efi",
)
system_context.set_substitution(
"CLRM_IMAGE_FILENAME",
"${PRETTY_SYSTEM_NAME}_${TIMESTAMP}-${DISTRO_VERSION_ID}",
)
self._execute(location.next_line(), system_context,
"create_os_release")
