def __call__(self, location: Location, system_context: SystemContext,
*args: typing.Any, **kwargs: typing.Any) -> None:
"""Execute command."""
self._setup(*args, **kwargs)
h2('Exporting system "{}".'.format(system_context.system_name))
debug('Running Hooks.')
self._run_all_exportcommand_hooks(system_context)
verbose('Preparing system for export.')
self.prepare_for_export(location, system_context)
info('Validating installation for export.')
if not self._skip_validation:
_validate_installation(location.next_line(), system_context)
export_directory \
= self.create_export_directory(system_context)
assert export_directory
system_context.set_substitution('EXPORT_DIRECTORY', export_directory)
verbose('Exporting all data in {}.'.format(export_directory))
self._execute(location.next_line(),
system_context,
'_export_directory',
export_directory,
compression=self._repository_compression,
compression_level=self._repository_compression_level,
repository=self._repository)
info('Cleaning up export location.')
self.delete_export_directory(export_directory)
def __call__(
self,
location: Location,
system_context: SystemContext,
*args: typing.Any,
**kwargs: typing.Any,
) -> None:
"""Execute command."""
if not os.path.exists(
os.path.join(system_context.boot_directory, "vmlinuz")):
info("Skipping initrd generation: No vmlinuz in boot directory.")
return
initrd = args[0]
copy(
system_context,
os.path.join(system_context.boot_directory, "vmlinuz"),
"/boot/vmlinuz",
from_outside=True,
)
# Use pre-created initrs if possible!
pre_created = os.path.join(system_context.initrd_parts_directory,
"50-clr")
assert os.path.exists(pre_created)
if pre_created != initrd:
shutil.copyfile(pre_created, initrd)
assert os.path.isfile(initrd)
def __call__(self, location: Location, system_context: SystemContext,
*args: typing.Any, **kwargs: typing.Any) -> None:
"""Execute command."""
to_sign = args[0]
keep_unsigned = kwargs.get("keep_unsigned", False)
if not kwargs.get("outside", False):
to_sign = system_context.file_name(to_sign)
systems_directory = system_context.systems_definition_directory
key = os.path.join(systems_directory,
kwargs.get("key", "config/efi/sign.key"))
cert = os.path.join(systems_directory,
kwargs.get("cert", "config/efi/sign.crt"))
info("Signing EFI binary {} using key {} and cert {}.".format(
input, key, cert))
output = to_sign + ".signed"
assert os.path.isfile(key)
assert os.path.isfile(cert)
assert os.path.isfile(to_sign)
assert not os.path.exists(output)
run(
self._binary(Binaries.SBSIGN),
"--key",
key,
"--cert",
cert,
"--output",
output,
to_sign,
)
if not keep_unsigned:
os.remove(to_sign)
shutil.move(output, to_sign)
def __call__(self, location: Location, system_context: SystemContext,
*args: typing.Any, **kwargs: typing.Any) -> None:
"""Execute command."""
if not os.path.exists(system_context.file_name("usr/bin/mkinitcpio")):
info("Skipping initrd generation: No mkinitcpio binary.")
return
if not os.path.exists(
os.path.join(system_context.boot_directory, "vmlinuz")):
info("Skipping initrd generation: No vmlinuz in boot directory.")
return
self._vg = system_context.substitution("DEFAULT_VG", None)
if not self._vg:
self._vg = None
self._image_fs = system_context.substitution("IMAGE_FS", "ext2")
self._image_device = _deviceify(
system_context.substitution("IMAGE_DEVICE", ""))
self._image_options = system_context.substitution(
"IMAGE_OPTIONS", "rw")
name_prefix = system_context.substitution("DISTRO_ID", "clrm")
name_version = system_context.substitution("DISTRO_VERSION_ID",
system_context.timestamp)
self._full_name = "{}_{}".format(name_prefix, name_version)
initrd = args[0]
to_clean_up = [] # type: typing.List[str]
to_clean_up += "/boot/vmlinuz"
to_clean_up += self._install_extra_binaries(location, system_context)
to_clean_up += self._create_systemd_units(location, system_context)
to_clean_up += self._install_mkinitcpio(location, system_context)
to_clean_up += self._install_mkinitcpio_hooks(location, system_context)
copy(
system_context,
os.path.join(system_context.boot_directory, "vmlinuz"),
"/boot/vmlinuz",
from_outside=True,
)
run(
"/usr/bin/mkinitcpio",
"-p",
"cleanroom",
chroot=system_context.fs_directory,
chroot_helper=self._binary(Binaries.CHROOT_HELPER),
)
initrd_directory = os.path.dirname(initrd)
os.makedirs(initrd_directory, exist_ok=True)
move(system_context, "/boot/initramfs.img", initrd, to_outside=True)
_cleanup_extra_files(location, system_context, *to_clean_up)
self._remove_mkinitcpio(location, system_context)
assert os.path.isfile(initrd)
def __call__(
self,
location: Location,
system_context: SystemContext,
*args: typing.Any,
**kwargs: typing.Any,
) -> None:
"""Execute command."""
user = args[0]
keyfile = args[1]
info(f"Adding ssh key to {user}'s authorized_keys file.")
data = UserHelper.user_data(user, root_directory=system_context.fs_directory)
if data is None:
raise GenerateError(
f'"{self.name}" could not find user "{user}".', location=location,
)
trace(f"{user} mapping: UID {data.uid}, GID {data.gid}, home: {data.home}.")
self._check_or_create_directory(
location,
system_context,
data.home,
mode=0o750,
user=data.uid,
group=data.gid,
)
ssh_directory = os.path.join(data.home, ".ssh")
self._check_or_create_directory(
location,
system_context,
ssh_directory,
mode=0o700,
user=data.uid,
group=data.gid,
)
key = read_file(system_context, keyfile, outside=True).decode("utf-8")
authorized_file = os.path.join(ssh_directory, "authorized_keys")
line = ""
options = kwargs.get("options", "")
if options:
line = options + " " + key + "\n"
else:
line += key + "\n"
self._execute(
location.next_line(),
system_context,
"append",
authorized_file,
line,
force=True,
)
chown(system_context, data.uid, data.gid, authorized_file)
chmod(system_context, 0o600, authorized_file)
def __call__(
self,
location: Location,
system_context: SystemContext,
*args: typing.Any,
**kwargs: typing.Any,
) -> None:
"""Execute command."""
if not os.path.exists(
os.path.join(system_context.boot_directory, "vmlinuz")):
info("Skipping initrd generation: No vmlinuz in boot directory.")
return
root_hash = kwargs.get("root_hash", "")
vg = system_context.substitution_expanded("DEFAULT_VG", None)
image_fs = system_context.substitution_expanded("IMAGE_FS", None)
image_device = _device_ify(
system_context.substitution_expanded("IMAGE_DEVICE", None))
image_options = system_context.substitution_expanded(
"IMAGE_OPTIONS", "")
image_name = system_context.substitution_expanded(
"CLRM_IMAGE_FILENAME", "")
initrd = args[0]
staging_area = os.path.join(system_context.cache_directory,
"clrm_extra")
os.makedirs(staging_area)
modules = [
*system_context.substitution_expanded("INITRD_EXTRA_MODULES",
"").split(","),
"squashfs",
*_install_image_file_support(staging_area, image_fs, image_device,
image_options, image_name),
*_install_lvm_support(staging_area, vg, image_name),
*_install_sysroot_setup_support(staging_area),
*_install_verity_support(staging_area, system_context, root_hash),
*_install_volatile_support(staging_area, system_context),
*_install_var_mount_support(staging_area, system_context),
*_install_etc_shadow(staging_area, system_context),
]
modules = [m for m in modules
if m] # Trim empty modules (e.g. added by the substitution)
system_context.set_or_append_substitution("INITRD_EXTRA_MODULES",
",".join(modules))
debug(
f'INITRD_EXTRA_MODULES is now {system_context.substitution("INITRD_EXTRA_MODULES", "")}.'
)
# Create Initrd:
run(
"/bin/sh",
"-c",
f'cd "{staging_area}" ; "{self._binary(Binaries.FIND)}" . | "{self._binary(Binaries.CPIO)}" -o -H newc > "{initrd}"',
)
assert os.path.exists(initrd)
def __call__(self, location: Location, system_context: SystemContext,
*args: typing.Any, **kwargs: typing.Any) -> None:
"""Execute command."""
if not os.path.exists(system_context.file_name('usr/bin/mkinitcpio')):
info('Skipping initrd generation: No mkinitcpio binary.')
return
if not os.path.exists(
os.path.join(system_context.boot_directory, 'vmlinuz')):
info('Skipping initrd generation: No vmlinuz in boot directory.')
return
self._vg = system_context.substitution('DEFAULT_VG', None)
if not self._vg:
self._vg = None
self._image_fs = system_context.substitution('IMAGE_FS', 'ext2')
self._image_device = \
_deviceify(system_context.substitution('IMAGE_DEVICE', ''))
self._image_options = system_context.substitution(
'IMAGE_OPTIONS', 'rw')
name_prefix = system_context.substitution('DISTRO_ID', 'clrm')
name_version = system_context.substitution('DISTRO_VERSION_ID',
system_context.timestamp)
self._full_name = "{}_{}".format(name_prefix, name_version)
initrd = args[0]
to_clean_up = [] # type: typing.List[str]
to_clean_up += '/boot/vmlinuz'
to_clean_up += self._install_extra_binaries(location, system_context)
to_clean_up += self._create_systemd_units(location, system_context)
to_clean_up += self._install_mkinitcpio(location, system_context)
to_clean_up += self._install_mkinitcpio_hooks(location, system_context)
copy(system_context,
os.path.join(system_context.boot_directory, 'vmlinuz'),
'/boot/vmlinuz',
from_outside=True)
run('/usr/bin/mkinitcpio',
'-p',
'cleanroom',
chroot=system_context.fs_directory,
chroot_helper=self._binary(Binaries.CHROOT_HELPER))
initrd_directory = os.path.dirname(initrd)
os.makedirs(initrd_directory, exist_ok=True)
move(system_context, '/boot/initramfs.img', initrd, to_outside=True)
_cleanup_extra_files(location, system_context, *to_clean_up)
self._remove_mkinitcpio(location, system_context)
assert (os.path.isfile(initrd))
def __call__(
self,
location: Location,
system_context: SystemContext,
*args: typing.Any,
**kwargs: typing.Any,
) -> None:
"""Execute command."""
if not os.path.exists(os.path.join(system_context.boot_directory, "vmlinuz")):
info("Skipping initrd generation: No vmlinuz in boot directory.")
return
initrd = args[0]
self._install_dracut(location, system_context)
copy(
system_context,
os.path.join(system_context.boot_directory, "vmlinuz"),
"/boot/vmlinuz",
from_outside=True,
)
dracut_args: typing.List[str] = []
kernel_version = system_context.substitution_expanded("KERNEL_VERSION", "")
assert kernel_version
run(
"/usr/bin/dracut",
*dracut_args,
"--no-early-microcode",
"--no-hostonly",
"--no-compress",
"--reproducible",
"--omit",
"iscsi nbd network network-legacy nfs qemu qemu-net stratis",
"--add",
"busybox",
"/boot/initramfs.img",
kernel_version,
chroot=system_context.fs_directory,
chroot_helper=self._binary(Binaries.SYSTEMD_NSPAWN),
)
initrd_directory = os.path.dirname(initrd)
os.makedirs(initrd_directory, exist_ok=True)
move(system_context, "/boot/initramfs.img", initrd, to_outside=True)
self._remove_dracut(location, system_context)
assert os.path.isfile(initrd)
def __call__(self, location: Location, system_context: SystemContext,
*args: typing.Any, **kwargs: typing.Any) -> None:
"""Execute command."""
user = args[0]
keyfile = args[1]
info('Adding ssh key to {}\'s authorized_keys file.'.format(user))
data = UserHelper.user_data(user,
root_directory=system_context.fs_directory)
if data is None:
raise GenerateError('"{}" could not find user "{}".'.format(
self.name, user),
location=location)
trace('{} mapping: UID {}, GID {}, home: {}.'.format(
user, data.uid, data.gid, data.home))
self._check_or_create_directory(location,
system_context,
data.home,
mode=0o750,
user=data.uid,
group=data.gid)
ssh_directory = os.path.join(data.home, '.ssh')
self._check_or_create_directory(location,
system_context,
ssh_directory,
mode=0o700,
user=data.uid,
group=data.gid)
key = read_file(system_context, keyfile, outside=True).decode('utf-8')
authorized_file = os.path.join(ssh_directory, 'authorized_keys')
line = ''
options = kwargs.get('options', '')
if options:
line = options + ' ' + key + '\n'
else:
line += key + '\n'
self._execute(location.next_line(),
system_context,
'append',
authorized_file,
line,
force=True)
chown(system_context, data.uid, data.gid, authorized_file)
chmod(system_context, 0o600, authorized_file)
def __call__(
self,
location: Location,
system_context: SystemContext,
*args: typing.Any,
**kwargs: typing.Any,
) -> None:
"""Execute command."""
to_sign = args[0]
keep_unsigned = kwargs.get("keep_unsigned", False)
if not kwargs.get("outside", False):
to_sign = system_context.file_name(to_sign)
systems_directory = system_context.systems_definition_directory
key = os.path.join(systems_directory,
kwargs.get("key", "config/efi/sign.key"))
cert = os.path.join(systems_directory,
kwargs.get("cert", "config/efi/sign.crt"))
info(
f"Signing EFI binary {to_sign} using key {key} and cert {cert} (keep unsigned: {keep_unsigned})."
)
signed = to_sign + ".signed"
assert os.path.isfile(key)
assert os.path.isfile(cert)
assert os.path.isfile(to_sign)
assert not os.path.exists(signed)
run(
self._binary(Binaries.SBSIGN),
"--key",
key,
"--cert",
cert,
"--output",
signed,
to_sign,
)
if keep_unsigned:
assert os.path.isfile(to_sign)
assert os.path.isfile(signed)
else:
trace(f"Moving {signed} to {to_sign}.")
os.remove(to_sign)
os.rename(signed, to_sign)
assert os.path.isfile(to_sign)
def _copy_modules(
system_context: SystemContext,
modules: typing.List[str],
modules_dir: str,
etc_dir: str,
*,
kernel_version: str,
):
src_top = os.path.join(system_context.file_name("/usr/lib/modules"),
kernel_version)
target_top = os.path.join(modules_dir, kernel_version)
known_modules = _kernel_module_map(src_top)
to_load: typing.List[str] = []
to_install: typing.Set[str] = set()
debug(f'Installing modules from "{src_top}" into "{target_top}"')
for module in modules:
m = module.replace("_", "-")
if m not in known_modules:
info(f"Module {m} not found. Was it built into the kernel?")
continue
p = known_modules[m]
if p not in to_install:
to_install.add(p)
to_load.append(m)
_write_modules_load_file(etc_dir, to_load)
if not to_install:
return
# Install dependencies:
dependencies = _parse_module_dependencies(src_top)
to_test = to_install
for t in to_test:
to_install = to_install.union(_module_dependencies(t, dependencies))
for i in to_install:
os.makedirs(os.path.dirname(os.path.join(target_top, i)),
exist_ok=True)
trace(f"Copying module {i}.")
trace(
f' "{os.path.join(src_top, i)}" => "{os.path.join(target_top, i)}".'
)
shutil.copy2(os.path.join(src_top, i), os.path.join(target_top, i))
def _find_tests(system_context: SystemContext) -> typing.Generator[str, None, None]:
"""Find tests to run."""
tests_directory = system_context.system_tests_directory
debug('Searching for tests in "{}".'.format(tests_directory))
for f in sorted(os.listdir(tests_directory)):
test = os.path.join(tests_directory, f)
if not os.path.isfile(test):
trace('"{}": Not a file, skipping.'.format(test))
continue
if not os.access(test, os.X_OK):
trace('"{}": Not executable, skipping.'.format(test))
continue
info('Found test: "{}"'.format(test))
yield test
def __call__(
self,
location: Location,
system_context: SystemContext,
*args: typing.Any,
**kwargs: typing.Any,
) -> None:
"""Execute command."""
if not os.path.exists(
os.path.join(system_context.boot_directory, "vmlinuz")):
info("Skipping initrd generation: No vmlinuz in boot directory.")
return
initrd = args[0]
to_clean_up: typing.List[str] = []
to_clean_up += "/boot/vmlinuz"
to_clean_up += self._install_mkinitcpio(location, system_context)
copy(
system_context,
os.path.join(system_context.boot_directory, "vmlinuz"),
"/boot/vmlinuz",
from_outside=True,
)
run(
"/usr/bin/mkinitcpio",
"-p",
"cleanroom",
chroot=system_context.fs_directory,
chroot_helper=self._binary(Binaries.SYSTEMD_NSPAWN),
)
initrd_directory = os.path.dirname(initrd)
os.makedirs(initrd_directory, exist_ok=True)
move(system_context, "/boot/initramfs.img", initrd, to_outside=True)
_cleanup_extra_files(location, system_context, *to_clean_up)
self._remove_mkinitcpio(location, system_context)
assert os.path.isfile(initrd)
def __call__(
self,
location: Location,
system_context: SystemContext,
*args: typing.Any,
**kwargs: typing.Any,
) -> None:
"""Execute command."""
cert = kwargs.get("efi_cert", "")
efi_emulator = kwargs.get("efi_emulator", "")
key = kwargs.get("efi_key", "")
skip_validation = kwargs.get("skip_validation", False)
repository = args[0]
repository_compression = kwargs.get("repository_compression", "zstd")
repository_compression_level = kwargs.get(
"repository_compression_level", 5)
usr_only = kwargs.get("usr_only", True)
debug_initrd = kwargs.get("debug_initrd", False)
h2(f'Exporting system "{system_context.system_name}".')
debug("Running Hooks.")
self._run_all_exportcommand_hooks(system_context)
verbose("Preparing system for export.")
self._execute(location.next_line(), system_context,
"_write_deploy_info")
# Create some extra data:
self._create_root_tarball(location, system_context)
root_partition = self._create_root_fsimage(location,
system_context,
usr_only=usr_only)
assert root_partition
(verity_partition, root_hash) = self._create_rootverity_fsimage(
location,
system_context,
rootfs=root_partition,
)
assert root_hash
has_kernel = os.path.exists(
os.path.join(system_context.boot_directory, "vmlinuz"))
if has_kernel:
self._create_initrd(location, system_context)
self._create_clrm_config_initrd(location,
system_context,
root_hash,
debug=debug_initrd)
cmdline = system_context.set_or_append_substitution(
"KERNEL_CMDLINE", "systemd.volatile=true rootfstype=squashfs")
cmdline = _setup_kernel_commandline(cmdline, root_hash)
kernel_file = ""
if has_kernel:
trace(
f'KERNEL_FILENAME: {system_context.substitution("KERNEL_FILENAME", "")}'
)
kernel_file = os.path.join(
system_context.boot_directory,
system_context.substitution_expanded("KERNEL_FILENAME", ""),
)
assert kernel_file
self._create_complete_kernel(
location,
system_context,
cmdline,
kernel_file=kernel_file,
efi_key=key,
efi_cert=cert,
)
efi_partition = os.path.join(system_context.cache_directory,
"efi_partition.img")
self._create_efi_partition(
location,
system_context,
efi_partition=efi_partition,
kernel_file=kernel_file,
efi_emulator=efi_emulator,
root_hash=root_hash,
)
info("Validating installation for export.")
if not skip_validation:
_validate_installation(location.next_line(), system_context)
export_directory = self.create_export_directory(system_context)
assert export_directory
self.create_image(
location,
system_context,
export_directory,
efi_partition=efi_partition,
root_partition=root_partition,
verity_partition=verity_partition,
root_hash=root_hash,
)
system_context.set_substitution("EXPORT_DIRECTORY", export_directory)
verbose(f"Exporting all data in {export_directory}.")
self._execute(
location.next_line(),
system_context,
"_export_directory",
export_directory,
compression=repository_compression,
compression_level=repository_compression_level,
repository=repository,
)
info("Cleaning up export location.")
self.delete_export_directory(export_directory)
system_context.set_substitution("EXPORT_DIRECTORY", "")
def __call__(
self,
location: Location,
system_context: SystemContext,
*args: typing.Any,
**kwargs: typing.Any,
) -> None:
"""Execute command."""
if not os.path.exists(os.path.join(system_context.boot_directory, "vmlinuz")):
info("Skipping initrd generation: No vmlinuz in boot directory.")
return
initrd = args[0]
self._install_dracut(location, system_context)
copy(
system_context,
os.path.join(system_context.boot_directory, "vmlinuz"),
"/boot/vmlinuz",
from_outside=True,
)
dracut_args: typing.List[str] = []
modules = (
system_context.substitution_expanded("INITRD_EXTRA_MODULES", "")
.replace(",", " ")
.replace(" ", " ")
.split(" ")
)
modules = list(set(modules))
modules.sort()
if modules:
dracut_args += [
"--add-drivers",
" ".join(modules),
]
run(
"/usr/bin/dracut",
*dracut_args,
"--no-early-microcode",
"--no-hostonly",
"--no-compress",
"--reproducible",
"--omit",
"iscsi nbd network network-legacy nfs qemu qemu-net stratis",
"--add",
"busybox",
"/boot/initramfs.img",
chroot=system_context.fs_directory,
chroot_helper=self._binary(Binaries.CHROOT_HELPER),
)
initrd_directory = os.path.dirname(initrd)
os.makedirs(initrd_directory, exist_ok=True)
move(system_context, "/boot/initramfs.img", initrd, to_outside=True)
self._remove_dracut(location, system_context)
assert os.path.isfile(initrd)
def __call__(
self,
location: Location,
system_context: SystemContext,
*args: typing.Any,
**kwargs: typing.Any,
) -> None:
"""Execute command."""
if not os.path.exists(
os.path.join(system_context.boot_directory, "vmlinuz")):
info("Skipping initrd generation: No vmlinuz in boot directory.")
return
root_hash = kwargs.get("root_hash", "")
debug = kwargs.get("debug", False)
vg = system_context.substitution_expanded("DEFAULT_VG", None)
image_fs = system_context.substitution_expanded("IMAGE_FS", None)
image_device = _device_ify(
system_context.substitution_expanded("IMAGE_DEVICE", None))
image_options = system_context.substitution_expanded(
"IMAGE_OPTIONS", "")
image_name = system_context.substitution_expanded(
"CLRM_IMAGE_FILENAME", "")
kernel_version = system_context.substitution_expanded(
"KERNEL_VERSION", "")
assert kernel_version
extra_modules = system_context.substitution_expanded(
"INITRD_EXTRA_MODULES", "").split()
initrd = args[0]
assert not os.path.exists(initrd)
staging_area = os.path.join(system_context.cache_directory,
"clrm_extra")
os.makedirs(staging_area)
with TemporaryDirectory(prefix="clrm_cpio_") as tmp:
_extract_initrds(system_context,
tmp,
cpio_command=self._binary(Binaries.CPIO))
cpio_files = _hash_directory(tmp)
modules: typing.List[str] = [
*extra_modules,
*system_context.substitution_expanded("INITRD_EXTRA_MODULES",
"").split(","),
"squashfs",
*_install_image_file_support(tmp, image_fs, image_device,
image_options, image_name),
*_install_lvm_support(tmp, vg, image_name),
*_install_sysroot_setup_support(tmp, system_context),
*_install_verity_support(tmp, system_context, root_hash),
*_install_volatile_support(tmp, system_context),
*_install_var_mount_support(tmp, system_context),
]
modules = [m for m in modules if m
] # Trim empty modules (e.g. added by the substitution)
_install_extra_modules(
system_context,
tmp,
modules,
kernel_version=kernel_version,
depmod_command=self._binary(Binaries.DEPMOD),
)
if debug:
_install_debug_support(
system_context,
tmp,
)
updated_files = _hash_directory(tmp)
for t in [
k for k, v in updated_files.items()
if k not in cpio_files or cpio_files[k] != v
]:
target = os.path.join(staging_area, t)
src = os.path.join(tmp, t)
trace(
f' Installing module-related file: "{src}" => "{target}".'
)
os.makedirs(os.path.dirname(target), exist_ok=True)
shutil.copy2(src, target, follow_symlinks=False)
trace(f"Copied {target} into {staging_area}.")
# Create Initrd:
run(
"/bin/sh",
"-c",
f'"{self._binary(Binaries.FIND)}" . | "{self._binary(Binaries.CPIO)}" -o -H newc > "{initrd}"',
work_directory=staging_area,
)
assert os.path.exists(initrd)