def add_trusted_apt_key(self):
run([
'apt-key', '--keyring',
'/etc/apt/trusted.gpg.d/debian-archive-truenas-automatic.gpg',
'add',
os.path.join(BUILDER_DIR, 'keys/truenas.gpg')
])
def umount_chroot_basedir():
for command in (
['umount', '-f', PACKAGE_PATH],
['umount', '-f', os.path.join(CHROOT_BASEDIR, 'proc')],
['umount', '-f', os.path.join(CHROOT_BASEDIR, 'sys')],
):
run(command, check=False, log=False)
def clean_mounts(self):
for command in (
['umount', '-f',
os.path.join(self.chroot_basedir, 'proc')],
['umount', '-f',
os.path.join(self.chroot_basedir, 'sys')],
):
run(command, check=False, log=False)
def setup_mounts(self):
run([
'mount', 'proc',
os.path.join(self.chroot_basedir, 'proc'), '-t', 'proc'
])
run([
'mount', 'sysfs',
os.path.join(self.chroot_basedir, 'sys'), '-t', 'sysfs'
])
def setup_chroot_basedir(self):
self.logger.debug('Restoring CHROOT_BASEDIR for runs...')
os.makedirs(self.tmpfs_path, exist_ok=True)
if self.tmpfs:
run([
'mount', '-t', 'tmpfs', '-o', f'size={self.tmpfs_size}G',
'tmpfs', self.tmpfs_path
])
PackageBootstrapDirectory().restore_cache(self.chroot_base_directory)
def sign_manifest(signing_key, signing_pass):
run(
f'echo "{signing_pass}" | gpg -ab --batch --yes --no-use-agent --pinentry-mode loopback --passphrase-fd 0 '
f'--default-key {signing_key} --output {os.path.join(UPDATE_DIR, "MANIFEST.sig")} '
f'--sign {os.path.join(UPDATE_DIR, "MANIFEST")}',
shell=True,
exception_msg='Failed gpg signing with SIGNING_PASSWORD',
log=False,
)
def run_in_chroot(self, command, exception_message=None):
run(
f'chroot {self.dpkg_overlay} /bin/bash -c "{command}"',
shell=True,
exception_msg=exception_message,
env={
**os.environ,
**APT_ENV,
'CONFIG_DEBUG_INFO': 'Y', # Build kernel with debug symbols
'CONFIG_LOCALVERSION': '+truenas',
})
def get_image_version():
if not os.path.exists(UPDATE_FILE):
raise CallError(f'{UPDATE_FILE!r} update file does not exist')
with tempfile.TemporaryDirectory() as mount_dir:
try:
run(['mount', UPDATE_FILE, mount_dir, '-t', 'squashfs', '-o', 'loop'], log=False)
with open(os.path.join(mount_dir, 'manifest.json'), 'r') as f:
update_manifest = json.loads(f.read())
return update_manifest['version']
finally:
run(['umount', '-f', mount_dir], log=False)
def setup_impl(self):
if self.mirror_cache_intact:
# Mirror cache is intact, we do not need to re-create the bootstrap directory
self.logger.debug(
'Basechroot cache is intact and does not need to be changed')
return
self.add_trusted_apt_key()
manifest = get_manifest()
apt_repos = manifest['apt-repos']
run(['debootstrap'] + self.deopts + [
'--keyring',
'/etc/apt/trusted.gpg.d/debian-archive-truenas-automatic.gpg',
manifest['debian_release'], self.chroot_basedir, apt_repos['url']
])
self.setup_mounts()
if self.extra_packages_to_install:
run(['chroot', self.chroot_basedir, 'apt', 'install', '-y'] +
self.extra_packages_to_install)
installed_packages = self.get_packages()
self.after_extra_packages_installation_steps()
# Save the correct repo in sources.list
apt_path = os.path.join(self.chroot_basedir, 'etc/apt')
apt_sources_path = os.path.join(apt_path, 'sources.list')
apt_sources = [
f'deb {apt_repos["url"]} {apt_repos["distribution"]} {apt_repos["components"]}'
]
# Set bullseye repo as the priority
with open(os.path.join(apt_path, 'preferences'), 'w') as f:
f.write(get_apt_preferences())
# Add additional repos
for repo in apt_repos['additional']:
self.logger.debug('Adding additional repo: %r', repo['url'])
shutil.copy(os.path.join(BUILDER_DIR, repo['key']),
os.path.join(self.chroot_basedir, 'apt.key'))
run(['chroot', self.chroot_basedir, 'apt-key', 'add', '/apt.key'])
os.unlink(os.path.join(self.chroot_basedir, 'apt.key'))
apt_sources.append(
f'deb {repo["url"]} {repo["distribution"]} {repo["component"]}'
)
with open(apt_sources_path, 'w') as f:
f.write('\n'.join(apt_sources))
# Update apt
run(['chroot', self.chroot_basedir, 'apt', 'update'])
# Put our local package up at the top of the food chain
apt_sources.insert(0, 'deb [trusted=yes] file:/packages /')
with open(apt_sources_path, 'w') as f:
f.write('\n'.join(apt_sources))
self.clean_mounts()
self.save_build_cache(installed_packages)
def delete_overlayfs(self):
for command in (
['umount', '-f', os.path.join(self.dpkg_overlay, 'proc')],
['umount', '-f', os.path.join(self.dpkg_overlay, 'sys')],
['umount', '-f', self.dpkg_overlay],
['umount', '-R', '-f', self.dpkg_overlay],
['umount', '-R', '-f', self.tmpfs_path],
):
run(command, check=False)
for path in filter(os.path.exists, (
self.chroot_overlay, self.dpkg_overlay, self.workdir_overlay, self.chroot_base_directory,
self.sources_overlay, self.tmpfs_path
)):
shutil.rmtree(path)
def binary_packages(self):
if self._binary_packages:
return self._binary_packages
if self.name == 'kernel' or (self.predepscmd and not self.deps_path):
# We cannot determine dependency of this package because it does not probably have a control file
# in it's current state - the only example we have is grub right now. Let's improve this if there are
# more examples
self._binary_packages.append(
BinaryPackage(self.name, self.build_depends, self.name,
self.name, set()))
return self._binary_packages
cp = run([DEPENDS_SCRIPT_PATH, self.debian_control_file_path],
log=False)
info = json.loads(cp.stdout)
default_dependencies = {'kernel'} if self.kernel_module else set()
self.build_depends = set(
normalize_build_depends(info['source_package']
['build_depends'])) | default_dependencies
self.source_package = info['source_package']['name']
for bin_package in info['binary_packages']:
self._binary_packages.append(
BinaryPackage(
bin_package['name'], self.build_depends,
self.source_package, self.name,
set(
normalize_bin_packages_depends(bin_package['depends']
or ''))))
if self.name == 'truenas':
self._binary_packages[
-1].build_dependencies |= self._binary_packages[
-1].install_dependencies
return self._binary_packages
def setup_chroot_basedir(bootstrapdir_obj):
if os.path.exists(CHROOT_BASEDIR):
shutil.rmtree(CHROOT_BASEDIR)
os.makedirs(TMPFS, exist_ok=True)
run(['mount', '-t', 'tmpfs', '-o', 'size=12G', 'tmpfs', TMPFS])
bootstrapdir_obj.restore_cache(CHROOT_BASEDIR)
run(['mount', 'proc', os.path.join(CHROOT_BASEDIR, 'proc'), '-t', 'proc'])
run(['mount', 'sysfs', os.path.join(CHROOT_BASEDIR, 'sys'), '-t', 'sysfs'])
os.makedirs(PACKAGE_PATH, exist_ok=True)
run(['mount', '--bind', PKG_DIR, PACKAGE_PATH])
def get_packages(self):
return {
e[0]: {
'version': e[1],
'architecture': e[2]
}
for e in INSTALLED_PACKAGES_REGEX.findall(
run([
'chroot', self.chroot_basedir, 'dpkg-query', '-W', '-f',
'${Package}\t${Version}\t${Architecture}\n'
],
log=False).stdout)
}
def make_overlayfs(self):
for path in (self.chroot_overlay, self.dpkg_overlay, self.sources_overlay, self.workdir_overlay):
os.makedirs(path, exist_ok=True)
for entry in (
([
'mount', '-t', 'overlay', '-o',
f'lowerdir={self.chroot_base_directory},upperdir={self.chroot_overlay},workdir={self.workdir_overlay}',
'none', f'{self.dpkg_overlay}/'
], 'Failed overlayfs'),
(['mount', 'proc', os.path.join(self.dpkg_overlay, 'proc'), '-t', 'proc'], 'Failed mount proc'),
(['mount', 'sysfs', os.path.join(self.dpkg_overlay, 'sys'), '-t', 'sysfs'], 'Failed mount sysfs'),
(
['mount', '--bind', self.sources_overlay, self.source_in_chroot],
'Failed mount --bind /dpkg-src', self.source_in_chroot
)
):
if len(entry) == 2:
command, msg = entry
else:
command, msg, path = entry
os.makedirs(path, exist_ok=True)
run(command, exception_msg=msg)
def build_rootfs_image():
for f in glob.glob(os.path.join('./tmp/release', '*.update*')):
os.unlink(f)
if os.path.exists(UPDATE_DIR):
shutil.rmtree(UPDATE_DIR)
os.makedirs(RELEASE_DIR, exist_ok=True)
os.makedirs(UPDATE_DIR)
# We are going to build a nested squashfs image.
# Why nested? So that during update we can easily RO mount the outer image
# to read a MANIFEST and verify signatures of the real rootfs inner image
#
# This allows us to verify without ever extracting anything to disk
# Create the inner image
run([
'mksquashfs', CHROOT_BASEDIR,
os.path.join(UPDATE_DIR, 'rootfs.squashfs'), '-comp', 'xz'
])
# Build any MANIFEST information
build_manifest()
# Sign the image (if enabled)
if SIGNING_KEY and SIGNING_PASSWORD:
sign_manifest(SIGNING_KEY, SIGNING_PASSWORD)
# Create the outer image now
run(['mksquashfs', UPDATE_DIR, UPDATE_FILE, '-noD'])
update_hash = run(['sha256sum', UPDATE_FILE],
log=False).stdout.strip().split()[0]
with open(UPDATE_FILE_HASH, 'w') as f:
f.write(update_hash)
build_update_manifest(update_hash)
def hash_changed(self):
if self.name == 'truenas':
# truenas is special and we want to rebuild it always
# TODO: Do see why that is so
return True
source_hash = self.source_hash
existing_hash = None
if os.path.exists(self.hash_path):
with open(self.hash_path, 'r') as f:
existing_hash = f.read().strip()
if source_hash == existing_hash:
return run(
['git', '-C', self.source_path, 'diff-files', '--quiet', '--ignore-submodules'], check=False, log=False
).returncode != 0
else:
return True
def save_build_cache(self, installed_packages):
self.logger.debug('Caching CHROOT_BASEDIR for future runs...')
run(['mksquashfs', self.chroot_basedir, self.cache_file_path])
self.update_saved_packages_list(installed_packages)
self.update_mirror_cache()
def umount_tmpfs_and_clean_chroot_dir():
if os.path.exists(CHROOT_BASEDIR):
shutil.rmtree(CHROOT_BASEDIR)
run(['umount', '-f', TMPFS], check=False, log=False)
def checkout(self, branch_override=None):
origin_url = self.retrieve_current_remote_origin_and_sha()['url']
branch = branch_override or self.branch
if branch == self.existing_branch and self.origin == origin_url:
logger.debug('Updating git repo [%s] (%s)', self.name,
GIT_LOG_PATH)
with LoggingContext('git-checkout', 'w'):
run(['git', '-C', self.source_path, 'fetch', '--unshallow'],
check=False)
run(['git', '-C', self.source_path, 'fetch', 'origin', branch])
run([
'git', '-C', self.source_path, 'reset', '--hard',
f'origin/{branch}'
])
run([
'git', '-C', self.source_path, 'submodule', 'update',
'--checkout', '--depth=1'
])
else:
logger.debug('Checking out git repo [%s] (%s)', self.name,
GIT_LOG_PATH)
if os.path.exists(self.source_path):
shutil.rmtree(self.source_path)
with LoggingContext('git-checkout', 'w'):
run([
'git', 'clone', '--depth=1', '-b', branch, self.origin,
self.source_path
])
run([
'git', '-C', self.source_path, 'submodule', 'update',
'--init', '--checkout', '--depth=1'
])
self.update_git_manifest()
def source_hash(self):
return run(
['git', '-C', self.source_path, 'rev-parse', '--verify', 'HEAD'],
log=False).stdout.strip()
def restore_cache(self, chroot_basedir):
run(['unsquashfs', '-f', '-d', chroot_basedir, self.cache_file_path])
def make_iso_file():
for f in glob.glob(os.path.join(RELEASE_DIR, '*.iso*')):
os.unlink(f)
# Set default PW to root
run(fr'chroot {CHROOT_BASEDIR} /bin/bash -c "echo -e \"root\nroot\" | passwd root"',
shell=True)
# Create /etc/version
with open(os.path.join(CHROOT_BASEDIR, 'etc/version'), 'w') as f:
f.write(get_image_version())
# Set /etc/hostname so that hostname of builder is not advertised
with open(os.path.join(CHROOT_BASEDIR, 'etc/hostname'), 'w') as f:
f.write('truenas.local')
# Copy the CD files
distutils.dir_util._path_created = {}
distutils.dir_util.copy_tree(CD_FILES_DIR,
CHROOT_BASEDIR,
preserve_symlinks=True)
# Create the CD assembly dir
if os.path.exists(CD_DIR):
shutil.rmtree(CD_DIR)
os.makedirs(CD_DIR, exist_ok=True)
# Prune away the fat
prune_cd_basedir()
# Lets make squashfs now
tmp_truenas_path = os.path.join(TMP_DIR, 'truenas.squashfs')
run(['mksquashfs', CHROOT_BASEDIR, tmp_truenas_path, '-comp', 'xz'])
os.makedirs(os.path.join(CD_DIR, 'live'), exist_ok=True)
shutil.move(tmp_truenas_path,
os.path.join(CD_DIR, 'live/filesystem.squashfs'))
# Copy over boot and kernel before rolling CD
shutil.copytree(os.path.join(CHROOT_BASEDIR, 'boot'),
os.path.join(CD_DIR, 'boot'))
# Dereference /initrd.img and /vmlinuz so this ISO can be re-written to a FAT32 USB stick using Windows tools
shutil.copy(os.path.join(CHROOT_BASEDIR, 'initrd.img'), CD_DIR)
shutil.copy(os.path.join(CHROOT_BASEDIR, 'vmlinuz'), CD_DIR)
for f in itertools.chain(
glob.glob(os.path.join(CD_DIR, 'boot/initrd.img-*')),
glob.glob(os.path.join(CD_DIR, 'boot/vmlinuz-*')),
):
os.unlink(f)
shutil.copy(UPDATE_FILE, os.path.join(CD_DIR, 'TrueNAS-SCALE.update'))
os.makedirs(os.path.join(CHROOT_BASEDIR, RELEASE_DIR), exist_ok=True)
os.makedirs(os.path.join(CHROOT_BASEDIR, CD_DIR), exist_ok=True)
try:
run([
'mount', '--bind', RELEASE_DIR,
os.path.join(CHROOT_BASEDIR, RELEASE_DIR)
])
run(['mount', '--bind', CD_DIR, os.path.join(CHROOT_BASEDIR, CD_DIR)])
run_in_chroot(['apt-get', 'update'], check=False)
run_in_chroot([
'apt-get', 'install', '-y', 'grub-common', 'grub2-common',
'grub-efi-amd64-bin', 'grub-efi-amd64-signed', 'grub-pc-bin',
'mtools', 'xorriso'
])
run_in_chroot([
'grub-mkrescue', '-o',
os.path.join(RELEASE_DIR,
f'TrueNAS-SCALE-{get_image_version()}.iso'), CD_DIR
])
finally:
run(['umount', '-f', os.path.join(CHROOT_BASEDIR, CD_DIR)])
run(['umount', '-f', os.path.join(CHROOT_BASEDIR, RELEASE_DIR)])
with open(
os.path.join(RELEASE_DIR,
f'TrueNAS-SCALE-{get_image_version()}.iso.sha256'),
'w') as f:
f.write(
run([
'sha256sum',
os.path.join(RELEASE_DIR,
f'TrueNAS-SCALE-{get_image_version()}.iso')
],
log=False).stdout.strip().split()[0])
def run_in_chroot(command, exception_message=None, **kwargs):
return run(
['chroot', CHROOT_BASEDIR] + command, exception_msg=exception_message, env={**APT_ENV, **os.environ}, **kwargs
)