def hashit(self):
""" Generate various hash values. We hijack the 'logfile' which will
actually be the file containing the hashes.
"""
# We need to be in the parent of the system's portage configroot because
# that's where we created the above tarball. This should be the workdir,
# but its probably safer to be pedantic here.
cwd = os.getcwd()
os.chdir(os.path.join(self.portage_configroot, '..'))
# Note: this first cmd clobbers the contents
cmd = 'echo "# MD5 HASH"'
Execute(cmd, logfile=self.digest_name)
cmd = 'md5sum %s' % self.medium_name
Execute(cmd, timeout=60, logfile=self.digest_name)
cmd = 'echo "# SHA1 HASH"'
Execute(cmd, logfile=self.digest_name)
cmd = 'sha1sum %s' % self.medium_name
Execute(cmd, timeout=60, logfile=self.digest_name)
cmd = 'echo "# SHA512 HASH"'
Execute(cmd, logfile=self.digest_name)
cmd = 'sha512sum %s' % self.medium_name
Execute(cmd, timeout=60, logfile=self.digest_name)
cmd = 'echo "# WHIRLPOOL HASH"'
Execute(cmd, logfile=self.digest_name)
cmd = 'whirlpooldeep %s' % self.medium_name
Execute(cmd, timeout=60, logfile=self.digest_name)
os.chdir(cwd)
def isoit(self, alt_name=None):
# Create the ISO with the default name unless an alt_name is given.
if alt_name:
self.medium_name = '%s-%s%s%s.iso' % (alt_name, self.year,
self.month, self.day)
self.digest_name = '%s.DIGESTS' % self.medium_name
iso_dir = os.path.join(self.tmpdir, 'iso')
isoboot_dir = os.path.join(iso_dir, 'boot')
isogrub_dir = os.path.join(isoboot_dir, 'grub')
shutil.rmtree(iso_dir, ignore_errors=True)
os.makedirs(isogrub_dir, mode=0o755, exist_ok=False)
# 1. build initramfs image and copy it in
self.initramfs(isoboot_dir)
# 2. Move the kernel image into the iso/boot directory.
# TODO: we are assuming a static kernel
kernelimage_dir = os.path.join(self.portage_configroot, 'boot')
kernelimage_path = os.path.join(kernelimage_dir, 'kernel')
shutil.copy(kernelimage_path, isoboot_dir)
# If this fails, we'll have to rebuild the kernel!
#shutil.rmtree(kernelimage_dir, ignore_errors=True)
# 3. Make the squashfs image and copy it into the iso/boot.
# This can take a long time.
squashfs_path = os.path.join(iso_dir, 'rootfs')
cmd = 'mksquashfs %s %s -xattrs -comp xz' % (self.portage_configroot,
squashfs_path)
Execute(cmd, timeout=None, logfile=self.logfile)
# 4. Emerge grub:0 to grab stage2_eltorito
grub_root = os.path.join(self.tmpdir, 'grub')
eltorito_path = os.path.join(grub_root, 'boot/grub/stage2_eltorito')
menulst_path = os.path.join(self.libdir, 'scripts/menu.lst')
cmd = 'emerge --nodeps -1q grub:0'
emerge_env = {'USE': '-* savedconfig', 'ROOT': grub_root}
Execute(cmd, timeout=600, extra_env=emerge_env, logfile=self.logfile)
shutil.copy(eltorito_path, isogrub_dir)
shutil.copy(menulst_path, isogrub_dir)
# 5. Create the iso image. This can take a long time.
args = '-R ' # Rock Ridge protocol
args += '-b boot/grub/stage2_eltorito ' # El Torito boot image
args += '-no-emul-boot ' # No disk emulation for El Torito
args += '-boot-load-size 4 ' # 4x512-bit sectors for no-emulation mode
args += '-boot-info-table ' # Create El Torito boot info table
medium_path = os.path.join(self.tmpdir, self.medium_name)
cmd = 'mkisofs %s -o %s %s' % (args, medium_path, iso_dir)
Execute(cmd, timeout=None, logfile=self.logfile)
def sync(self):
if self.isgitdir():
# If the local repo exists, then make it pristine an pull
cmd = 'git -C %s reset HEAD --hard' % self.local_repo
Execute(cmd, timeout=60, logfile=self.logfile)
cmd = 'git -C %s clean -f -x -d' % self.local_repo
Execute(cmd, timeout=60, logfile=self.logfile)
cmd = 'git -C %s pull' % self.local_repo
Execute(cmd, timeout=60, logfile=self.logfile)
else:
# else clone afresh.
cmd = 'git clone %s %s' % (self.remote_repo, self.local_repo)
Execute(cmd, timeout=60, logfile=self.logfile)
# Make sure we're on the correct branch for the desired GRS system.
cmd = 'git -C %s checkout %s' % (self.local_repo, self.branch)
Execute(cmd, timeout=60, logfile=self.logfile)
def populate(self, cycle=True):
""" Copy the core files from the GRS repo, to a local workdir and
then to the system's portage configroot, selecting for a paricular
cycle number.
"""
# rsync from the GRS repo to the workdir, removing the .git directory
cmd = 'rsync -av --delete --exclude=\'.git*\' %s/core/ %s' % (
self.libdir, self.workdir)
Execute(cmd, timeout=60, logfile=self.logfile)
# Select the cycle
if cycle:
self.select_cycle(cycle)
# Copy from the workdir to the system's portage configroot.
cmd = 'rsync -av %s/ %s' % (self.workdir, self.portage_configroot)
Execute(cmd, timeout=60, logfile=self.logfile)
def tarit(self, alt_name=None):
# Create the tarball with the default name unless an alt_name is given.
if alt_name:
self.medium_name = '%s.tar.xz' % alt_name
self.digest_name = '%s.DIGESTS' % self.medium_name
# We have to cd into the system's portage configroot and then out again.
cwd = os.getcwd()
os.chdir(self.portage_configroot)
tarball_path = os.path.join('..', self.medium_name)
# TODO: This needs to be generalized for systems that don't support xattrs
xattr_opts = '--xattrs --xattrs-include=security.capability --xattrs-include=user.pax.flags'
cmd = 'tar %s -Jcf %s .' % (xattr_opts, tarball_path)
Execute(cmd, timeout=None, logfile=self.logfile)
os.chdir(cwd)
def runscript(self, script_name):
# Copy the script form the GRS repo to the system's portage configroot's /tmp.
# Don't add a suffix to the script since we will admit bash, python, etc.
script_org = os.path.join(self.libdir, 'scripts/%s' % script_name)
script_dst = os.path.join(self.portage_configroot, 'tmp/script')
shutil.copy(script_org, script_dst)
# Mark the script as excutable and execute it.
os.chmod(script_dst, 0o0755)
cmd = 'chroot %s /tmp/script' % self.portage_configroot
Execute(cmd, timeout=None, logfile=self.logfile)
# In the case of a clean script, it can delete itself, so
# check if the script exists before trying to delete it.
if os.path.isfile(script_dst):
os.unlink(script_dst)
def seed(self):
# Rotate the old portage_configroot and package out of the way
for directory in [self.portage_configroot, self.package]:
self.full_rotate(directory)
os.makedirs(directory, mode=0o755, exist_ok=False)
# Download a stage tarball if we don't have one
if not os.path.isfile(self.filepath):
request = urllib.request.urlopen(self.stage_uri)
with open(self.filepath, 'wb') as _file:
shutil.copyfileobj(request, _file)
# Because python's tarfile sucks
cmd = 'tar --xattrs -xf %s -C %s' % (self.filepath, self.portage_configroot)
Execute(cmd, timeout=120, logfile=self.logfile)
def umount_all(self):
""" Unmount all the self.directories[]. """
# We must unmount in the opposite order that we mounted.
for mount in self.rev_directories:
if isinstance(mount, str):
target_directory = mount
elif isinstance(mount, list):
target_directory = mount[1]
elif isinstance(mount, dict):
tmp = list(mount.keys())
target_directory = tmp[0]
target_directory = os.path.join(self.portage_configroot,
target_directory)
if self.ismounted(target_directory):
cmd = 'umount --force %s' % target_directory
Execute(cmd, timeout=60, logfile=self.logfile)
def mount_all(self):
""" Mount all the self.directories[] under the system's portage configroot. """
# If any are mounted, let's first unmount all, then mount all
some_mounted, all_mounted = self.are_mounted()
if some_mounted:
self.umount_all()
# Now go through each of the self.directories[] to be mounted in order.
for mount in self.directories:
if isinstance(mount, str):
# In this case, the source_directory is assumed to exist relative to /
# and we will just bind mount it in the system's portage configroot.
source_directory = mount
target_directory = mount
elif isinstance(mount, list):
# In this case, the source_directory is assumed to be an abspath, and
# we create it if it doesn't already exist.
source_directory = mount[0]
os.makedirs(source_directory, mode=0o755, exist_ok=True)
target_directory = mount[1]
elif isinstance(mount, dict):
# In this case, we are given the mountpoint, type and name,
# so we just go right ahead and mount -t type name mountpoint.
# This is useful for tmpfs filesystems.
tmp = list(mount.values())
tmp = tmp[0]
vfstype = tmp[0]
vfsname = tmp[1]
tmp = list(mount.keys())
target_directory = tmp[0]
# Let's make sure the target_directory exists.
target_directory = os.path.join(self.portage_configroot,
target_directory)
os.makedirs(target_directory, mode=0o755, exist_ok=True)
# Okay now we're ready to do the actual mounting.
if isinstance(mount, str):
cmd = 'mount --bind /%s %s' % (source_directory,
target_directory)
elif isinstance(mount, list):
cmd = 'mount --bind %s %s' % (source_directory,
target_directory)
elif isinstance(mount, dict):
cmd = 'mount -t %s %s %s' % (vfstype, vfsname,
target_directory)
Execute(cmd, timeout=60, logfile=self.logfile)
def sync(self):
if self.isgitdir():
# If the local repo exists, then make it pristine an pull
cmd = 'git -C %s reset HEAD --hard' % self.local_repo
Execute(cmd, timeout=60, logfile=self.logfile)
cmd = 'git -C %s clean -f -x -d' % self.local_repo
Execute(cmd, timeout=60, logfile=self.logfile)
cmd = 'git -C %s pull' % self.local_repo
Execute(cmd, timeout=60, logfile=self.logfile)
else:
# else clone afresh.
cmd = 'git clone %s %s' % (self.remote_repo, self.local_repo)
Execute(cmd, timeout=60, logfile=self.logfile)
# Make sure we're on the correct branch for the desired GRS system.
cmd = 'git -C %s checkout %s' % (self.local_repo, self.branch)
Execute(cmd, timeout=60, logfile=self.logfile)
# If there is a .gitmodules, then init/update the submodules
git_modulesfile = os.path.join(self.local_repo, '.gitmodules')
if os.path.isfile(git_modulesfile):
# Recursively update any submodules following the remote branch
cmd = 'git -C %s submodule update --init --recursive --remote' % self.local_repo
Execute(cmd, timeout=60, logfile=self.logfile)
def initramfs(self, isoboot_dir):
""" TODO """
# Paths to where we'll build busybox and the initramfs.
busybox_root = os.path.join(self.tmpdir, 'busybox')
busybox_path = os.path.join(busybox_root, 'bin/busybox')
makeprofile_path = os.path.join(busybox_root,
'etc/portage/make.profile')
savedconfig_dir = os.path.join(busybox_root,
'etc/portage/savedconfig/sys-apps')
savedconfig_path = os.path.join(savedconfig_dir, 'busybox')
busybox_config = os.path.join(self.libdir, 'scripts/busybox-config')
# Remove any old busybox build directory and prepare new one.
shutil.rmtree(busybox_root, ignore_errors=True)
os.makedirs(savedconfig_dir, mode=0o755, exist_ok=True)
shutil.copy(busybox_config, savedconfig_path)
# Emerge busybox.
os.symlink('/usr/portage/profiles/hardened/linux/amd64',
makeprofile_path)
cmd = 'emerge --nodeps -1q busybox'
emerge_env = {
'USE': '-* savedconfig',
'ROOT': busybox_root,
'PORTAGE_CONFIGROOT': busybox_root
}
Execute(cmd, timeout=600, extra_env=emerge_env, logfile=self.logfile)
# Remove any old initramfs root and prepare a new one.
initramfs_root = os.path.join(self.tmpdir, 'initramfs')
shutil.rmtree(initramfs_root, ignore_errors=True)
root_paths = [
'bin', 'dev', 'etc', 'mnt/cdrom', 'mnt/squashfs', 'mnt/tmpfs',
'proc', 'sbin', 'sys', 'tmp', 'usr/bin', 'usr/sbin', 'var',
'var/run'
]
for _path in root_paths:
_dir = os.path.join(initramfs_root, _path)
os.makedirs(_dir, mode=0o755, exist_ok=True)
# Copy the static busybox to the initramfs root.
# TODO: we are assuming a static busybox, so we should check.
busybox_initramfs_path = os.path.join(initramfs_root, 'bin/busybox')
shutil.copy(busybox_path, busybox_initramfs_path)
os.chmod(busybox_initramfs_path, 0o0755)
cmd = 'chroot %s /bin/busybox --install -s' % initramfs_root
Execute(cmd, timeout=60, logfile=self.logfile)
# Copy the init script to the initramfs root.
initscript_path = os.path.join(self.libdir, 'scripts/initramfs-init')
init_initramfs_path = os.path.join(initramfs_root, 'init')
shutil.copy(initscript_path, init_initramfs_path)
os.chmod(init_initramfs_path, 0o0755)
# TODO: we are assuming a static kernel and so not copying in
# any modules. This is where we should copy in modules.
# cpio-gzip the initramfs root to the iso boot dir
initramfs_path = os.path.join(isoboot_dir, 'initramfs')
cwd = os.getcwd()
os.chdir(initramfs_root)
cmd = 'find . -print | cpio -H newc -o | gzip -9 > %s' % initramfs_path
# Piped commands must be run in a shell.
Execute(cmd, timeout=600, logfile=self.logfile, shell=True)
os.chdir(cwd)
def kernel(self, arch='x86_64'):
""" This emerges the kernel sources to a directory outside of the
fledgeling system's portage configroot, builds and installs it
to yet another external directory, bundles the kernel and modules
as a .tar.xz in the packages directory for downloads via grsup,
and finally installs it to the system's portage configroot.
"""
# Grab the parsed verison and pkg atom.
(gentoo_version, pkg_name, has_modules) = self.parse_kernel_config()
# Prepare the paths to where we'll emerge and build the kernel,
# as well as paths for genkernel.
kernel_source = os.path.join(self.kernelroot, 'usr/src/linux')
image_dir = os.path.join(self.kernelroot, gentoo_version)
boot_dir = os.path.join(image_dir, 'boot')
modprobe_dir = os.path.join(image_dir, 'etc/modprobe.d')
modules_dir = os.path.join(image_dir, 'lib/modules')
# The firmware directory, if it exists, will be in self.portage_configroot
firmware_dir = os.path.join(self.portage_configroot, 'lib/firmware')
# Prepare tarball filename and path. If the tarball already exists,
# don't rebuild/reinstall it. Note: It should have been installed to
# the system's portage configroot when it was first built, so no need
# to reinstall it.
linux_images = os.path.join(self.package, 'linux-images')
tarball_name = 'linux-image-%s.tar.xz' % gentoo_version
tarball_path = os.path.join(linux_images, tarball_name)
if os.path.isfile(tarball_path):
return
# Remove any old kernel image directory and create a boot directory.
# Note genkernel assumes a boot directory is present.
shutil.rmtree(image_dir, ignore_errors=True)
os.makedirs(boot_dir, mode=0o755, exist_ok=True)
# emerge the kernel source.
cmd = 'emerge --nodeps -1n %s' % pkg_name
emerge_env = {
'USE': 'symlink',
'ROOT': self.kernelroot,
'ACCEPT_KEYWORDS': '**'
}
Execute(cmd, timeout=600, extra_env=emerge_env, logfile=self.logfile)
# Build and install the image outside the portage configroot so
# we can both rsync it in *and* tarball it for downloads via grsup.
# NOTE: more options (eg splash and firmware), can be specified
# via the kernel line in the build script.
cmd = 'genkernel '
cmd += '--logfile=/dev/null '
cmd += '--no-save-config '
cmd += '--makeopts=-j9 '
cmd += '--symlink '
cmd += '--no-mountboot '
cmd += '--kernel-config=%s ' % self.kernel_config
cmd += '--kerneldir=%s ' % kernel_source
cmd += '--bootdir=%s ' % boot_dir
cmd += '--module-prefix=%s ' % image_dir
cmd += '--modprobedir=%s ' % modprobe_dir
cmd += '--arch-override=%s ' % arch
if os.path.isfile(self.busybox_config):
cmd += '--busybox-config=%s ' % self.busybox_config
if os.path.isfile(self.genkernel_config):
cmd += '--config=%s ' % self.genkernel_config
if os.path.isdir(firmware_dir):
cmd += '--firmware-dir=%s ' % firmware_dir
if has_modules:
cmd += 'all'
else:
cmd += 'bzImage'
Execute(cmd, timeout=None, logfile=self.logfile)
# Strip the modules to shrink their size enormously!
# This will do nothing if there is not modules_dir
for dirpath, dirnames, filenames in os.walk(modules_dir):
for filename in filenames:
if filename.endswith('.ko'):
module = os.path.join(dirpath, filename)
cmd = 'objcopy -v --strip-unneeded %s' % module
Execute(cmd)
# Copy the newly compiled kernel image and modules to portage configroot
cmd = 'rsync -aK %s/ %s' % (image_dir, self.portage_configroot)
Execute(cmd, timeout=60, logfile=self.logfile)
# Tar up the kernel image and modules and place them in package/linux-images
os.makedirs(linux_images, mode=0o755, exist_ok=True)
cwd = os.getcwd()
os.chdir(image_dir)
cmd = 'tar -Jcf %s .' % tarball_path
Execute(cmd, timeout=600, logfile=self.logfile)
os.chdir(cwd)