def gen_fstab(self):
rootdir = self.devices['rootdir']
if not rootdir:
raise cliapp.AppException("rootdir not set")
# https://bbs.archlinux.org/viewtopic.php?id=220215 and
# https://stackoverflow.com/questions/36379789/python-subprocess-unable-to-escape-quotes
runcmd("bash -c 'genfstab -U %s >> %s/etc/fstab'" % (rootdir, rootdir),
shell=True)
def run_extlinux_install(self, rootdir):
if os.path.exists("/usr/bin/extlinux"):
self.message('Running extlinux --install')
runcmd(['extlinux', '--install', rootdir])
runcmd(['sync'])
time.sleep(2)
else:
msg = "extlinux enabled but /usr/bin/extlinux not found" \
" - please install the extlinux package."
raise cliapp.AppException(msg)
def install_mbr(self):
if not self.settings['mbr'] or not self.settings['extlinux']:
return
if os.path.exists("/sbin/install-mbr"):
self.message('Installing MBR')
runcmd(['install-mbr', self.settings['image']])
else:
msg = "mbr enabled but /sbin/install-mbr not found" \
" - please install the mbr package."
raise cliapp.AppException(msg)
def enable_systemd_resolved(self, rootdir):
"""
only for unstable or testing, not present in jessie
"""
self.message('Enabling systemctl-resolved for DNS')
runcmd(['chroot', rootdir, 'systemctl', 'enable', 'systemd-resolved'])
runcmd([
'chroot', rootdir, 'ln', '-sfT',
'/run/systemd/resolve/resolv.conf', '/etc/resolv.conf'
])
def mask_udev_predictable_rules(self, rootdir):
"""
This can be reset later but to get networking using eth0
immediately upon boot, the interface we're going to use must
be known and must update the initramfs after setting up the
mask.
"""
self.message('Disabling systemd predictable interface names')
udev_path = os.path.join('etc', 'udev', 'rules.d',
'80-net-setup-link.rules')
runcmd(['chroot', rootdir, 'ln', '-s', '/dev/null', udev_path])
def set_time_zone(self):
rootdir = self.devices['rootdir']
region = self.settings['region']
city = self.settings['city']
if not rootdir:
raise cliapp.AppException("rootdir not set")
args = "chroot %s ln -sf /usr/share/zoneinfo/%s/%s /etc/localtime" % (
rootdir, region, city)
if self.is_arm():
self.arm_chroot(rootdir, args, shell=True)
else:
runcmd(args, shell=True)
def chown(self):
if not self.settings['owner']:
return
# Change image owner after completed build
if self.settings['image']:
filename = self.settings['image']
elif self.settings['tarball']:
filename = self.settings['tarball']
elif self.settings['squash']:
filename = self.settings['squash']
else:
return
self.message("Changing owner to %s" % self.settings["owner"])
runcmd(["chown", "-R", self.settings["owner"], filename])
def setup_kpartx(self):
bootindex = None
swapindex = None
out = runcmd(['kpartx', '-avs', self.settings['image']])
if self.settings['bootsize'] and self.settings['swap'] > 0:
bootindex = 0
rootindex = 1
swapindex = 2
parts = 3
elif self.settings['use-uefi']:
bootindex = 0
rootindex = 1
parts = 2
elif self.settings['use-uefi'] and self.settings['swap'] > 0:
bootindex = 0
rootindex = 1
swapindex = 2
parts = 3
elif self.settings['bootsize']:
bootindex = 0
rootindex = 1
parts = 2
elif self.settings['swap'] > 0:
rootindex = 0
swapindex = 1
parts = 2
else:
rootindex = 0
parts = 1
boot = None
swap = None
devices = [
line.decode('utf-8').split()[2] for line in out.splitlines()
if line.decode('utf-8').startswith('add map ')
]
if len(devices) != parts:
msg = 'Surprising number of partitions %d:%d- check output of losetup -a' % (
len(devices), parts)
logging.debug("%s", runcmd(['losetup', '-a']))
logging.debug("%s: devices=%s parts=%s", msg, devices, parts)
raise cliapp.AppException(msg)
root = '/dev/mapper/%s' % devices[rootindex]
if self.settings['bootsize'] or self.settings['use-uefi']:
boot = '/dev/mapper/%s' % devices[bootindex].decode('utf-8')
if self.settings['swap'] > 0:
swap = '/dev/mapper/%s' % devices[swapindex]
self.devices['rootdev'] = root
self.devices['bootdev'] = boot
self.devices['swap'] = swap
def set_localization(self):
rootdir = self.devices['rootdir']
locale = self.settings['locale']
lang = self.settings['lang']
locale_gen_template = self.env.get_template("locale.gen.j2")
etc_locale_gen = os.path.join(str(rootdir), 'etc', 'locale.gen')
locale_gen_template.stream(settings_locale=locale).dump(etc_locale_gen)
args = ['chroot', rootdir, 'locale-gen']
if self.is_arm():
self.arm_chroot(rootdir, args)
else:
runcmd(args)
locale_conf_template = self.env.get_template("locale.conf.j2")
etc_locale_conf = os.path.join(str(rootdir), 'etc', 'locale.conf')
locale_conf_template.stream(settings_lang=lang).dump(etc_locale_conf)
def convert_image_to_qcow2(self):
"""
Current images are all prepared as raw
rename to .raw and let the conversion put the
original name back
"""
if not self.settings['convert-qcow2'] or not self.settings['image']:
return
self.message('Converting raw image to qcow2')
tmpname = self.settings['image'] + '.raw'
os.rename(self.settings['image'], tmpname)
runcmd([
'qemu-img', 'convert', '-O', 'qcow2', tmpname,
self.settings['image']
])
def update_initramfs(self):
rootdir = self.devices['rootdir']
if not rootdir:
raise cliapp.AppException("rootdir not set")
if not os.path.exists(
os.path.join(rootdir, 'usr', 'sbin', 'update-initramfs')):
self.message("Error: Unable to run update-initramfs.")
return
if 'no-update-initramfs' in self.settings or not self.settings[
'update-initramfs']:
return
cmd = os.path.join('usr', 'sbin', 'update-initramfs')
if os.path.exists(os.path.join(str(rootdir), cmd)):
self.message("Updating the initramfs")
runcmd(['chroot', rootdir, cmd, '-u'])
def link_uuid(rootdev):
"""
This is mainly to fix a problem in update-grub where /etc/grub.d/10_linux
Checks if the $GRUB_DEVICE_UUID exists in /dev/disk/by-uuid and falls
back to $GRUB_DEVICE if it doesn't.
$GRUB_DEVICE is /dev/mapper/loopXpY (on docker)
Creating the symlink ensures that grub consistently uses
$GRUB_DEVICE_UUID when creating /boot/grub/grub.cfg
"""
if os.path.exists('/.dockerenv'):
logging.info("Running in docker container")
runcmd(['mkdir', '-p', '/dev/disk/by-uuid'])
uuid = runcmd(
['blkid', '-c', '/dev/null', '-o', 'value', '-s', 'UUID', rootdev])
uuid = uuid.splitlines()[0].strip()
os.symlink(rootdev, os.path.join('/dev/disk/by-uuid', uuid))
def enable_systemd_networkd(self, rootdir):
"""
Get networking working immediately on boot, allow any en* interface
to be enabled by systemd-networkd using DHCP
https://coreos.com/os/docs/latest/network-config-with-networkd.html
http://www.freedesktop.org/wiki/Software/systemd/PredictableNetworkInterfaceNames/
"""
self.message('Enabling systemd-networkd for DHCP')
ethpath = os.path.join(rootdir, 'etc', 'systemd', 'network',
'99-dhcp.network')
with open(ethpath, 'w') as eth:
eth.write('[Match]\n')
eth.write('Name=e*\n') # jessie uses eth*, stretch uses ens*
eth.write('\n[Network]\n')
eth.write('DHCP=yes\n')
runcmd(['chroot', rootdir, 'systemctl', 'enable', 'systemd-networkd'])
def unlink_uuid(rootdev):
"""
Reset the link created with link_uuid.
"""
if os.path.exists('/.dockerenv'):
uuid = runcmd(
['blkid', '-c', '/dev/null', '-o', 'value', '-s', 'UUID', rootdev])
uuid = uuid.splitlines()[0].strip()
os.remove(os.path.join('/dev/disk/by-uuid', uuid))
def install_grub_uefi(self, rootdir):
ret = True
self.message("Configuring grub-uefi")
target = arch_table[self.settings['arch']]['target']
grub_opts = "--target=%s" % target
logging.debug("Running grub-install with options: %s", grub_opts)
mount_wrapper(rootdir)
try:
runcmd(['chroot', rootdir, 'update-grub'])
runcmd(['chroot', rootdir, 'grub-install', grub_opts])
except cliapp.AppException as exc:
logging.warning(exc)
ret = False
self.message("Failed to configure grub-uefi for %s" %
self.settings['arch'])
finally:
umount_wrapper(rootdir)
if not ret:
raise cliapp.AppException("Failed to install grub uefi")
def install_extra_grub_uefi(self, rootdir):
ret = True
extra = arch_table[self.settings['arch']]['extra']
if extra:
logging.debug("Installing extra grub support for %s", extra)
mount_wrapper(rootdir)
target = arch_table[extra]['target']
grub_opts = "--target=%s" % target
self.message("Adding grub target %s" % grub_opts)
try:
runcmd(['chroot', rootdir, 'update-grub'])
runcmd(['chroot', rootdir, 'grub-install', grub_opts])
except cliapp.AppException as exc:
logging.warning(exc)
ret = False
self.message("Failed to configure grub-uefi for %s" % extra)
finally:
umount_wrapper(rootdir)
if not ret:
raise cliapp.AppException("Failed to install extra grub uefi")
def install_grub2(self, rootdev, rootdir):
self.message("Configuring grub2")
# rely on kpartx using consistent naming to map loop0p1 to loop0
grub_opts = os.path.join('/dev', os.path.basename(rootdev)[:-2])
if self.settings['serial-console']:
grub_serial_console(rootdir)
logging.debug("Running grub-install with options: %s", grub_opts)
mount_wrapper(rootdir)
link_uuid(rootdev)
try:
runcmd(['chroot', rootdir, 'update-grub'])
runcmd(['chroot', rootdir, 'grub-install', grub_opts])
except cliapp.AppException as exc:
logging.warning(exc)
self.message("Failed. Is grub2-common installed? Using extlinux.")
umount_wrapper(rootdir)
return False
unlink_uuid(rootdev)
umount_wrapper(rootdir)
return True
def install_extlinux(self, rootdev, rootdir):
if not os.path.exists("/usr/bin/extlinux"):
self.message("extlinux not installed, skipping.")
return
self.message('Installing extlinux')
def find(pattern):
dirname = os.path.join(rootdir, 'boot')
basenames = os.listdir(dirname)
logging.debug('find: %s', basenames)
for basename in basenames:
if re.search(pattern, basename):
return os.path.join('boot', basename)
raise cliapp.AppException('Cannot find match: %s' % pattern)
try:
kernel_image = find('vmlinuz-.*')
initrd_image = find('initrd.img-.*')
except cliapp.AppException as exc:
self.message("Unable to find kernel. Not installing extlinux.")
logging.debug("No kernel found. %s. Skipping install of extlinux.", exc)
return
out = runcmd(['blkid', '-c', '/dev/null', '-o', 'value',
'-s', 'UUID', rootdev])
uuid = out.splitlines()[0].strip()
conf = os.path.join(rootdir, 'extlinux.conf')
logging.debug('configure extlinux %s', conf)
kserial = 'console=ttyS0,115200' if self.settings['serial-console'] else ''
extserial = 'serial 0 115200' if self.settings['serial-console'] else ''
msg = '''
default linux
timeout 1
label linux
kernel %(kernel)s
append initrd=%(initrd)s root=UUID=%(uuid)s ro %(kserial)s
%(extserial)s
''' % {
'kernel': kernel_image, # pylint: disable=bad-continuation
'initrd': initrd_image, # pylint: disable=bad-continuation
'uuid': uuid, # pylint: disable=bad-continuation
'kserial': kserial, # pylint: disable=bad-continuation
'extserial': extserial, # pylint: disable=bad-continuation
} # pylint: disable=bad-continuation
logging.debug("extlinux config:\n%s", msg)
# python multiline string substitution is just ugly.
# use an external file or live with the mangling, no point in
# mangling the string to remove spaces just to keep it pretty in source.
ext_f = open(conf, 'w')
ext_f.write(msg)
def list_installed_pkgs(self):
if not self.settings['pkglist']:
return
rootdir = self.devices['rootdir']
# output the list of installed packages for sources identification
self.message("Creating a list of installed binary package names:")
args = ['chroot', rootdir, 'pacman', '-Qqe']
if self.is_arm():
out = self.arm_chroot(rootdir, args)
self.message(out.decode("utf-8"))
else:
out = runcmd(args)
with open('pkg.list', 'w') as pkg:
pkg.write(out)
def upgrade_rootfs(self, rootdir):
self.message("Updating resolv.conf")
etc_resolv_conf_template = self.env.get_template("resolv.conf.j2")
etc_resolf_conf = os.path.join(str(rootdir), 'etc', 'resolv.conf')
etc_resolv_conf_template.stream(
settings_nameserver="8.8.8.8").dump(etc_resolf_conf)
self.message("Upgrading rootfs mounted at %s" % rootdir)
# Perform pacman upggrade
args = ['chroot', rootdir, 'pacman', '-Syu', '--noconfirm']
if self.is_arm():
self.arm_chroot(rootdir, args)
# Re-install _e_v_e_r_y_t_h_i_n_g_ to fix any installs that didn't run - mainly triggers when pacstrap was
# invoked
args = "chroot %s /bin/bash -c 'pacman -Qnq | pacman -S --noconfirm -'" % rootdir
self.message("Performing extra step for ARM based rootfs")
self.arm_chroot(rootdir, args, shell=True)
else:
runcmd(args)
self.message("Updating resolv.conf")
etc_resolv_conf_template = self.env.get_template("resolv.conf.j2")
etc_resolv_conf = os.path.join(str(rootdir), 'etc', 'resolv.conf')
etc_resolv_conf_template.stream(
settings_nameserver="<enter DNS IP>").dump(etc_resolv_conf)
def process_args(self, args):
"""
Optionally unpack a tarball of the disk's contents,
shell out to runcmd since it more easily handles rootdir.
Then run the vmpacstrap Filesystem squash_rootfs.
"""
if self.settings['directory'] and self.settings['tarball']:
raise cliapp.AppException(
'tarball and directory cannot be used together.')
if not self.settings['directory'] and not self.settings['tarball']:
raise cliapp.AppException('Specify either directory or a tarball.')
try:
self.filesystem.define_settings(self.settings)
if self.settings['tarball']:
# unpacking tarballs containing device nodes needs root
if os.geteuid() != 0:
sys.exit(
"You need to have root privileges to unpack the tarball."
)
rootdir = tempfile.mkdtemp()
self.remove_dirs.append(rootdir)
logging.debug('mkdir %s', rootdir)
self.message('Unpacking tarball of disk contents')
self.filesystem.devices['rootdir'] = rootdir
runcmd(['tar', '-xf', self.settings['tarball'], '-C', rootdir])
else:
self.message("Using %s directory" % self.settings['directory'])
self.filesystem.devices['rootdir'] = self.settings['directory']
self.filesystem.squash_rootfs()
except BaseException as e:
self.message('EEEK! Something bad happened...')
self.message(e)
self.cleanup_system()
raise
else:
self.cleanup_system()
def partition_esp(self):
if not self.settings['use-uefi']:
return
espsize = self.settings['esp-size'] / (1024 * 1024)
self.message("Using ESP size: %smib %s bytes" %
(espsize, self.settings['esp-size']))
runcmd([
'parted', '-s', self.settings['image'], 'mkpart', 'primary',
'fat32', '1',
str(espsize)
])
runcmd(
['parted', '-s', self.settings['image'], 'set', '1', 'boot', 'on'])
runcmd(
['parted', '-s', self.settings['image'], 'set', '1', 'esp', 'on'])
def squash_rootfs(self):
"""
Run squashfs on the rootfs within the image.
Copy the initrd and the kernel out, squashfs the rest.
Also UEFI files, if enabled, ESP partition as a vfat image. TBD.
"""
if not self.settings['squash']:
return
if not os.path.exists('/usr/bin/mksquashfs'):
logging.warning("Squash selected but mksquashfs not found!")
return
if not os.path.exists(self.settings['squash']):
os.makedirs(self.settings['squash'])
suffixed = os.path.join(self.settings['squash'], "filesystem.squashfs")
if os.path.exists(suffixed):
os.unlink(suffixed)
_, exclusions = tempfile.mkstemp()
with open(exclusions, 'w') as exclude:
exclude.write("/proc\n")
exclude.write("/dev\n")
exclude.write("/sys\n")
exclude.write("/run\n")
self.message("Running mksquashfs on rootfs.")
msg = runcmd([
'nice', 'mksquashfs', self.devices['rootdir'], suffixed,
'-no-progress', '-comp', 'xz', '-e', exclusions
],
ignore_fail=False)
os.unlink(exclusions)
logging.debug(msg)
check_size = os.path.getsize(suffixed)
logging.debug("Created squashfs: %s", suffixed)
if check_size < (1024 * 1024):
logging.warning("%s appears to be too small! %s bytes", suffixed,
check_size)
else:
logging.debug("squashed size: %s", check_size)
bootdir = os.path.join(self.devices['rootdir'], 'boot')
# copying the boot/* files
self.message("Copying boot files out of squashfs")
copy_files(bootdir, self.settings['squash'])
def mkfs(self, device, fstype, opt=None):
self.message('Creating filesystem %s' % fstype)
if opt:
runcmd(['mkfs', '-t', fstype, '-O', opt, device])
else:
runcmd(['mkfs', '-t', fstype, device])
def make_rootfs_part(self, extent):
bootsize = self.settings['esp-size'] / (1024 * 1024) + 1
runcmd([
'parted', '-s', self.settings['image'], 'mkpart', 'primary',
str(bootsize), extent
])
