def disable() -> None:
with Console('Removing: \'[email protected]/noclear.conf\''):
Utils.shell.chroot(
'rm -vf /etc/systemd/system/[email protected]/noclear.conf')
with Console('Disabling: \'[email protected]\''):
Utils.shell.chroot('systemctl disable [email protected]')
def setup(self, keymap: str, layout: str):
# Configure console
with Console("Generating console configuration"):
Utils.shell.chroot('bash -c \'\
echo "console-setup console-setup/charmap47 select UTF-8" | debconf-set-selections -v; \
echo "console-setup console-setup/codeset47 select Guess optimal character set\" | debconf-set-selections -v; \
echo "console-setup console-setup/fontface47 select Do not change the boot/kernel font" | debconf-set-selections -v\
\'')
# Configure keyboard
with Console(
"Generating keyboard configuration: \'{}\'".format(keymap)):
Utils.shell.chroot('bash -c \'\
echo "keyboard-configuration keyboard-configuration/altgr select The default for the keyboard layout" | debconf-set-selections -v; \
echo "keyboard-configuration keyboard-configuration/model select Generic 105-key (Intl) PC" | debconf-set-selections -v; \
echo "keyboard-configuration keyboard-configuration/xkb-keymap select {}" | debconf-set-selections -v; \
echo "keyboard-configuration keyboard-configuration/compose select No compose key" | debconf-set-selections -v; \
echo "keyboard-configuration keyboard-configuration/ctrl_alt_bksp boolean true" | debconf-set-selections -v; \
echo "keyboard-configuration keyboard-configuration/variant select {}" | debconf-set-selections -v\
\''.format(keymap, layout))
# Install package
with Console("Installing packages"):
Utils.shell.chroot('apt-get install -y {}'.format(' '.join(
self.packages)))
# Run configuration
with Console("Running setup"):
Utils.install('/etc/default/console-setup')
Utils.shell.chroot('setupcon --force --save-only -v')
def install(self, output: str):
source = env.paths['build'] + '/usr/lib/u-boot-olinuxino/a64-olinuxino'
# Write SPL image
with Console("Writing \'sunxi-spl.bin\'"):
Utils.shell.run(
'dd if={}/sunxi-spl.bin of={} bs=1k seek=8 conv=sync,fsync,notrunc'
.format(source, output))
# Generate U-Boot FIT image
with Console("Generating \'u-boot.itb\'"):
temp = Utils.shell.run('mktemp -d').decode().strip()
Utils.shell.run(
'cp -vf {build}/usr/lib/arm-trusted-firmware-olinuxino/sun50i_a64/bl31.bin '
'{source}/sun50i-a64-olinuxino-emmc.dtb '
'{source}/sun50i-a64-olinuxino.dtb '
'{source}/u-boot-nodtb.bin '
'{source}/u-boot.bin '
'{build}/usr/bin/mksunxi_fit_atf '
'{temp}/'.format(build=env.paths['build'],
source=source,
temp=temp))
Utils.shell.run('cd {} && '
'bash mksunxi_fit_atf *.dtb > u-boot.its && '
'mkimage -f u-boot.its u-boot.itb'.format(temp),
shell=True)
with Console("Writing \'u-boot.itb\'"):
Utils.shell.run(
'dd if={}/u-boot.itb of={} conv=notrunc,fsync bs=1k seek=40'.
format(temp, output))
Utils.shell.run('rm -rvf {}'.format(temp))
def configure(self):
# Copy resolv.conf
with Console("Copying /etc/resolv.conf"):
Utils.shell.run('rm -vf {}/etc/resolv.conf'.format(
self._build_dir),
ignore_fail=True)
Utils.shell.run(
'cp -vf /etc/resolv.conf {}/etc/resolv.conf'.format(
self._build_dir))
# Install packages
self._install_packages()
# Configure blueman
with Console("Configuring blueman"):
Setup.blueman()
# Enabling auto-login
with Console("Enabling auto-login"):
Utils.install('/etc/lightdm/lightdm.conf')
# Set default displaymanager
with Console("Setting default display-manager"):
Setup.displaymanager("lightdm")
def install(self, output: str):
# map
lines = Utils.shell.run('kpartx -avs {}'.format(output)).decode(
'utf-8', 'ignore')
for line in lines.splitlines():
w = line.split()
if w[0] == 'add':
index = int(re.match(r'^loop(\d+)p(\d+)$', w[2])[1])
device = '/dev/mapper/loop' + str(index)
with Console("Writing \'u-boot-spl.stm32\'"):
# u-boot-spl.stm32 on part1 and part2
Utils.shell.run('dd if={} of={} conv=sync,fsync,notrunc'.format(
env.paths['build'] +
"/usr/lib/u-boot-olinuxino/stm32mp1-olinuxino/u-boot-spl.stm32",
device + "p1"))
Utils.shell.run('dd if={} of={} conv=sync,fsync,notrunc'.format(
env.paths['build'] +
"/usr/lib/u-boot-olinuxino/stm32mp1-olinuxino/u-boot-spl.stm32",
device + "p2"))
with Console("Writing \'u-boot.img\'"):
# u-boot.img on part3
Utils.shell.run('dd if={} of={} conv=sync,fsync,notrunc'.format(
env.paths['build'] +
"/usr/lib/u-boot-olinuxino/stm32mp1-olinuxino/u-boot.img",
device + "p3"))
# unmap
Utils.shell.run('kpartx -dvs {}'.format(output))
def _install_packages(self) -> None:
# Install packages
with Console("Installing packages"):
# Get release independent packages
packages = self._variant_packages.packages
# Append release specific packages
if self._release_packages:
packages += self._release_packages.get_variant(str(self._variant_packages))
# Try 5 times to install packages
count = 5
_e = None
while True:
try:
Utils.shell.chroot('apt-get update')
Utils.shell.chroot('apt-get install -y {}'.format(' '.join(packages)), log_error=False)
break
except Exception as e:
count -= 1
if count == 0:
_e = e
break
with Console("Retrying..."):
Utils.shell.chroot('apt-get clean')
if _e:
raise _e
def partition(self) -> None:
"""
Partition the output blank image
:return: None
"""
if self._board.soc == "stm32mp1xx":
with Console("Creating gpt partition table"):
Utils.shell.run('sgdisk -o {}'.format(self._output))
Utils.shell.run('sgdisk --resize-table=128 -a 1' +
' -n 1:34:545 -c 1:fsbl1' +
' -n 2:546:1057 -c 2:fsbl2' +
' -n 3:1058:5153 -c 3:ssbl' +
' -n 4:5154: -c 4:rootfs' +
' -p {}'.format(self._output))
Utils.shell.run('sgdisk -A 4:set:2 {}'.format(self._output))
else:
# Create label
with Console("Creating msdos partition table"):
Utils.shell.run('parted -s {} mklabel msdos'.format(
self._output))
# Create partitions
for partition in self._partitions:
with Console("Creating partition: \'{}\'".format(
str(partition))):
Utils.shell.run(
'parted -s {} mkpart primary {} {} {}'.format(
self._output, partition.parted.type,
partition.parted.start, partition.parted.end))
def enable() -> None:
with Console('Installing: \'[email protected]/noclear.conf\''):
Utils.install('/etc/systemd/system/[email protected]/noclear.conf')
with Console('Enabling: \'[email protected]\''):
# Install and enable service
Utils.shell.chroot('mkdir -p /etc/systemd/system/[email protected]')
Utils.shell.chroot('systemctl --no-reload enable [email protected]')
# Configure ttyGS0
Utils.shell.chroot('/bin/bash -c "echo \'\n# USB-ACM tty serial console\nttyGS0\n\' >> /etc/securetty"')
def enable() -> None:
with Console('Installing: \'[email protected]/noclear.conf\''):
Utils.install(
'/etc/systemd/system/[email protected]/noclear.conf')
with Console('Enabling: \'[email protected]\''):
# Install and enable service
Utils.shell.chroot(
'mkdir -p /etc/systemd/system/[email protected]')
Utils.shell.chroot(
'systemctl --no-reload enable [email protected]')
def setup(self, locale: str):
# Configure
with Console("Generating locales: \'{}\'".format(locale)):
Utils.shell.chroot(
'bash -c \'\
echo "locales locales/locales_to_be_generated multiselect {locale} UTF-8" | debconf-set-selections -v; \
echo "locales locales/default_environment_locale select {locale}" | debconf-set-selections -v \
\''.format(locale=locale), env.paths['build'])
# Install
with Console("Installing packages"):
Utils.shell.chroot('apt-get install -y {}'.format(' '.join(
self.packages)))
def build_filesystem(**kwargs):
"""
Build root filesystem and configure it.
"""
# Update environment options
env.options.update(kwargs)
verify_options()
# Build filesystem
board: Board = Boards().get_board(kwargs['board'])
env.objects['board'] = board
# Prepare build tree
env.paths['board'] = os.path.join(env.paths['output'], str(board).lower())
if not os.path.exists(env.paths['board']):
os.mkdir(env.paths['board'])
for directory in ['filesystem', 'images']:
path = os.path.join(env.paths['board'], directory)
# Create directory
if not os.path.exists(path):
os.mkdir(path)
# Append paths
env.paths[directory] = path
builders = [VariantMinimal, VariantBase]
for builder in builders:
_builder = builder()
Console().info("Creating \'{}\' filesystem...".format(
_builder.variant))
for stage in _builder.stages:
method = getattr(_builder, stage)
with Console(stage.capitalize()):
method()
# If this current stage is the target one break
if builder.variant == env.options['variant']:
break
if env.options['apt_cacher']:
Service.apt_cache.disable()
def _generate_boot_cmd(board: Board) -> None:
with Console("Generating /boot/boot.scr"):
Utils.install('/boot/boot.cmd')
# The FIT image is always located in /boot directory.
# If there is such defined partition retrieve it's number. Do the same for /
parts = {
'boot': 1,
'root': 1
}
if board.soc == "stm32mp1xx":
parts = {
'boot': 4,
'root': 4
}
# for i in range(len(partitions)):
# partition = partitions[i]
# if partition.fstab.mount == '/':
# parts['root'] = i + 1
# elif partition.fstab.mount == '/boot':
# parts['boot'] = i + 1
# Generate template
bootargs={
'console': 'ttyS0,115200',
'panic': 10,
'loglevel': 4,
}
if board.soc == "stm32mp1xx":
bootargs={
'console': 'ttySTM0,115200',
'panic': 10,
'loglevel': 10,
}
Utils.template.install(
env.paths['build'] + '/boot/boot.cmd',
board=board,
bootargs=bootargs,
fit={
'file': 'kernel.itb',
'load': board.loading.fit,
},
partitions=parts,
stamp={
'date': str(datetime.datetime.now()),
'uuid': str(uuid.uuid4()),
},
uenv={
'file': 'uEnv.txt',
'load': board.loading.uenv,
},
)
# Generate boot.scr
Utils.shell.run(
"mkimage -C none -A arm -T script -d {build}/boot/boot.cmd {build}/boot/boot.scr".format(
build=env.paths['build']),
shell=True
)
def wrapper(*args, **kwargs):
fs: FileSystemBase = args[0]
# Create empty build folder
if os.path.exists(fs.build_dir):
# Make sure the target path is not mounted
Utils.shell.unbind(fs.build_dir)
shutil.rmtree(fs.build_dir)
os.mkdir(fs.build_dir)
stage = func.__name__
index = fs.stages.index(stage)
file = None
# If this is the first stage, there is nothing to extract
if index != 0:
file = fs.build_dir + '.' + fs.stages[index - 1] + '.tar.gz'
else:
variants = ['minimal', 'base']
_index = variants.index(fs.variant)
if _index:
file = fs.build_dir + '.tar.gz'
file = file.replace(fs.variant, variants[_index -1])
if file:
with Console("Extracting archive: {}".format(os.path.basename(file))):
Utils.archive.extract(file, fs.build_dir)
return func(*args, **kwargs)
def install(self, output: str):
with Console("Writing \'u-boot-sunxi-with-spl.bin\'"):
Utils.shell.run(
'dd if={} of={} bs=1k seek=8 conv=sync,fsync,notrunc'.format(
env.paths['build'] +
"/usr/lib/u-boot-olinuxino/s3-olinuxino/u-boot-sunxi-with-spl.bin",
output))
def verify_options():
# Verify release
release = None
for dist in Distributions():
if env.options['release'] == str(dist):
release = dist.recommended
elif env.options['release'] in dist.releases:
release = env.options['release']
if release:
env.objects['distribution'] = dist
break
if release is None:
raise Exception(
"Target distribution \'{}\' not found in configuration files".
format(release))
if release != env.options['release']:
Console().warning(
"Using distribution stable release: \'{}\'".format(release))
env.options['release'] = release
def setup(self):
for interface in NetworkParser().interfaces:
interface: Interface
with Console('Installing: \'/etc/network/interfaces\''):
Utils.install('/etc/network/interfaces')
with Console("Configuring interface: \'{}\'".format(str(interface))):
file = '/etc/network/interfaces.d/{}'.format(str(interface))
source = env.paths['overlay'] + '/etc/network/interfaces.d/default'
destination = env.paths['build'] + file
# Install source list
Utils.shell.run("install -m 644 {} {}".format(source, destination))
Utils.template.install(destination, interface=interface)
def configure(self):
with Mounter.mount(self._output, self._partitions) as m:
with Console("Generating /etc/fstab"):
# Append UUID
for partition in self._partitions:
partition.fstab.uuid = m.uuid(partition)
# TODO: This need a fix
Setup.fstab(self._partitions, m.mountpoint('root'))
def _generate_uboot_env(board: Board, configs: dict = None) -> None:
with Console("Generating /boot/uEnv.txt"):
Utils.install('/boot/uEnv.txt')
Utils.template.install(
env.paths['build'] + '/boot/uEnv.txt',
configs=configs,
stamp={
'date': str(datetime.datetime.now()),
'uuid': str(uuid.uuid4()),
}
)
with Console("Installing /etc/fw_env.config"):
Utils.install('/etc/fw_env.config')
with Console("Installing /uboot.env"):
src = env.paths['build'] + "/usr/lib/u-boot-olinuxino/{}/uboot.env".format(board.name.lower())
dest = env.paths['build'] + "/uboot.env"
Utils.shell.run('install -D -v -m {} {} {}'.format(644, src, dest))
def wrapper(*args, **kwargs):
# Run function
ret = func(*args, **kwargs)
file = env.paths['build'] + '.' + func.__name__ + '.tar.gz'
with Console("Creating archive: {}".format(os.path.basename(file))):
Utils.archive.gzip(env.paths['build'], file,
exclude=['/dev/*', '/proc/*', '/run/*', '/tmp/*', '/sys/*'])
return ret
def setup(self, displaymanager: str):
with Console("Setting default display-manager: \'{}\'".format(
displaymanager)):
Utils.shell.chroot(
'bash -c \'\
echo "/usr/sbin/{displaymanager}" > /etc/X11/default-display-manager; \
\''.format(displaymanager=displaymanager), env.paths['build'])
# Reconfigure
Utils.shell.chroot(
'dpkg-reconfigure -f noninteractive {}'.format(displaymanager))
def _generate_fit(board):
with Console("Generating /usr/lib/olinuxino/kernel.its"):
Utils.install('/etc/kernel/postinst.d/uboot-fit', mode='755')
Utils.install('/usr/lib/olinuxino/kernel.its')
# Generate fdts and overlay data
fdts = []
overlays = []
for model in board.models:
if model.fdt not in fdts:
fdts.append(model.fdt)
for overlay in model.overlays:
file = env.paths['build'] + '/usr/lib/olinuxino-overlays/{}/{}'.format(board.soc, overlay)
if overlay not in overlays and os.path.exists(file):
overlays.append(overlay)
# Remap board fdt and overlays
models = []
for model in board.models:
dtbo = []
for overlay in model.overlays:
if overlay in overlays:
dtbo.append(overlays.index(overlay) + 1)
models.append({
'name': str(model),
'fdt': fdts.index(model.fdt) + 1,
'id': model.id,
'overlays': dtbo,
'compatible': 'olimex,{}'.format(str(model).lower())
})
# Generate load addresses for overlays
addr = int(board.loading.overlays, 16)
temp = []
for overlay in overlays:
temp.append({overlay: {'load': '0x{:08X}'.format(addr)}})
addr += 0x10000
overlays = temp
Utils.template.install(
env.paths['build'] + '/usr/lib/olinuxino/kernel.its',
arch='arm' if board.arch == 'armhf' else board.arch,
board=board,
fdts=fdts,
overlays=overlays,
stamp={
'date': str(datetime.datetime.now()),
'uuid': str(uuid.uuid4()),
},
models=models,
)
def setup(self, release: str):
with Console("Installing packages"):
Utils.shell.chroot('apt-get install -y {}'.format(' '.join(
self.packages)))
for repo in Repositories():
repo: Repository
if repo.testing and env.options['releaseimage']:
continue
with Console("Adding: \'{}\'".format(repo.url)):
file = '/etc/apt/sources.list.d/{}.list'.format(str(repo))
source = env.paths[
'overlay'] + '/etc/apt/sources.list.d/default.list'
destination = env.paths['build'] + file
# # Install source list
Utils.shell.run("install -m 644 {} {}".format(
source, destination))
Utils.template.install(destination, repo=repo, release=release)
if repo.key and repo.keyserver:
# Import gpg key
Utils.shell.chroot(
'apt-key adv --keyserver {} --recv-keys {}'.format(
repo.keyserver, repo.key))
elif repo.keyfile:
# Import keyfile
Utils.install(repo.keyfile)
Utils.shell.chroot('apt-key add {}'.format(repo.keyfile))
# Update sources
# It's possible for some repository to have missing release files, so
# for now ignore error upon update
with Console("Updating"):
Utils.shell.chroot('apt-get update', ignore_fail=True)
def setup(self):
with Console("Installing Linux image"):
Utils.shell.chroot('apt-get install -y {}'.format(' '.join(
self.packages)))
with Console("Configuring modules"):
with open(env.paths['build'] + '/etc/modules', 'a') as f:
for module in self.parser.modules['enable']:
f.write(module + '\n')
# Append new line at the end of file
f.write('\n')
with open(
env.paths['build'] +
'/etc/modprobe.d/blacklist-olinuxino.conf', 'w') as f:
for module in self.parser.modules['blacklist']:
f.write("blacklist {}\n".format(module))
# Append new line at the end of file
f.write('\n')
class MyTestCase(unittest.TestCase):
def setUp(self) -> None:
self._console = Console()
def test_something(self):
self._console.info("Printing with info()")
with Console("Context manager level 0"):
time.sleep(1.5)
with Console("Context manager level 1"):
time.sleep(1.5)
with Console("Context manager level 2"):
time.sleep(1.5)
with Console("Context manager level 1"):
time.sleep(1.5)
with Console("Context manager level 1"):
time.sleep(1.5)
with Console("Context manager level 1"):
with Console("Context manager level 2"):
time.sleep(1.5)
with Console("Context manager level 2"):
time.sleep(1.5)
def configure(self):
# Copy resolv.conf
with Console("Copying /etc/resolv.conf"):
Utils.shell.run('rm -vf {}/etc/resolv.conf'.format(
self._build_dir),
ignore_fail=True)
Utils.shell.run(
'cp -vf /etc/resolv.conf {}/etc/resolv.conf'.format(
self._build_dir))
# Install packages
self._install_packages()
# Configure blueman
with Console("Configuring blueman"):
Setup.blueman()
# Enabling auto-login
with Console("Enabling auto-login"):
Utils.install('/etc/lightdm/lightdm.conf')
# Set default displaymanager
with Console("Setting default display-manager"):
Setup.displaymanager("lightdm")
# post-install
with Console("Post-install tasks"):
Utils.shell.chroot(
"/bin/bash -c 'echo -en > /etc/modules-load.d/cups-filters.conf'",
ignore_fail=True)
# meh broken light-locker in focal
Utils.shell.chroot('apt-get -y --purge remove light-locker',
log_error=False)
# xfce panel defaults
Utils.install('/etc/X11/Xsession.d/99olimex')
# set xfce background
Utils.shell.run(
'dpkg-divert --rename --add --divert /usr/share/backgrounds/xfce/xfce-stripes.png.real /usr/share/backgrounds/xfce/xfce-stripes.png'
)
Utils.shell.run(
'dpkg-divert --rename --add --divert /usr/share/backgrounds/xfce/xfce-blue.jpg.real /usr/share/backgrounds/xfce/xfce-blue.jpg'
)
Utils.install('/usr/share/backgrounds/xfce/xfce-stripes.png')
Utils.install('/usr/share/backgrounds/xfce/xfce-blue.jpg')
Utils.install('/usr/share/backgrounds/xfce/xfce-red.jpg')
# restore resolv.conf
with Console("Restore /etc/resolv.conf"):
Utils.shell.run('rm -vf {}/etc/resolv.conf'.format(
self._build_dir),
ignore_fail=True)
Utils.shell.run(
'ln -nsf ../run/resolvconf/resolv.conf {}/etc/resolv.conf'.
format(self._build_dir))
def build(self):
# Built a new file system
with Console("Running qemu-debootstrap"):
# TODO: Store date, UUID and configs and compare them to the stamp
board: Board = env.objects['board']
distribution: Distribution = env.objects['distribution']
release = env.options['release']
Utils.qemu.debootstrap(arch=board.arch,
release=release,
path=self._build_dir,
components=distribution.components,
include=None,
mirror=distribution.repository)
def test_something(self):
self._console.info("Printing with info()")
with Console("Context manager level 0"):
time.sleep(1.5)
with Console("Context manager level 1"):
time.sleep(1.5)
with Console("Context manager level 2"):
time.sleep(1.5)
with Console("Context manager level 1"):
time.sleep(1.5)
with Console("Context manager level 1"):
time.sleep(1.5)
with Console("Context manager level 1"):
with Console("Context manager level 2"):
time.sleep(1.5)
with Console("Context manager level 2"):
time.sleep(1.5)
def setup(self):
with Console("Installing packages"):
# This will fail due blueman not detecting chroot environment.
# Ignore the error and do some dpkg magic
try:
Utils.shell.chroot('apt-get install -y {}'.format(' '.join(
self.packages)),
log_error=False)
return
except Exception:
pass
Utils.shell.run(
'sed -i "/^set -e/d" {}/var/lib/dpkg/info/blueman.postinst'.
format(env.paths['build']))
Utils.shell.chroot('apt-get install -y -f')
def format(self):
with Mounter.map(self._output, self._partitions) as m:
for partition in self._partitions:
with Console("Formating partition: \'{}\'".format(
str(partition))):
# Get parted related information
device = m.device(partition)
opts = ''
if partition.fstab.type == 'ext4':
opts = '-O ^64bit,^metadata_csum'
# Make filesystem
Utils.shell.run('mkfs.{} {} {}'.format(
partition.fstab.type, opts, device))
Utils.shell.run('udevadm trigger {}'.format(device))
Utils.shell.run('udevadm settle'.format(device))
def copy(self):
exclude = ['/dev/*', '/proc/*', '/run/*', '/tmp/*', '/sys/*']
with Mounter.mount(self._output, self._partitions) as m:
for partition in self._partitions:
with Console("Copying partition: \'{}\'".format(
str(partition))):
x = ""
for e in exclude:
x += '--exclude={} '.format(e)
_e = None
try:
Utils.shell.run('rsync -aHWXh {} {} {}/'.format(
x, env.paths['build'] + partition.fstab.mount,
m.mountpoint(partition)))
except KeyboardInterrupt:
_e = KeyboardInterrupt
except Exception as e:
_e = e
if _e:
raise _e