def execute(self, _buildenv, target):
with target:
att = self.node.et.attrib
options = ""
if 'groups' in att:
options += '-G "%s" ' % att['groups']
if 'shell' in att:
options += '-s "%s" ' % att['shell']
if 'uid' in att:
options += '-u "%s" ' % att['uid']
if 'gid' in att:
options += '-g "%s" ' % att['gid']
if 'home' in att:
options += '-d "%s" ' % att['home']
if 'system' in att and att['system'] == 'true':
options += '-r'
if 'create_home' in att and att['create_home'] == 'false':
options += '-M '
else:
options += '-m '
if 'create_group' in att and att['create_group'] == 'false':
options += '-N '
else:
options += '-U '
cmd = '/usr/sbin/useradd %s "%s"' % (options, self.node.et.text)
chroot(target.path, cmd)
if 'passwd' in att:
cmd = "passwd %s" % self.node.et.text
stdin = "%s\n%s\n" % (att["passwd"], att["passwd"])
chroot(target.path, cmd, stdin=stdin)
def execute(self, _buildenv, target):
target_name = self.node.et.attrib['path']
link_name = self.node.et.text
with target.protect({link_name}):
chroot(
target.path,
"""/bin/sh -c 'ln -sf %s "%s"' """ % (target_name, link_name))
def add_key(self, key):
cmd = 'echo "%s" > %s' % (key, self.rfs.fname("tmp/key.pub"))
clean = 'rm -f %s' % self.rfs.fname("tmp/key.pub")
do(cmd)
with self.rfs:
chroot(self.rfs.path, 'apt-key add /tmp/key.pub')
do(clean)
def install(self, target):
if '/' not in self.fs:
return
imagemnt = os.path.join(target, "imagemnt")
imagemntfs = Filesystem(imagemnt)
try:
do('cp -a /dev/loop0 /dev/poop0')
do('losetup /dev/poop0 "%s"' % self.fs['/'].filename)
do('kpartx -as /dev/poop0')
for entry in self.fs.depthlist():
do('mount /dev/mapper/poop0p%d %s' %
(entry.partnum, imagemntfs.fname(entry.mountpoint)))
do("mount --bind /dev %s" % imagemntfs.fname("dev"))
do("mount --bind /proc %s" % imagemntfs.fname("proc"))
do("mount --bind /sys %s" % imagemntfs.fname("sys"))
do('mkdir -p "%s"' % imagemntfs.fname("boot/grub"))
devmap = open(imagemntfs.fname("boot/grub/device.map"), "w")
devmap.write("(hd0) /dev/poop0\n")
devmap.close()
chroot(imagemnt, "update-initramfs -u -k all")
chroot(imagemnt, "update-grub2")
if "efi" in self.fw_type:
grub_tgt = next(t for t in self.fw_type if t.endswith("-efi"))
do("chroot %s grub-install --target=%s --removable "
"--no-floppy /dev/poop0" % (imagemnt, grub_tgt))
if "shimfix" in self.fw_type:
# grub-install is heavily dependent on the running system having
# a BIOS or EFI. The initvm is BIOS-based, so fix the resulting
# shim installation.
do("chroot %s /bin/bash -c '"
"cp -r /boot/efi/EFI/BOOT /boot/efi/EFI/debian && "
"cd /usr/lib/shim && f=( shim*.efi.signed ) && cp "
"${f[0]} /boot/efi/EFI/debian/${f[0]%%.signed}'" % imagemnt)
if not self.fw_type or "bios" in self.fw_type:
do("chroot %s grub-install --no-floppy /dev/poop0" %
(imagemnt))
finally:
os.unlink(imagemntfs.fname("boot/grub/device.map"))
do("umount %s" % imagemntfs.fname("dev"), allow_fail=True)
do("umount %s" % imagemntfs.fname("proc"), allow_fail=True)
do("umount %s" % imagemntfs.fname("sys"), allow_fail=True)
for entry in reversed(self.fs.depthlist()):
do('umount /dev/mapper/poop0p%d' % entry.partnum,
allow_fail=True)
do('kpartx -d /dev/poop0', allow_fail=True)
do("losetup -d /dev/poop0", allow_fail=True)
def install(self, target):
if '/' not in self.fs:
return
imagemnt = os.path.join(target, "imagemnt")
imagemntfs = Filesystem(imagemnt)
try:
loopdev = self.losetup(self.fs['/'].filename)
loopnum = loopdev.replace("/dev/loop", "")
poopdev = "/dev/poop" + loopnum
do('cp -a %s %s' % (loopdev, poopdev))
do('kpartx -as %s' % poopdev)
for entry in self.fs.depthlist():
do('mount /dev/mapper/poop%sp%d %s' %
(loopnum, entry.partnum, imagemntfs.fname(
entry.mountpoint)))
do("mount --bind /dev %s" % imagemntfs.fname("dev"))
do("mount --bind /proc %s" % imagemntfs.fname("proc"))
do("mount --bind /sys %s" % imagemntfs.fname("sys"))
do('mkdir -p "%s"' % imagemntfs.fname("boot/grub"))
devmap = open(imagemntfs.fname("boot/grub/device.map"), "w")
devmap.write("(hd0) %s\n" % poopdev)
devmap.close()
chroot(imagemnt, "update-initramfs -u -k all")
# Replace groot and kopt because
# else they will be given bad values
do('chroot %s sed -in "s/^# groot=.*$/# groot=\(hd0,%d\)/" %s' %
(imagemnt, int(entry.partnum) - 1, "/boot/grub/menu.lst"))
do('chroot %s sed -in "s/^# kopt=.*$/# kopt=root=LABEL=%s/" %s' %
(imagemnt, entry.label, "/boot/grub/menu.lst"))
chroot(imagemnt, "update-grub")
do("chroot %s grub-install --no-floppy %s" % (imagemnt, poopdev))
except Exception as E:
logging.exception(E)
finally:
os.unlink(imagemntfs.fname("boot/grub/device.map"))
do("umount %s" % imagemntfs.fname("dev"), allow_fail=True)
do("umount %s" % imagemntfs.fname("proc"), allow_fail=True)
do("umount %s" % imagemntfs.fname("sys"), allow_fail=True)
for entry in reversed(self.fs.depthlist()):
do('umount /dev/mapper/poop%sp%d' % (loopnum, entry.partnum),
allow_fail=True)
do("kpartx -d %s" % poopdev, allow_fail=True)
do("losetup -d %s" % poopdev, allow_fail=True)
def execute(self, _buildenv, target):
with target:
att = self.node.et.attrib
# we use -f always
options = "-f "
if 'gid' in att:
options += '-g "%s" ' % att['gid']
if 'system' in att and att['system'] == 'True':
options += '-r'
cmd = '/usr/sbin/groupadd %s "%s"' % (options, self.node.et.text)
chroot(target.path, cmd)
def install(self, target):
if '/' not in self.fs:
return
imagemnt = os.path.join(target, "imagemnt")
imagemntfs = Filesystem(imagemnt)
try:
do('cp -a /dev/loop0 /dev/poop0')
do('losetup /dev/poop0 "%s"' % self.fs['/'].filename)
do('kpartx -as /dev/poop0')
for entry in self.fs.depthlist():
do('mount /dev/mapper/poop0p%d %s' %
(entry.partnum, imagemntfs.fname(entry.mountpoint)))
do("mount --bind /dev %s" % imagemntfs.fname("dev"))
do("mount --bind /proc %s" % imagemntfs.fname("proc"))
do("mount --bind /sys %s" % imagemntfs.fname("sys"))
do('mkdir -p "%s"' % imagemntfs.fname("boot/grub"))
devmap = open(imagemntfs.fname("boot/grub/device.map"), "w")
devmap.write("(hd0) /dev/poop0\n")
devmap.close()
chroot(imagemnt, "update-initramfs -u -k all")
# Replace groot and kopt because
# else they will be given bad values
do('chroot %s sed -in "s/^# groot=.*$/# groot=\(hd0,%d\)/" %s' %
(imagemnt, int(entry.partnum) - 1, "/boot/grub/menu.lst"))
do('chroot %s sed -in "s/^# kopt=.*$/# kopt=root=LABEL=%s/" %s' %
(imagemnt, entry.label, "/boot/grub/menu.lst"))
chroot(imagemnt, "update-grub")
do("chroot %s grub-install --no-floppy /dev/poop0" % (imagemnt))
finally:
os.unlink(imagemntfs.fname("boot/grub/device.map"))
do("umount %s" % imagemntfs.fname("dev"), allow_fail=True)
do("umount %s" % imagemntfs.fname("proc"), allow_fail=True)
do("umount %s" % imagemntfs.fname("sys"), allow_fail=True)
for entry in reversed(self.fs.depthlist()):
do('umount /dev/mapper/poop0p%d' % entry.partnum,
allow_fail=True)
do('kpartx -d /dev/poop0', allow_fail=True)
do("losetup -d /dev/poop0", allow_fail=True)
def initialize_dirs(self, build_sources=False):
mirror = self.xml.create_apt_sources_list(build_sources=build_sources)
if self.rfs.lexists("etc/apt/sources.list"):
self.rfs.remove("etc/apt/sources.list")
self.rfs.write_file("etc/apt/sources.list", 0o644, mirror)
self.rfs.mkdir_p("var/cache/elbe")
preseed = get_preseed(self.xml)
preseed_txt = preseed_to_text(preseed)
self.rfs.write_file("var/cache/elbe/preseed.txt", 0o644, preseed_txt)
with self.rfs:
cmd = ('debconf-set-selections < %s' %
self.rfs.fname("var/cache/elbe/preseed.txt"))
chroot(self.rfs.path, cmd)
def __enter__(self):
if os.path.exists(self.path + '/../repo/pool'):
do("mv %s/../repo %s" % (self.path, self.path))
do('echo "deb copy:///repo %s main" > '
'%s/etc/apt/sources.list.d/local.list' %
(self.xml.text("project/suite"), self.path))
do('echo "deb-src copy:///repo %s main" >> '
'%s/etc/apt/sources.list.d/local.list' %
(self.xml.text("project/suite"), self.path))
self.cdrom_mount()
self.rfs.__enter__()
if self.xml.has("project/mirror/cdrom"):
chroot(self.rfs.path,
'apt-key '
'--keyring /etc/apt/trusted.gpg.d/elbe-cdrepo.gpg '
'add /cdrom/repo.pub')
chroot(self.rfs.path,
'apt-key '
'--keyring /etc/apt/trusted.gpg.d/elbe-cdtargetrepo.gpg '
'add /cdrom/targetrepo/repo.pub')
if os.path.exists(os.path.join(self.rfs.path, 'repo/pool')):
chroot(self.rfs.path,
'apt-key '
'--keyring /etc/apt/trusted.gpg.d/elbe-localrepo.gpg '
'add /repo/repo.pub')
return self
def execute(self, _buildenv, target):
att = self.node.et.attrib
dst = att["dst"]
content = self.node.et.text
encoding = "plain"
owner = None
group = None
mode = None
if "encoding" in att:
encoding = att["encoding"]
if "owner" in att:
owner = att["owner"]
if "group" in att:
group = att["group"]
if "mode" in att:
mode = att["mode"]
try:
target.mkdir_p(os.path.dirname(dst))
except OSError as E:
if E.errno is not errno.EEXIST:
raise
content = AddFileAction.decode(content, encoding)
if "append" in att and att["append"] == "true":
target.append_file(dst, content)
else:
target.write_file(dst, None, content)
if owner is not None:
chroot(target.path, 'chown "%s" "%s"' % (owner, dst))
if group is not None:
chroot(target.path, 'chgrp "%s" "%s"' % (group, dst))
if mode is not None:
chroot(target.path, 'chmod "%s" "%s"' % (mode, dst))
def execute(self, buildenv, _target):
with buildenv:
chroot(buildenv.path, "/bin/sh", stdin=self.node.et.text)
def debootstrap(self, arch="default"):
# pylint: disable=too-many-statements
# pylint: disable=too-many-branches
cleanup = False
suite = self.xml.prj.text("suite")
primary_mirror = self.xml.get_primary_mirror(
self.rfs.fname('/cdrom/targetrepo'))
if self.xml.prj.has("mirror/primary_proxy"):
os.environ["no_proxy"] = "10.0.2.2,localhost,127.0.0.1"
proxy = self.xml.prj.text("mirror/primary_proxy")
proxy = proxy.strip().replace("LOCALMACHINE", "localhost")
os.environ["http_proxy"] = proxy
os.environ["https_proxy"] = proxy
else:
os.environ["no_proxy"] = ""
os.environ["http_proxy"] = ""
os.environ["https_proxy"] = ""
os.environ["LANG"] = "C"
os.environ["LANGUAGE"] = "C"
os.environ["LC_ALL"] = "C"
os.environ["DEBIAN_FRONTEND"] = "noninteractive"
os.environ["DEBONF_NONINTERACTIVE_SEEN"] = "true"
logging.info("Debootstrap log")
if arch == "default":
arch = self.xml.text("project/buildimage/arch", key="arch")
host_arch = get_command_out("dpkg --print-architecture").strip().decode()
includepkgs = None
strapcmd = 'debootstrap '
if self.xml.has("target/debootstrapvariant"):
bootstrapvariant = self.xml.text("target/debootstrapvariant")
includepkgs = self.xml.node("target/debootstrapvariant").et.get("includepkgs")
strapcmd += '--variant="%s" ' % bootstrapvariant
if includepkgs and not "gnupg" in includepkgs.split(','):
includepkgs += ",gnupg"
if not includepkgs:
includepkgs = "gnupg"
strapcmd += ' --include="%s"' % includepkgs
if not self.xml.is_cross(host_arch):
# ignore gpg verification if install from cdrom, cause debootstrap
# seems to ignore /etc/apt/trusted.gpg.d/elbe-keyring.gpg
# 01/2017 manut
if self.xml.has(
"project/noauth") or self.xml.has("project/mirror/cdrom"):
cmd = '%s --no-check-gpg --arch=%s "%s" "%s" "%s"' % (
strapcmd, arch, suite, self.rfs.path, primary_mirror)
else:
cmd = '%s --arch=%s "%s" "%s" "%s"' % (
strapcmd, arch, suite, self.rfs.path, primary_mirror)
try:
self.cdrom_mount()
do(cmd)
except CommandError:
cleanup = True
raise DebootstrapException()
finally:
self.cdrom_umount()
if cleanup:
self.rfs.rmtree("/")
return
if self.xml.has("project/noauth"):
cmd = '%s --no-check-gpg --foreign --arch=%s "%s" "%s" "%s"' % (
strapcmd, arch, suite, self.rfs.path, primary_mirror)
else:
if self.xml.has("project/mirror/cdrom"):
keyring = ' --keyring="%s/targetrepo/elbe-keyring.gpg"' % (
self.rfs.fname("cdrom"))
else:
keyring = ''
cmd = '%s --foreign --arch=%s %s "%s" "%s" "%s"' % (
strapcmd, arch, keyring, suite, self.rfs.path, primary_mirror)
try:
self.cdrom_mount()
do(cmd)
ui = "/usr/share/elbe/qemu-elbe/" + self.xml.defs["userinterpr"]
if not os.path.exists(ui):
ui = "/usr/bin/" + self.xml.defs["userinterpr"]
do('cp %s %s' % (ui, self.rfs.fname("usr/bin")))
if self.xml.has("project/noauth"):
chroot(self.rfs.path,
'/debootstrap/debootstrap --no-check-gpg --second-stage')
else:
chroot(self.rfs.path,
'/debootstrap/debootstrap --second-stage')
chroot(self.rfs.path, 'dpkg --configure -a')
except CommandError:
cleanup = True
raise DebootstrapException()
finally:
self.cdrom_umount()
if cleanup:
self.rfs.rmtree("/")
def build_sysroot(self):
do('rm -rf %s; mkdir "%s"' % (self.sysrootpath, self.sysrootpath))
self.sysrootenv = BuildEnv(self.xml, self.sysrootpath, clean=True)
# Import keyring
self.sysrootenv.import_keys()
logging.info("Keys imported")
self.install_packages(self.sysrootenv, buildenv=False)
# ignore packages from debootstrap
tpkgs = self.xml.get_target_packages()
bspkgs = self.xml.node("debootstrappkgs")
ignore_pkgs = [p.et.text for p in bspkgs if p.et.text not in tpkgs]
ignore_dev_pkgs = []
if self.xml.has('target/pkg-blacklist/sysroot'):
ignore_dev_pkgs = [
p.et.text
for p in self.xml.node("target/pkg-blacklist/sysroot")
]
with self.sysrootenv:
try:
cache = self.get_rpcaptcache(env=self.sysrootenv)
cache.update()
except Exception as e:
raise AptCacheUpdateError(e)
try:
cache.mark_install_devpkgs(set(ignore_pkgs),
set(ignore_dev_pkgs))
except SystemError as e:
logging.exception("Mark install devpkgs failed")
try:
cache.commit()
except SystemError as e:
logging.exception("Commiting changes failed")
raise AptCacheCommitError(str(e))
self.gen_licenses("sysroot-target", self.sysrootenv,
[p.name for p in cache.get_installed_pkgs()])
try:
self.sysrootenv.rfs.dump_elbeversion(self.xml)
except IOError:
logging.exception("Dump elbeversion into sysroot failed")
sysrootfilelist = os.path.join(self.builddir, "sysroot-filelist")
with self.sysrootenv.rfs:
chroot(self.sysrootpath, "/usr/bin/symlinks -cr /usr/lib")
paths = self.get_sysroot_paths()
do("rm %s" % sysrootfilelist, allow_fail=True)
os.chdir(self.sysrootpath)
for p in paths:
do('find -path "%s" >> %s' % (p, sysrootfilelist))
# include /lib if it is a symlink (buster and later)
if os.path.islink(self.sysrootpath + '/lib'):
with open(sysrootfilelist, 'a') as filelist_fd:
filelist_fd.write('./lib')
do("tar cfJ %s/sysroot.tar.xz -C %s -T %s" %
(self.builddir, self.sysrootpath, sysrootfilelist))
def extract_target(src, xml, dst, cache):
# pylint: disable=too-many-locals
# pylint: disable=too-many-branches
# create filelists describing the content of the target rfs
if xml.tgt.has("tighten") or xml.tgt.has("diet"):
pkglist = [
n.et.text for n in xml.node('target/pkg-list') if n.tag == 'pkg'
]
arch = xml.text("project/buildimage/arch", key="arch")
if xml.tgt.has("diet"):
withdeps = []
for p in pkglist:
deps = cache.get_dependencies(p)
withdeps += [d.name for d in deps]
withdeps += [p]
pkglist = list(set(withdeps))
file_list = []
for line in pkglist:
file_list += src.cat_file("var/lib/dpkg/info/%s.list" % (line))
file_list += src.cat_file("var/lib/dpkg/info/%s.conffiles" %
(line))
file_list += src.cat_file("var/lib/dpkg/info/%s:%s.list" %
(line, arch))
file_list += src.cat_file("var/lib/dpkg/info/%s:%s.conffiles" %
(line, arch))
file_list = sorted(set(file_list),
key=lambda k: k[4:] if k.startswith('/usr') else k)
copy_filelist(src, file_list, dst)
else:
# first copy most diretories
for f in src.listdir():
subprocess.call(["cp", "-a", "--reflink=auto", f, dst.fname('')])
try:
dst.mkdir_p("dev")
except BaseException:
pass
try:
dst.mkdir_p("proc")
except BaseException:
pass
try:
dst.mkdir_p("sys")
except BaseException:
pass
if xml.tgt.has("setsel"):
pkglist = [
n.et.text for n in xml.node('target/pkg-list') if n.tag == 'pkg'
]
psel = 'var/cache/elbe/pkg-selections'
with open(dst.fname(psel), 'w+') as f:
for item in pkglist:
f.write("%s install\n" % item)
host_arch = get_command_out("dpkg --print-architecture").strip()
if xml.is_cross(host_arch):
ui = "/usr/share/elbe/qemu-elbe/" + str(xml.defs["userinterpr"])
if not os.path.exists(ui):
ui = "/usr/bin/" + str(xml.defs["userinterpr"])
do('cp %s %s' % (ui, dst.fname("usr/bin")))
cmds = [
"--clear-selections",
"--set-selections < %s" % dst.fname(psel), "--purge -a"
]
for cmd in cmds:
chroot(dst.path, "/usr/bin/dpkg %s" % cmd)
def chroot(self, directory, cmd, **kwargs):
return chroot(directory, cmd, **kwargs)
def execute(self, _buildenv, target):
with target:
chroot(target.path, "/bin/sh", stdin=self.node.et.text)
def execute(self, _buildenv, target):
with target:
chroot(target.path, self.node.et.text)
def seed_etc(self):
passwd = self.xml.text("target/passwd").encode(encoding='utf-8')
stdin = "%s\n%s\n" % (passwd, passwd)
chroot(self.rfs.path, "passwd", stdin=stdin)
hostname = self.xml.text("target/hostname")
fqdn = hostname
if self.xml.has("target/domain"):
fqdn = ("%s.%s" % (hostname, self.xml.text("target/domain")))
chroot(self.rfs.path,
"""/bin/sh -c 'echo "127.0.1.1 %s %s elbe-daemon" >> """
"""/etc/hosts'""" % (fqdn,hostname))
chroot(self.rfs.path,
"""/bin/sh -c 'echo "%s" > /etc/hostname'""" % hostname)
chroot(self.rfs.path,
"""/bin/sh -c 'echo "%s" > """
"""/etc/mailname'""" % (fqdn))
if self.xml.has("target/console"):
serial_con, serial_baud = self.xml.text(
"target/console").split(',')
if serial_baud:
chroot(self.rfs.path,
"""/bin/sh -c '[ -f /etc/inittab ] && """
"""echo "T0:23:respawn:/sbin/getty -L %s %s vt100" >> """
"""/etc/inittab'""" % (serial_con, serial_baud),
allow_fail=True)
chroot(self.rfs.path,
"""/bin/sh -c """
"""'[ -f /lib/systemd/system/[email protected] ] && """
"""ln -s /lib/systemd/system/[email protected] """
"""/etc/systemd/system/getty.target.wants/"""
"""serial-getty@%s.service'""" % serial_con,
allow_fail=True)
else:
logging.error("parsing console tag failed, needs to be of "
"'/dev/ttyS0,115200' format.")
def execute(self, _buildenv, target):
with target:
chroot(target.path, "dpkg --purge %s" % (self.node.et.text))
