def build_cdroms(self,
build_bin=True,
build_sources=False,
cdrom_size=None):
self.repo_images = []
elog = ASCIIDocLog(self.validationpath, True)
env = None
sysrootstr = ""
if os.path.exists(self.sysrootpath):
sysrootstr = "(including sysroot packages)"
env = BuildEnv(self.xml,
self.log,
self.sysrootpath,
build_sources=build_sources,
clean=False)
else:
env = BuildEnv(self.xml,
self.log,
self.chrootpath,
build_sources=build_sources,
clean=False)
# ensure the /etc/apt/sources.list is created according to
# buil_sources, # build_bin flag, ensure to reopen it with
# the new 'sources.list'
with env:
env.seed_etc()
self.drop_rpcaptcache(env=env)
with env:
init_codename = self.xml.get_initvm_codename()
if build_bin:
elog.h1("Binary CD %s" % sysrootstr)
self.repo_images += mk_binary_cdrom(env.rfs,
self.arch,
self.codename,
init_codename,
self.xml,
self.builddir,
self.log,
cdrom_size=cdrom_size)
if build_sources:
elog.h1("Source CD %s" % sysrootstr)
try:
self.repo_images += mk_source_cdrom(env.rfs,
self.arch,
self.codename,
init_codename,
self.builddir,
self.log,
cdrom_size=cdrom_size,
xml=self.xml)
except SystemError as e:
# e.g. no deb-src urls specified
elog.printo(str(e))
def build_cdroms(self,
build_bin=True,
build_sources=False,
cdrom_size=None):
self.repo_images = []
env = None
sysrootstr = ""
if os.path.exists(self.sysrootpath):
sysrootstr = "(including sysroot packages)"
env = BuildEnv(self.xml,
self.sysrootpath,
build_sources=build_sources,
clean=False)
else:
env = BuildEnv(self.xml,
self.chrootpath,
build_sources=build_sources,
clean=False)
# ensure the /etc/apt/sources.list is created according to
# buil_sources, # build_bin flag, ensure to reopen it with
# the new 'sources.list'
with env:
env.seed_etc()
self.drop_rpcaptcache(env=env)
with env:
init_codename = self.xml.get_initvm_codename()
if build_bin:
validation.info("Binary CD %s", sysrootstr)
self.repo_images += mk_binary_cdrom(env.rfs, self.arch,
self.codename,
init_codename, self.xml,
self.builddir)
if build_sources:
if not cdrom_size and self.xml.has("src-cdrom/size"):
cdrom_size = size_to_int(self.xml.text("src-cdrom/size"))
validation.info("Source CD %s", sysrootstr)
try:
self.repo_images += mk_source_cdrom(env.rfs,
self.arch,
self.codename,
init_codename,
self.builddir,
cdrom_size=cdrom_size,
xml=self.xml)
except SystemError as e:
# e.g. no deb-src urls specified
validation.error(str(e))
def build_host_sysroot(self, pkgs, hostsysrootpath):
do('rm -rf %s; mkdir "%s"' % (hostsysrootpath, hostsysrootpath))
self.host_sysrootenv = BuildEnv(self.xml,
hostsysrootpath,
clean=True,
arch="amd64",
hostsysroot=True)
# Import keyring
self.host_sysrootenv.import_keys()
logging.info("Keys imported")
with self.host_sysrootenv:
try:
cache = self.get_rpcaptcache(env=self.host_sysrootenv,
norecommend=True)
cache.update()
except Exception as e:
raise AptCacheUpdateError(e)
for p in pkgs:
try:
cache.mark_install(p, None)
except KeyError:
logging.exception("No Package %s", p)
except SystemError:
logging.exception(
"Unable to correct problems in "
"package %s", p)
try:
cache.commit()
except SystemError as e:
logging.exception("Commiting changes failed")
raise AptCacheCommitError(str(e))
self.gen_licenses("sysroot-host", self.host_sysrootenv,
[p.name for p in cache.get_installed_pkgs()])
# This is just a sysroot, some directories
# need to be removed.
#
# This can move into finetuning in the
# second implementation step.
self.host_sysrootenv.rfs.rmtree('/boot')
self.host_sysrootenv.rfs.rmtree('/dev')
self.host_sysrootenv.rfs.rmtree('/etc')
self.host_sysrootenv.rfs.rmtree('/home')
self.host_sysrootenv.rfs.rmtree('/media')
self.host_sysrootenv.rfs.rmtree('/mnt')
self.host_sysrootenv.rfs.rmtree('/proc')
self.host_sysrootenv.rfs.rmtree('/root')
self.host_sysrootenv.rfs.rmtree('/run')
self.host_sysrootenv.rfs.rmtree('/sys')
self.host_sysrootenv.rfs.rmtree('/tmp')
self.host_sysrootenv.rfs.rmtree('/var')
def build_host_sysroot(self, pkgs, hostsysrootpath):
self.log.do('rm -rf %s; mkdir "%s"' %
(hostsysrootpath, hostsysrootpath))
self.host_sysrootenv = BuildEnv(self.xml,
self.log,
hostsysrootpath,
clean=True,
arch="amd64")
# Import keyring
self.host_sysrootenv.import_keys()
self.log.printo("Keys imported")
with self.host_sysrootenv:
try:
cache = self.get_rpcaptcache(env=self.host_sysrootenv,
norecommend=True)
cache.update()
except Exception as e:
raise AptCacheUpdateError(e)
for p in pkgs:
try:
cache.mark_install(p, None)
except KeyError:
self.log.printo("No Package " + p)
except SystemError as e:
self.log.printo("Error: Unable to correct problems "
"in package %s (%s)" % (p, str(e)))
try:
cache.commit()
except SystemError as e:
self.log.printo("commiting changes failed: %s" % str(e))
raise AptCacheCommitError(str(e))
# This is just a sysroot, some directories
# need to be removed.
#
# This can move into finetuning in the
# second implementation step.
self.host_sysrootenv.rfs.rmtree('/boot')
self.host_sysrootenv.rfs.rmtree('/dev')
self.host_sysrootenv.rfs.rmtree('/etc')
self.host_sysrootenv.rfs.rmtree('/home')
self.host_sysrootenv.rfs.rmtree('/media')
self.host_sysrootenv.rfs.rmtree('/mnt')
self.host_sysrootenv.rfs.rmtree('/proc')
self.host_sysrootenv.rfs.rmtree('/root')
self.host_sysrootenv.rfs.rmtree('/run')
self.host_sysrootenv.rfs.rmtree('/sys')
self.host_sysrootenv.rfs.rmtree('/tmp')
self.host_sysrootenv.rfs.rmtree('/var')
def set_xml(self, xmlpath):
# Use supplied XML file, if given, otherwise change to source.xml
if not xmlpath:
xmlpath = os.path.join(self.builddir, "source.xml")
newxml = ElbeXML(xmlpath,
buildtype=self.override_buildtype,
skip_validate=self.skip_validate,
url_validation=self.url_validation)
# New XML file has to have the same architecture
oldarch = self.xml.text("project/arch", key="arch")
newarch = newxml.text("project/arch", key="arch")
if newarch != oldarch:
raise IncompatibleArchitectureException(oldarch, newarch)
# Throw away old APT cache, targetfs and buildenv
self.targetfs = None
self.buildenv = None
# dont create sysroot instance, it should be build from scratch
# each time, because the pkglist including the -dev packages is
# tracked nowhere.
self.sysrootenv = None
self.log.do('rm -rf %s' % self.sysrootpath)
self.xml = newxml
# Create a new BuildEnv instance, if we have a build directory
if self.has_full_buildenv():
self.buildenv = BuildEnv(self.xml,
self.log,
self.chrootpath,
clean=False)
# Create TargetFs instance, if the target directory exists.
# We use the old content of the directory if no rebuild is done, so
# don't clean it (yet).
if os.path.exists(self.targetpath):
self.targetfs = TargetFs(self.targetpath,
self.log,
self.xml,
clean=False)
else:
self.targetfs = None
def set_xml(self, xmlpath):
# Use supplied XML file, if given, otherwise change to source.xml
if not xmlpath:
xmlpath = os.path.join(self.builddir, "source.xml")
newxml = ElbeXML(xmlpath,
buildtype=self.override_buildtype,
skip_validate=self.skip_validate,
skip_urlcheck=self.skip_urlcheck)
# New XML file has to have the same architecture
oldarch = self.xml.text("project/arch", key="arch")
newarch = newxml.text("project/arch", key="arch")
if newarch != oldarch:
raise IncompatibleArchitectureException(oldarch, newarch)
# Throw away old APT cache, targetfs and buildenv
self._rpcaptcache = None
self.targetfs = None
self.buildenv = None
self.xml = newxml
# Create a new BuildEnv instance, if we have a build directory
if self.has_full_buildenv():
self.buildenv = BuildEnv(self.xml, self.log, self.chrootpath)
# Create TargetFs instance, if the target directory exists.
# We use the old content of the directory if no rebuild is done, so
# don't clean it (yet).
if os.path.exists(self.targetpath):
self.targetfs = TargetFs(self.targetpath,
self.log,
self.buildenv.xml,
clean=False)
else:
self.targetfs = None
def build(self,
build_bin=False,
build_sources=False,
cdrom_size=None,
skip_pkglist=False,
skip_pbuild=False):
# pylint: disable=too-many-arguments
# pylint: disable=too-many-locals
# pylint: disable=too-many-statements
# pylint: disable=too-many-branches
# Write the log header
self.write_log_header()
# Validate Apt Sources
if build_sources:
m = ValidationMode.CHECK_ALL
else:
m = ValidationMode.CHECK_BINARIES
self.xml.validate_apt_sources(m, self.arch)
if self.xml.has('target/pbuilder') and not skip_pbuild:
if not os.path.exists(os.path.join(self.builddir, "pbuilder")):
self.create_pbuilder(cross=False)
for p in self.xml.node('target/pbuilder'):
self.pbuild(p)
# the package might be needed by a following pbuild, so update
# the project repo that it can be installed in as
# build-dependency
self.repo.finalize()
# To avoid update cache errors, the project repo needs to have
# Release and Packages files, even if it's empty. So don't do this
# in the if case above!
self.repo.finalize()
# Create the build environment, if it does not a valid one
# self.buildenv might be set when we come here.
# However, if its not a full_buildenv, we specify clean here,
# so it gets rebuilt properly.
if not self.has_full_buildenv():
do('mkdir -p "%s"' % self.chrootpath)
self.buildenv = BuildEnv(self.xml,
self.chrootpath,
build_sources=build_sources,
clean=True)
skip_pkglist = False
# Import keyring
self.buildenv.import_keys()
logging.info("Keys imported")
# Install packages
if not skip_pkglist:
self.install_packages(self.buildenv)
try:
self.buildenv.rfs.dump_elbeversion(self.xml)
except IOError:
logging.exception("Dump elbeversion failed")
# Extract target FS. We always create a new instance here with
# clean=true, because we want a pristine directory.
self.targetfs = TargetFs(self.targetpath,
self.buildenv.xml,
clean=True)
os.chdir(self.buildenv.rfs.fname(''))
extract_target(self.buildenv.rfs, self.xml, self.targetfs,
self.get_rpcaptcache())
# Package validation and package list
if not skip_pkglist:
pkgs = self.xml.xml.node("/target/pkg-list")
if self.xml.has("fullpkgs"):
check_full_pkgs(pkgs, self.xml.xml.node("/fullpkgs"),
self.get_rpcaptcache())
else:
check_full_pkgs(pkgs, None, self.get_rpcaptcache())
dump_fullpkgs(self.xml, self.buildenv.rfs, self.get_rpcaptcache())
self.xml.dump_elbe_version()
self.targetfs.write_fstab(self.xml)
# Dump ELBE version
try:
self.targetfs.dump_elbeversion(self.xml)
except MemoryError:
logging.exception("Dump elbeversion failed")
# install packages for buildenv
if not skip_pkglist:
self.install_packages(self.buildenv, buildenv=True)
# Write source.xml
try:
sourcexmlpath = os.path.join(self.builddir, "source.xml")
self.xml.xml.write(sourcexmlpath)
except MemoryError:
logging.exception("Write source.xml failed (archive to huge?)")
# Elbe report
cache = self.get_rpcaptcache()
tgt_pkgs = elbe_report(self.xml, self.buildenv, cache, self.targetfs)
# chroot' licenses
self.gen_licenses("chroot", self.buildenv,
[p.name for p in cache.get_installed_pkgs()])
self.gen_licenses("target", self.buildenv, tgt_pkgs)
# Use some handwaving to determine grub version
grub_arch = "ia32" if self.arch == "i386" else self.arch
grub_fw_type = []
grub_version = 0
if self.get_rpcaptcache().is_installed('grub-pc'):
grub_version = 202
grub_fw_type.append("bios")
if self.get_rpcaptcache().is_installed('grub-efi-%s-bin' % grub_arch):
grub_version = 202
grub_tgt = "x86_64" if self.arch == "amd64" else self.arch
grub_fw_type.extend(["efi", grub_tgt + "-efi"])
if (self.get_rpcaptcache().is_installed('shim-signed')
and self.get_rpcaptcache().is_installed(
'grub-efi-%s-signed' % grub_arch)):
grub_version = 202
grub_fw_type.append("shimfix")
if self.get_rpcaptcache().is_installed('grub-legacy'):
logging.warning("package grub-legacy is installed, "
"this is obsolete.")
grub_version = 97
grub_fw_type.append("bios")
elif not grub_fw_type:
logging.warning(
"neither package grub-pc nor grub-efi-%s-bin "
"are installed, skipping grub", grub_arch)
self.targetfs.part_target(self.builddir, grub_version, grub_fw_type)
self.build_cdroms(build_bin,
build_sources,
cdrom_size,
tgt_pkg_lst=tgt_pkgs)
if self.postbuild_file:
logging.info("Postbuild script")
cmd = ' "%s %s %s"' % (self.builddir,
self.xml.text("project/version"),
self.xml.text("project/name"))
do(self.postbuild_file + cmd, allow_fail=True)
do_prj_finetuning(self.xml, self.buildenv, self.targetfs,
self.builddir)
self.targetfs.pack_images(self.builddir)
system('cat "%s"' % self.validationpath)
def build_cdroms(self,
build_bin=True,
build_sources=False,
cdrom_size=None,
tgt_pkg_lst=None):
# pylint: disable=too-many-branches
# pylint: disable=too-many-locals
self.repo_images = []
env = None
sysrootstr = ""
if os.path.exists(self.sysrootpath):
sysrootstr = "(including sysroot packages)"
env = BuildEnv(self.xml,
self.sysrootpath,
build_sources=build_sources,
clean=False)
else:
env = BuildEnv(self.xml,
self.chrootpath,
build_sources=build_sources,
clean=False)
# ensure the /etc/apt/sources.list is created according to
# buil_sources, # build_bin flag, ensure to reopen it with
# the new 'sources.list'
with env:
env.seed_etc()
self.drop_rpcaptcache(env=env)
with env:
init_codename = self.xml.get_initvm_codename()
if build_bin:
validation.info("Binary CD %s", sysrootstr)
self.repo_images += mk_binary_cdrom(env.rfs, self.arch,
self.codename,
init_codename, self.xml,
self.builddir)
if build_sources:
if not cdrom_size and self.xml.has("src-cdrom/size"):
cdrom_size = size_to_int(self.xml.text("src-cdrom/size"))
validation.info("Source CD %s", sysrootstr)
# Target component
cache = self.get_rpcaptcache(env=self.buildenv)
tgt_lst = cache.get_corresponding_source_packages(
pkg_lst=tgt_pkg_lst)
components = {"target": (self.targetfs, cache, tgt_lst)}
# Main component
main_lst = []
if self.xml is not None:
tmp_lst = []
for pkg_node in self.xml.node("debootstrappkgs"):
pkg = XMLPackage(pkg_node, self.arch)
tmp_lst.append(pkg.name)
main_lst = cache.get_corresponding_source_packages(
pkg_lst=tmp_lst)
components["main"] = (env.rfs, cache, main_lst)
# Added component
other_components = [(env, "added")]
# Let's build a list of (build_env, name) for the
# other RFS if they exist
host_sysroot_path = os.path.join(self.sdkpath, "sysroots",
"host")
for path, name in [(self.chrootpath, "chroot"),
(host_sysroot_path, "sysroot-host")]:
if os.path.exists(path) and env.path != path:
tmp_env = BuildEnv(self.xml, path)
with tmp_env:
tmp_env.seed_etc()
other_components.append((tmp_env, name))
# Now let's generate the correct (rfs, cache, pkg_lst)
# components using the full installed packages
for build_env, name in other_components:
cache = self.get_rpcaptcache(env=build_env)
src_lst = cache.get_corresponding_source_packages()
components[name] = (build_env.rfs, cache, src_lst)
try:
# Using kwargs here allows us to avoid making
# special case for when self.xml is None
kwargs = {"cdrom_size": cdrom_size, "xml": self.xml}
if self.xml is not None:
kwargs["mirror"] = self.xml.get_primary_mirror(
env.rfs.fname("cdrom"))
self.repo_images += mk_source_cdrom(
components, self.codename, init_codename,
self.builddir, **kwargs)
except SystemError as e:
# e.g. no deb-src urls specified
validation.error(str(e))
def build_sysroot(self):
do('rm -rf %s; mkdir "%s"' % (self.sysrootpath, self.sysrootpath))
self.sysrootenv = BuildEnv(self.xml, self.sysrootpath, clean=True)
# Import keyring
self.sysrootenv.import_keys()
logging.info("Keys imported")
self.install_packages(self.sysrootenv, buildenv=False)
# ignore packages from debootstrap
tpkgs = self.xml.get_target_packages()
bspkgs = self.xml.node("debootstrappkgs")
ignore_pkgs = [p.et.text for p in bspkgs if p.et.text not in tpkgs]
ignore_dev_pkgs = []
if self.xml.has('target/pkg-blacklist/sysroot'):
ignore_dev_pkgs = [
p.et.text
for p in self.xml.node("target/pkg-blacklist/sysroot")
]
with self.sysrootenv:
try:
cache = self.get_rpcaptcache(env=self.sysrootenv)
cache.update()
except Exception as e:
raise AptCacheUpdateError(e)
try:
cache.mark_install_devpkgs(set(ignore_pkgs),
set(ignore_dev_pkgs))
except SystemError as e:
logging.exception("Mark install devpkgs failed")
try:
cache.commit()
except SystemError as e:
logging.exception("Commiting changes failed")
raise AptCacheCommitError(str(e))
self.gen_licenses("sysroot-target", self.sysrootenv,
[p.name for p in cache.get_installed_pkgs()])
try:
self.sysrootenv.rfs.dump_elbeversion(self.xml)
except IOError:
logging.exception("Dump elbeversion into sysroot failed")
sysrootfilelist = os.path.join(self.builddir, "sysroot-filelist")
with self.sysrootenv.rfs:
chroot(self.sysrootpath, "/usr/bin/symlinks -cr /usr/lib")
paths = self.get_sysroot_paths()
do("rm %s" % sysrootfilelist, allow_fail=True)
os.chdir(self.sysrootpath)
for p in paths:
do('find -path "%s" >> %s' % (p, sysrootfilelist))
# include /lib if it is a symlink (buster and later)
if os.path.islink(self.sysrootpath + '/lib'):
with open(sysrootfilelist, 'a') as filelist_fd:
filelist_fd.write('./lib')
do("tar cfJ %s/sysroot.tar.xz -C %s -T %s" %
(self.builddir, self.sysrootpath, sysrootfilelist))
def __init__(self,
builddir,
xmlpath=None,
name=None,
override_buildtype=None,
skip_validate=False,
url_validation=ValidationMode.CHECK_ALL,
rpcaptcache_notifier=None,
private_data=None,
postbuild_file=None,
presh_file=None,
postsh_file=None,
savesh_file=None):
# pylint: disable=too-many-arguments
self.builddir = os.path.abspath(str(builddir))
self.chrootpath = os.path.join(self.builddir, "chroot")
self.targetpath = os.path.join(self.builddir, "target")
self.sysrootpath = os.path.join(self.builddir, "sysroot")
self.sdkpath = os.path.join(self.builddir, "sdk")
self.validationpath = os.path.join(self.builddir, "validation.txt")
self.name = name
self.override_buildtype = override_buildtype
self.skip_validate = skip_validate
self.url_validation = url_validation
self.postbuild_file = postbuild_file
self.presh_file = presh_file
self.postsh_file = postsh_file
self.savesh_file = savesh_file
self.private_data = private_data
# Apt-Cache will be created on demand with the specified notifier by
# the get_rpcaptcache method
self._rpcaptcache = None
self.rpcaptcache_notifier = rpcaptcache_notifier
# Initialise Repo Images to Empty list.
self.repo_images = []
self.orig_fname = None
self.orig_files = []
# Use supplied XML file, if given, otherwise use the source.xml
# file of the project
if xmlpath:
self.xml = ElbeXML(xmlpath,
buildtype=override_buildtype,
skip_validate=skip_validate,
url_validation=url_validation)
else:
sourcexmlpath = os.path.join(self.builddir, "source.xml")
self.xml = ElbeXML(sourcexmlpath,
buildtype=override_buildtype,
skip_validate=skip_validate,
url_validation=url_validation)
self.arch = self.xml.text("project/arch", key="arch")
self.codename = self.xml.text("project/suite")
if not self.name:
self.name = self.xml.text("project/name")
self.repo = ProjectRepo(self.arch, self.codename,
os.path.join(self.builddir, "repo"))
# Create BuildEnv instance, if the chroot directory exists and
# has an etc/elbe_version
if os.path.exists(self.chrootpath):
self.buildenv = BuildEnv(self.xml, self.chrootpath, clean=False)
else:
self.buildenv = None
# Create TargetFs instance, if the target directory exists
if os.path.exists(self.targetpath) and self.buildenv:
self.targetfs = TargetFs(self.targetpath,
self.buildenv.xml,
clean=False)
else:
self.targetfs = None
# don't create sysroot instance, it should be built from scratch
# each time, because the pkglist including the -dev packages is
# tracked nowhere.
self.sysrootenv = None
do('rm -rf %s' % self.sysrootpath)
# same for host_sysroot instance recreate it in any case
self.host_sysrootenv = None
def build(self,
build_bin=False,
build_sources=False,
cdrom_size=None,
skip_pkglist=False,
skip_pbuild=False):
# pylint: disable=too-many-arguments
# pylint: disable=too-many-locals
# pylint: disable=too-many-statements
# pylint: disable=too-many-branches
# Write the log header
self.write_log_header()
# Validate Apt Sources
if build_sources:
m = ValidationMode.CHECK_ALL
else:
m = ValidationMode.CHECK_BINARIES
self.xml.validate_apt_sources(m, self.arch)
if self.xml.has('target/pbuilder') and not skip_pbuild:
if not os.path.exists(os.path.join(self.builddir, "pbuilder")):
self.create_pbuilder()
for p in self.xml.node('target/pbuilder'):
self.pbuild(p)
# the package might be needed by a following pbuild, so update
# the project repo that it can be installed in as
# build-dependency
self.repo.finalize()
# To avoid update cache errors, the project repo needs to have
# Release and Packages files, even if it's empty. So don't do this
# in the if case above!
self.repo.finalize()
# Create the build environment, if it does not a valid one
# self.buildenv might be set when we come here.
# However, if its not a full_buildenv, we specify clean here,
# so it gets rebuilt properly.
if not self.has_full_buildenv():
self.log.do('mkdir -p "%s"' % self.chrootpath)
self.buildenv = BuildEnv(self.xml,
self.log,
self.chrootpath,
build_sources=build_sources,
clean=True)
skip_pkglist = False
# Import keyring
self.buildenv.import_keys()
self.log.printo("Keys imported")
# Install packages
if not skip_pkglist:
self.install_packages(self.buildenv)
try:
self.buildenv.rfs.dump_elbeversion(self.xml)
except IOError:
self.log.printo("dump elbeversion failed")
# Extract target FS. We always create a new instance here with
# clean=true, because we want a pristine directory.
self.targetfs = TargetFs(self.targetpath,
self.log,
self.buildenv.xml,
clean=True)
os.chdir(self.buildenv.rfs.fname(''))
extract_target(self.buildenv.rfs, self.xml, self.targetfs, self.log,
self.get_rpcaptcache())
# The validation file is created using check_full_pkgs() and
# elbe_report(), both opening the file in append mode. So if an
# old validation file already exists, it must be deleted first.
if os.path.isfile(self.validationpath):
os.unlink(self.validationpath)
# Package validation and package list
if not skip_pkglist:
pkgs = self.xml.xml.node("/target/pkg-list")
if self.xml.has("fullpkgs"):
check_full_pkgs(pkgs, self.xml.xml.node("/fullpkgs"),
self.validationpath, self.get_rpcaptcache())
else:
check_full_pkgs(pkgs, None, self.validationpath,
self.get_rpcaptcache())
dump_fullpkgs(self.xml, self.buildenv.rfs, self.get_rpcaptcache())
self.xml.dump_elbe_version()
self.targetfs.write_fstab(self.xml)
# Dump ELBE version
try:
self.targetfs.dump_elbeversion(self.xml)
except MemoryError:
self.log.printo("dump elbeversion failed")
# install packages for buildenv
if not skip_pkglist:
self.install_packages(self.buildenv, buildenv=True)
# Write source.xml
try:
sourcexmlpath = os.path.join(self.builddir, "source.xml")
self.xml.xml.write(sourcexmlpath)
except MemoryError:
self.log.printo("write source.xml failed (archive to huge?)")
# Elbe report
reportpath = os.path.join(self.builddir, "elbe-report.txt")
elbe_report(self.xml, self.buildenv, self.get_rpcaptcache(),
reportpath, self.validationpath, self.targetfs)
# the current license code raises an exception that interrupts the hole
# build if a licence can't be converted to utf-8. Exception handling
# can be removed as soon as the licence code is more stable
lic_err = False
try:
f = io.open(os.path.join(self.builddir, "licence.txt"),
"w+",
encoding='utf-8')
self.buildenv.rfs.write_licenses(
f, self.log, os.path.join(self.builddir, "licence.xml"))
except Exception:
self.log.printo("error during generating licence.txt/xml")
self.log.printo(sys.exc_info()[0])
lic_err = True
finally:
f.close()
if lic_err:
os.remove(os.path.join(self.builddir, "licence.txt"))
os.remove(os.path.join(self.builddir, "licence.xml"))
# Use some handwaving to determine grub version
# jessie and wheezy grubs are 2.0 but differ in behaviour
#
# We might also want support for legacy grub
if (self.get_rpcaptcache().is_installed('grub-pc')
and self.get_rpcaptcache().is_installed('grub-efi-amd64-bin')):
grub_version = 202
grub_fw_type = "hybrid"
elif self.get_rpcaptcache().is_installed('grub-pc'):
if self.codename == "wheezy":
grub_version = 199
else:
grub_version = 202
grub_fw_type = "bios"
elif self.get_rpcaptcache().is_installed('grub-efi-amd64'):
grub_version = 202
grub_fw_type = "efi"
elif self.get_rpcaptcache().is_installed('grub-legacy'):
self.log.printo("package grub-legacy is installed, "
"this is obsolete, skipping grub")
grub_version = 0
grub_fw_type = ""
else:
self.log.printo("package grub-pc is not installed, skipping grub")
# version 0 == skip_grub
grub_version = 0
grub_fw_type = ""
self.targetfs.part_target(self.builddir, grub_version, grub_fw_type)
self.build_cdroms(build_bin, build_sources, cdrom_size)
if self.postbuild_file:
self.log.h2("postbuild script:")
self.log.do(self.postbuild_file + ' "%s %s %s"' %
(self.builddir, self.xml.text("project/version"),
self.xml.text("project/name")),
allow_fail=True)
do_prj_finetuning(self.xml, self.log, self.buildenv, self.targetfs,
self.builddir)
self.targetfs.pack_images(self.builddir)
system('cat "%s"' % self.validationpath)
def build_sysroot(self):
self.log.do('rm -rf %s; mkdir "%s"' %
(self.sysrootpath, self.sysrootpath))
self.sysrootenv = BuildEnv(self.xml,
self.log,
self.sysrootpath,
clean=True)
# Import keyring
self.sysrootenv.import_keys()
self.log.printo("Keys imported")
self.install_packages(self.sysrootenv, buildenv=False)
# ignore packages from debootstrap
tpkgs = self.xml.get_target_packages()
bspkgs = self.xml.node("debootstrappkgs")
ignore_pkgs = [p.et.text for p in bspkgs if p.et.text not in tpkgs]
ignore_dev_pkgs = []
if self.xml.has('target/pkg-blacklist/sysroot'):
ignore_dev_pkgs = [
p.et.text
for p in self.xml.node("target/pkg-blacklist/sysroot")
]
with self.sysrootenv:
try:
self.get_rpcaptcache(env=self.sysrootenv).update()
except Exception as e:
raise AptCacheUpdateError(e)
try:
self.get_rpcaptcache(env=self.sysrootenv).mark_install_devpkgs(
set(ignore_pkgs), set(ignore_dev_pkgs))
except SystemError as e:
self.log.printo("mark install devpkgs failed: %s" % str(e))
try:
self.get_rpcaptcache(env=self.sysrootenv).commit()
except SystemError as e:
self.log.printo("commiting changes failed: %s" % str(e))
raise AptCacheCommitError(str(e))
try:
self.sysrootenv.rfs.dump_elbeversion(self.xml)
except IOError:
self.log.printo("dump elbeversion into sysroot failed")
sysrootfilelist = os.path.join(self.builddir, "sysroot-filelist")
with self.sysrootenv.rfs:
self.log.do("chroot %s /usr/bin/symlinks -cr /usr/lib" %
self.sysrootpath)
paths = self.get_sysroot_paths()
self.log.do("rm %s" % sysrootfilelist, allow_fail=True)
os.chdir(self.sysrootpath)
for p in paths:
self.log.do('find -path "%s" >> %s' % (p, sysrootfilelist))
self.log.do("tar cfJ %s/sysroot.tar.xz -C %s -T %s" %
(self.builddir, self.sysrootpath, sysrootfilelist))
def __init__(self,
builddir,
xmlpath=None,
logpath=None,
name=None,
override_buildtype=None,
skip_validate=False,
skip_urlcheck=False,
rpcaptcache_notifier=None,
private_data=None,
postbuild_file=None,
presh_file=None,
postsh_file=None,
savesh_file=None):
self.builddir = os.path.abspath(str(builddir))
self.chrootpath = os.path.join(self.builddir, "chroot")
self.targetpath = os.path.join(self.builddir, "target")
self.name = name
self.override_buildtype = override_buildtype
self.skip_validate = skip_validate
self.skip_urlcheck = skip_urlcheck
self.postbuild_file = postbuild_file
self.presh_file = presh_file
self.postsh_file = postsh_file
self.savesh_file = savesh_file
self.private_data = private_data
# Apt-Cache will be created on demand with the specified notifier by
# the get_rpcaptcache method
self._rpcaptcache = None
self.rpcaptcache_notifier = rpcaptcache_notifier
# Initialise Repo Images to Empty list.
self.repo_images = []
# Use supplied XML file, if given, otherwise use the source.xml
# file of the project
if xmlpath:
self.xml = ElbeXML(xmlpath,
buildtype=override_buildtype,
skip_validate=skip_validate,
skip_urlcheck=skip_urlcheck)
else:
sourcexmlpath = os.path.join(self.builddir, "source.xml")
self.xml = ElbeXML(sourcexmlpath,
buildtype=override_buildtype,
skip_validate=skip_validate,
skip_urlcheck=skip_urlcheck)
# If logpath is given, use an AsciiDocLog instance, otherwise log
# to stdout
if logpath:
self.log = ASCIIDocLog(logpath)
else:
self.log = StdoutLog()
# Create BuildEnv instance, if the chroot directory exists and
# has an etc/elbe_version
if self.has_full_buildenv():
self.buildenv = BuildEnv(self.xml, self.log, self.chrootpath)
else:
self.buildenv = None
self.targetfs = None
return
# Create TargetFs instance, if the target directory exists
if os.path.exists(self.targetpath):
self.targetfs = TargetFs(self.targetpath,
self.log,
self.buildenv.xml,
clean=False)
else:
self.targetfs = None
def build(self,
skip_debootstrap=False,
build_bin=False,
build_sources=False,
cdrom_size=None,
debug=False,
skip_pkglist=False):
# Write the log header
self.write_log_header()
# Create the build environment, if it does not exist yet
if not self.buildenv:
self.log.do('mkdir -p "%s"' % self.chrootpath)
self.buildenv = BuildEnv(self.xml,
self.log,
self.chrootpath,
build_sources=build_sources)
skip_pkglist = False
# Install packages
if not skip_pkglist:
self.install_packages()
try:
self.buildenv.rfs.dump_elbeversion(self.xml)
except IOError:
self.log.printo("dump elbeversion failed")
# Extract target FS. We always create a new instance here with
# clean=true, because we want a pristine directory.
self.targetfs = TargetFs(self.targetpath,
self.log,
self.buildenv.xml,
clean=True)
os.chdir(self.buildenv.rfs.fname(''))
extract_target(self.buildenv.rfs, self.xml, self.targetfs, self.log,
self.get_rpcaptcache())
# Package validation and package list
if not skip_pkglist:
validationpath = os.path.join(self.builddir, "validation.txt")
pkgs = self.xml.xml.node("/target/pkg-list")
if self.xml.has("fullpkgs"):
check_full_pkgs(pkgs, self.xml.xml.node("/fullpkgs"),
validationpath, self.get_rpcaptcache())
else:
check_full_pkgs(pkgs, None, validationpath,
self.get_rpcaptcache())
dump_fullpkgs(self.xml, self.buildenv.rfs, self.get_rpcaptcache())
self.xml.dump_elbe_version()
self.targetfs.write_fstab(self.xml)
# Dump ELBE version
try:
self.targetfs.dump_elbeversion(self.xml)
except MemoryError:
self.log.printo("dump elbeversion failed")
# install packages for buildenv
if not skip_pkglist:
self.install_packages(buildenv=True)
# Write source.xml
try:
sourcexmlpath = os.path.join(self.builddir, "source.xml")
self.xml.xml.write(sourcexmlpath)
except MemoryError:
self.log.printo("write source.xml failed (archive to huge?)")
# Elbe report
reportpath = os.path.join(self.builddir, "elbe-report.txt")
elbe_report(self.xml, self.buildenv, self.get_rpcaptcache(),
reportpath, self.targetfs)
# Licenses
f = open(os.path.join(self.builddir, "licence.txt"), "w+")
self.buildenv.rfs.write_licenses(f, self.log)
f.close()
# Read arch and codename from xml
arch = self.xml.text("project/arch", key="arch")
codename = self.xml.text("project/suite")
# Use some handwaving to determine grub version
# jessie and wheezy grubs are 2.0 but differ in behaviour
#
# We might also want support for legacy grub
if self.get_rpcaptcache().is_installed('grub-pc'):
if codename == "jessie":
grub_version = 202
else:
grub_version = 199
elif self.get_rpcaptcache().is_installed('grub-legacy'):
self.log.printo(
"package grub-legacy is installed, this is obsolete, skipping grub"
)
grub_version = 0
else:
self.log.printo("package grub-pc is not installed, skipping grub")
# version 0 == skip_grub
grub_version = 0
self.targetfs.part_target(self.builddir, grub_version)
# Build cdrom images
self.repo_images = []
with self.buildenv:
init_codename = self.xml.get_initvm_codename()
if build_bin:
self.repo_images += mk_binary_cdrom(self.buildenv.rfs,
arch,
codename,
init_codename,
self.xml,
self.builddir,
self.log,
cdrom_size=cdrom_size)
if build_sources:
try:
self.repo_images += mk_source_cdrom(self.buildenv.rfs,
arch,
codename,
init_codename,
self.builddir,
self.log,
cdrom_size=cdrom_size)
except SystemError as e:
# e.g. no deb-src urls specified
self.log.printo(str(e))
if self.postbuild_file:
self.log.h2("postbuild script:")
self.log.do(self.postbuild_file + ' "%s %s %s"' %
(self.builddir, self.xml.text("project/version"),
self.xml.text("project/name")),
allow_fail=True)
os.system('cat "%s"' % os.path.join(self.builddir, "validation.txt"))
def build(self,
skip_debootstrap=False,
build_bin=False,
build_sources=False,
cdrom_size=None,
debug=False,
skip_pkglist=False,
skip_pbuild=False):
# Write the log header
self.write_log_header()
# Validate Apt Sources
m = ValidationMode.NO_CHECK
if build_bin:
m = ValidationMode.CHECK_BINARIES
if build_sources:
m = ValidationMode.CHECK_ALL
self.xml.validate_apt_sources(m, self.arch)
if (self.xml.has('target/pbuilder') and not skip_pbuild):
if not os.path.exists(os.path.join(self.builddir, "pbuilder")):
self.create_pbuilder()
for p in self.xml.node('target/pbuilder'):
self.pbuild(p)
# the package might be needed by a following pbuild, so update
# the project repo that it can be installed in as
# build-dependency
self.repo.finalize()
# To avoid update cache errors, the project repo needs to have
# Release and Packages files, even if it's empty. So don't do this
# in the if case above!
self.repo.finalize()
# Create the build environment, if it does not a valid one
# self.buildenv might be set when we come here.
# However, if its not a full_buildenv, we specify clean here,
# so it gets rebuilt properly.
if not self.has_full_buildenv():
self.log.do('mkdir -p "%s"' % self.chrootpath)
self.buildenv = BuildEnv(self.xml,
self.log,
self.chrootpath,
build_sources=build_sources,
clean=True)
skip_pkglist = False
# Import keyring
self.buildenv.import_keys()
self.log.printo("Keys imported")
# Install packages
if not skip_pkglist:
self.install_packages()
try:
self.buildenv.rfs.dump_elbeversion(self.xml)
except IOError:
self.log.printo("dump elbeversion failed")
# Extract target FS. We always create a new instance here with
# clean=true, because we want a pristine directory.
self.targetfs = TargetFs(self.targetpath,
self.log,
self.buildenv.xml,
clean=True)
os.chdir(self.buildenv.rfs.fname(''))
extract_target(self.buildenv.rfs, self.xml, self.targetfs, self.log,
self.get_rpcaptcache())
# Package validation and package list
if not skip_pkglist:
validationpath = os.path.join(self.builddir, "validation.txt")
pkgs = self.xml.xml.node("/target/pkg-list")
if self.xml.has("fullpkgs"):
check_full_pkgs(pkgs, self.xml.xml.node("/fullpkgs"),
validationpath, self.get_rpcaptcache())
else:
check_full_pkgs(pkgs, None, validationpath,
self.get_rpcaptcache())
dump_fullpkgs(self.xml, self.buildenv.rfs, self.get_rpcaptcache())
self.xml.dump_elbe_version()
self.targetfs.write_fstab(self.xml)
# Dump ELBE version
try:
self.targetfs.dump_elbeversion(self.xml)
except MemoryError:
self.log.printo("dump elbeversion failed")
# install packages for buildenv
if not skip_pkglist:
self.install_packages(buildenv=True)
# Write source.xml
try:
sourcexmlpath = os.path.join(self.builddir, "source.xml")
self.xml.xml.write(sourcexmlpath)
except MemoryError:
self.log.printo("write source.xml failed (archive to huge?)")
# Elbe report
reportpath = os.path.join(self.builddir, "elbe-report.txt")
elbe_report(self.xml, self.buildenv, self.get_rpcaptcache(),
reportpath, self.targetfs)
# Licenses
f = io.open(os.path.join(self.builddir, "licence.txt"),
"w+",
encoding='utf-8')
self.buildenv.rfs.write_licenses(
f, self.log, os.path.join(self.builddir, "licence.xml"))
f.close()
# Use some handwaving to determine grub version
# jessie and wheezy grubs are 2.0 but differ in behaviour
#
# We might also want support for legacy grub
if self.get_rpcaptcache().is_installed('grub-pc'):
if self.codename == "wheezy":
grub_version = 199
else:
grub_version = 202
elif self.get_rpcaptcache().is_installed('grub-legacy'):
self.log.printo(
"package grub-legacy is installed, this is obsolete, skipping grub"
)
grub_version = 0
else:
self.log.printo("package grub-pc is not installed, skipping grub")
# version 0 == skip_grub
grub_version = 0
self.targetfs.part_target(self.builddir, grub_version)
# Build cdrom images
self.repo_images = []
with self.buildenv:
init_codename = self.xml.get_initvm_codename()
if build_bin:
self.repo_images += mk_binary_cdrom(self.buildenv.rfs,
self.arch,
self.codename,
init_codename,
self.xml,
self.builddir,
self.log,
cdrom_size=cdrom_size)
if build_sources:
try:
self.repo_images += mk_source_cdrom(self.buildenv.rfs,
self.arch,
self.codename,
init_codename,
self.builddir,
self.log,
cdrom_size=cdrom_size,
xml=self.xml)
except SystemError as e:
# e.g. no deb-src urls specified
self.log.printo(str(e))
if self.postbuild_file:
self.log.h2("postbuild script:")
self.log.do(self.postbuild_file + ' "%s %s %s"' %
(self.builddir, self.xml.text("project/version"),
self.xml.text("project/name")),
allow_fail=True)
os.system('cat "%s"' % os.path.join(self.builddir, "validation.txt"))
