def add_build_tasks_for_first_host(cfg, host, component, host_group):
host_b = host.build_cfg
srcdir = component.vars.srcdir.get()
objdir = cfg.objdir_path(host_b, 'linux')
instdir = cfg.install_tree_path(host_b, _INST_NAME)
task = BuildTask(cfg, host_group, 'linux-headers')
task.provide_install(host_b, _INST_NAME)
task.add_empty_dir(objdir)
task.add_empty_dir(instdir)
linux_arch = None
target = cfg.target.get()
for gnu_arch in _LINUX_ARCH_MAP:
if target.startswith(gnu_arch):
linux_arch = _LINUX_ARCH_MAP[gnu_arch]
break
if linux_arch is None:
cfg.context.error('unknown Linux kernel architecture for %s' %
target)
task.add_make([
'-C', srcdir,
'O=%s' % objdir,
'ARCH=%s' % linux_arch,
'INSTALL_HDR_PATH=%s' % instdir, 'headers_install'
], objdir)
_contribute_headers_tree(cfg, host, host_group, True)
def add_build_tasks_for_first_host(cfg, host, component, host_group):
task = BuildTask(cfg, host_group, 'first-host')
def py_test_fn():
"""Test Python build step failure."""
raise ValueError('test failure')
task.add_python(py_test_fn, ())
def add_build_tasks_for_host(cfg, host, component, host_group):
task = BuildTask(cfg, host_group, 'all-hosts')
host_b = host.build_cfg
objdir = cfg.objdir_path(host_b, '%s-all' % component.copy_name)
objdir2 = cfg.objdir_path(host_b, '%s-all2' % component.copy_name)
task.add_empty_dir(objdir)
# Create objdir2 with some contents to test add_empty_dir
# removal of an existing directory.
os.makedirs(os.path.join(objdir2, 'x', 'y'))
task.add_empty_dir(objdir2)
task.add_command(['sh', '-c', 'echo all-hosts > %s/out1' % objdir])
task.add_command(['sh', '-c', 'echo all-hosts-2 > out2'], cwd=objdir)
def add_build_tasks_for_other_hosts_multilib(cfg, host, component,
host_group, multilib):
task = BuildTask(cfg, host_group,
'other-multi-%s' % multilib.build_cfg.name)
host_b = host.build_cfg
objdir = cfg.objdir_path(host_b,
'%s-other-%s' % (component.copy_name,
multilib.build_cfg.name))
task.add_empty_dir(objdir)
task.add_command(['sh', '-c',
'echo "test %s" > %s/out' % (multilib.build_cfg.name,
objdir)])
def add_build_tasks_for_other_hosts(cfg, host, component, host_group):
task = BuildTask(cfg, host_group, 'other-hosts')
host_b = host.build_cfg
objdir = cfg.objdir_path(host_b, '%s-other' % component.copy_name)
task.add_empty_dir(objdir)
def py_test_fn(arg1, arg2):
"""Test Python build step."""
out_name = os.path.join(objdir, 'out')
with open(out_name, 'w', encoding='utf-8') as outfile:
outfile.write('%s %s\n' % (arg1, arg2))
task.add_python(py_test_fn, ('test', 'python'))
def add_build_tasks_for_first_host(cfg, host, component, host_group):
task = BuildTask(cfg, host_group, 'first-host')
# Test that output gets properly redirected by build wrappers.
# The generated output must not appear in the command being
# run, to avoid the copy of the command in the log being
# confused by the tests with the command's output.
task.add_command([
'sh', '-c', 'n=0; while [ $n -lt 10 ]; do '
'if [ $(($n & 1)) -eq 0 ]; then echo $n; '
'else echo $n >& 2; fi; n=$(($n + 1)); done'
])
task.add_command([
'sh', '-c', 'n=10; while [ $n -lt 20 ]; do '
'if [ $(($n & 1)) -eq 0 ]; then echo $n; '
'else echo $n >& 2; fi; n=$(($n + 1)); done'
])
def add_build_tasks_for_host(cfg, host, component, host_group):
task = BuildTask(cfg, host_group, 'example')
host_b = host.build_cfg
instdir = cfg.install_tree_path(host_b, 'example')
instdir = os.path.join(instdir, cfg.installdir_rel.get())
task.add_empty_dir(instdir)
task.provide_install(host_b, 'example')
tree = cfg.install_tree_fstree(host_b, 'example')
host_group.contribute_package(host, tree)
task.add_command(['sh', '-c', 'echo a > %s/a1' % instdir])
task.add_command(['ln', '%s/a1' % instdir, '%s/a2' % instdir])
task.add_command(['sh', '-c', 'echo a > %s/a3' % instdir])
task.add_command(['chmod', 'a-w', '%s/a3' % instdir])
instdir2 = cfg.install_tree_path(host_b, 'example2')
instdir2 = os.path.join(instdir2, cfg.installdir_rel.get())
task.add_empty_dir(instdir2)
task.provide_install(host_b, 'example2')
tree2 = cfg.install_tree_fstree(host_b, 'example2')
host_group.contribute_package(host, tree2)
task.add_command(['sh', '-c', 'echo b > %s/b' % instdir2])
task.add_command(['ln', '-s', 'b', '%s/c' % instdir2])
def add_build_tasks_for_first_host_multilib(cfg, host, component,
host_group, multilib):
if multilib.libc is not component:
return
host_b = host.build_cfg
inst_1 = cfg.install_tree_path(host_b, 'toolchain-1')
bindir_1 = os.path.join(inst_1, cfg.bindir_rel.get())
target_build = multilib.build_cfg
srcdir = component.vars.srcdir.get()
objdir = cfg.objdir_path(target_build, 'glibc')
instdir = cfg.install_tree_path(target_build, 'glibc')
group = BuildTask(cfg, host_group, 'glibc-%s' % target_build.name)
group.depend_install(host_b, 'toolchain-1')
group.env_prepend('PATH', bindir_1)
group.provide_install(target_build, 'glibc')
init_task = BuildTask(cfg, group, 'init')
init_task.add_empty_dir(objdir)
init_task.add_empty_dir(instdir)
cfg_task = BuildTask(cfg, group, 'configure')
cfg_cmd = [
os.path.join(srcdir, 'configure'),
'--build=%s' % host_b.triplet,
'--host=%s' % target_build.triplet, '--prefix=/usr'
]
cfg_cmd.extend(component.vars.configure_opts.get())
cfg_cmd.extend(target_build.configure_vars())
cfg_cmd.append('BUILD_CC=%s' % ' '.join(host_b.tool('c-compiler')))
cfg_task.add_command(cfg_cmd, cwd=objdir)
build_task = BuildTask(cfg, group, 'build')
build_task.add_make([], objdir)
install_task = BuildTask(cfg, group, 'install')
install_task.add_make(
['-j1', 'install', 'install_root=%s' % instdir], objdir)
_contribute_sysroot_tree(cfg, host, host_group, True, multilib)
def add_build_tasks_for_first_host(cfg, host, component, host_group):
_contribute_headers_tree(cfg, host, component, host_group, True)
_contribute_shared_tree(cfg, host, component, host_group, True)
# Build glibc for the build system, and use its localedef to
# build locales. The normal glibc configure options are not
# used for this (they may only be appropriate for the target),
# so the common support for autoconf-based components isn't
# used either.
host_b = host.build_cfg
srcdir = component.vars.srcdir.get()
objdir = cfg.objdir_path(host_b, 'glibc-host')
group = BuildTask(cfg, host_group, 'glibc-host')
init_task = BuildTask(cfg, group, 'init')
init_task.add_empty_dir(objdir)
cfg_task = BuildTask(cfg, group, 'configure')
cfg_cmd = [
os.path.join(srcdir, 'configure'),
'--build=%s' % host_b.triplet,
'--host=%s' % host_b.triplet, '--prefix=/usr'
]
cfg_cmd.extend(host_b.configure_vars())
cfg_cmd.append('BUILD_CC=%s' % ' '.join(host_b.tool('c-compiler')))
cfg_task.add_command(cfg_cmd, cwd=objdir)
build_task = BuildTask(cfg, group, 'build')
build_task.add_make([], objdir)
group = BuildTask(cfg, host_group, 'glibc-locales')
group.depend('/%s/glibc-host' % host.name)
# Building the host glibc itself does not depend on the target
# compiler. Building the locales does depend on the target
# compiler (but not on the target libc), as it is used to
# determine the endianness of each locale.
group.depend_install(host_b, 'toolchain-1')
group.provide_install(host_b, 'glibc-locales')
objdir2 = cfg.objdir_path(host_b, 'glibc-locales')
instdir = cfg.install_tree_path(host_b, 'glibc-locales')
init_task = BuildTask(cfg, group, 'init')
init_task.add_empty_dir(objdir2)
init_task.add_empty_dir(instdir)
build_task = BuildTask(cfg, group, 'build')
build_task.add_python(
_generate_locales_makefile,
(cfg, component, srcdir, objdir, objdir2, instdir))
build_task.add_make([], objdir2)
def setup_build_dir(self):
"""Set up tasks and build directory for a configuration.
This function creates the specified directory for files
related to the build process (not files used in the build of
individual components), removing and recreating it as
necessary; logs go in the specified directory for logs. The
separate directory for sockets is expected to exist and to be
empty; that directory is separate because paths to Unix-domain
sockets have a small length limit, so a short path in /tmp is
appropriate rather than the longer paths used for the rest of
the build.
As well as other files, a GNUmakefile is created in the
specified directory for controlling the build, via 'make all'
unless a more selective build of particular tasks is required.
"""
relcfg = self.relcfg
top_task = BuildTask(relcfg, None, '', True)
init_task = BuildTask(relcfg, top_task, 'init', True)
host_indep_task = BuildTask(relcfg, top_task, 'host-indep', True)
fini_task = BuildTask(relcfg, top_task, 'fini', True)
host_indep_task.depend('/init')
fini_task.depend('/host-indep')
for component in relcfg.list_components():
component.cls.add_build_tasks_init(relcfg, component, init_task)
component.cls.add_build_tasks_host_indep(relcfg, component,
host_indep_task)
component.cls.add_build_tasks_fini(relcfg, component, fini_task)
first_host = True
for host in relcfg.hosts.get():
host_task = BuildTask(relcfg, top_task, host.name, True)
host_task.depend('/init')
fini_task.depend('/%s' % host.name)
for component in relcfg.list_components():
component.cls.add_build_tasks_for_host(relcfg, host, component,
host_task)
if first_host:
component.cls.add_build_tasks_for_first_host(
relcfg, host, component, host_task)
else:
component.cls.add_build_tasks_for_other_hosts(
relcfg, host, component, host_task)
for multilib in relcfg.multilibs.get():
if first_host:
component.cls.add_build_tasks_for_first_host_multilib(
relcfg, host, component, host_task, multilib)
else:
component.cls.add_build_tasks_for_other_hosts_multilib(
relcfg, host, component, host_task, multilib)
first_host = False
build_objdir = self.build_objdir
if os.access(build_objdir, os.F_OK):
shutil.rmtree(build_objdir)
os.makedirs(build_objdir)
os.makedirs(self.logdir, exist_ok=True)
makefile_text = top_task.makefile_text(self)
makefile_name = os.path.join(build_objdir, 'GNUmakefile')
with open(makefile_name, 'w', encoding='utf-8') as file:
file.write(makefile_text)
def add_build_tasks_init(cfg, component, init_group):
pkgdir_task = BuildTask(cfg, init_group, 'pkgdir')
pkgdir_task.add_create_dir(cfg.args.pkgdir)
if not cfg.args.build_source_packages:
return
# Component sources are copied in an init task to ensure that
# source packages contain the sources at the start of the
# build, even if a bug in a component's build process
# (e.g. missing or insufficient configuration of files to
# touch on checkout) results in the source directory being
# modified during the build. If a component's build process
# is known to modify the source directory intentionally, that
# component's build tasks must create a separate copy for use
# in the build.
source_copy_group = BuildTask(cfg, init_group, 'source-copy', True)
objdir = cfg.objdir_path(None, 'source-copy')
for src_component in cfg.list_source_components():
name = src_component.copy_name
srcdir = src_component.vars.srcdir.get()
srcdir_copy = os.path.join(objdir, name)
copy_task = BuildTask(cfg, source_copy_group, name)
copy_task.add_empty_dir_parent(srcdir_copy)
copy_task.add_python(
src_component.vars.vc.get().copy_without_metadata,
(srcdir, srcdir_copy))
def add_build_tasks_for_host(cfg, host, component, host_group):
# The package-input install tree is contributed to by each
# component that installs files intended to go in the final
# release package. This is an install tree for a PkgHost, not
# for a BuildCfg. This install tree is then subject to global
# manipulations (such as hard-linking identical files,
# replacing symlinks by hard links on hosts not supporting
# symlinks, and stripping binaries) to produce the
# package-output tree that corresponds to the exact data for
# the release package. Most manipulations, such as moving
# files to different locations or removing files that are
# installed by default but should not go in the final release
# packages, should be done at the level of the individual
# components; only a few manipulations are most appropriately
# done globally just before packaging.
host_group.declare_implicit_install(host, 'package-input')
pkg_out_task = BuildTask(cfg, host_group, 'package-output')
pkg_out_task.depend_install(host, 'package-input')
pkg_out_task.provide_install(host, 'package-output')
inst_in_path = cfg.install_tree_path(host, 'package-input')
inst_out_path = cfg.install_tree_path(host, 'package-output')
pkg_out_task.add_empty_dir_parent(inst_out_path)
pkg_out_task.add_command(['cp', '-a', inst_in_path, inst_out_path])
# The top-level directory in a package corresponds to the
# contents of installdir. In degenerate cases of nothing in a
# package, installdir may not have been created (although the
# package-input tree will always have been created, even if
# empty).
inst_out_main = os.path.join(inst_out_path, cfg.installdir_rel.get())
pkg_out_task.add_create_dir(inst_out_main)
if not host.have_symlinks():
pkg_out_task.add_python(replace_symlinks,
(cfg.context, inst_out_main))
pkg_out_task.add_python(fix_perms, (inst_out_main,))
pkg_out_task.add_python(hard_link_files, (cfg.context, inst_out_main))
# Creating the package-output install tree is separated from
# creating a .tar.xz package from it so that .tar.xz creation
# can run in parallel with other package format creation using
# the same tree.
pkg_task = BuildTask(cfg, host_group, 'package-tar-xz')
pkg_task.depend_install(host, 'package-output')
pkg_path = cfg.pkgdir_path(host, '.tar.xz')
pkg_task.add_command(tar_command(
pkg_path, cfg.pkg_name_no_target_build.get(),
cfg.source_date_epoch.get()),
cwd=inst_out_main)
def add_build_tasks_host_indep(cfg, component, host_indep_group):
if not cfg.args.build_source_packages:
return
# The source package has the sources of open source
# components; the backup package has the sources of closed
# source components.
source_group = BuildTask(cfg, host_indep_group, 'source-package')
source_objdir = cfg.objdir_path(None, 'source-package')
source_init_task = BuildTask(cfg, source_group, 'init')
source_init_task.add_empty_dir(source_objdir)
source_components_group = BuildTask(cfg, source_group, 'components',
True)
source_package_task = BuildTask(cfg, source_group, 'package')
source_pkg_path = cfg.pkgdir_path(None, '.src.tar.xz')
source_package_task.add_command(tar_command(
source_pkg_path, cfg.pkg_name_full.get(),
cfg.source_date_epoch.get()),
cwd=source_objdir)
backup_group = BuildTask(cfg, host_indep_group, 'backup-package')
backup_objdir = cfg.objdir_path(None, 'backup-package')
backup_init_task = BuildTask(cfg, backup_group, 'init')
backup_init_task.add_empty_dir(backup_objdir)
backup_components_group = BuildTask(cfg, backup_group, 'components',
True)
backup_package_task = BuildTask(cfg, backup_group, 'package')
backup_pkg_path = cfg.pkgdir_path(None, '.backup.tar.xz')
backup_package_task.add_command(tar_command(
backup_pkg_path, '%s.backup' % cfg.pkg_name_full.get(),
cfg.source_date_epoch.get()),
cwd=backup_objdir)
copy_objdir = cfg.objdir_path(None, 'source-copy')
for src_component in cfg.list_source_components():
name = src_component.copy_name
srcdir_copy = os.path.join(copy_objdir, name)
if src_component.vars.source_type.get() == 'open':
group = source_components_group
objdir = source_objdir
else:
group = backup_components_group
objdir = backup_objdir
task = BuildTask(cfg, group, name)
pkg_dir = '%s-%s' % (name, cfg.version.get())
pkg_path = os.path.join(objdir, '%s.tar.xz' % pkg_dir)
task.add_command(tar_command(pkg_path, pkg_dir,
cfg.source_date_epoch.get()),
cwd=srcdir_copy)
def add_host_cfg_build_tasks(relcfg, host, component, parent, name, srcdir,
prefix, pkg_cfg_opts, target, make_target,
install_target, parallel):
"""Add and return a group of tasks using configure / make / make install.
The parent task passed is the main task group for the host, or any
other group within which this group is to be contained. The host
passed is a BuildCfg object. The component passed is the
ComponentInConfig object. The name passed is a name to use for
build directories and install trees as well as for task names; if
None, the name of the component (copy) is used (this is
appropriate unless a component is built multiple times for one
host, e.g. multiple GCC builds for bootstrapping a cross
compiler). If srcdir is None, the source directory of that
component is used (this is appropriate unless configuring a
subdirectory, e.g. for gdbserver). If prefix is None, the path to
the install tree is used as the configured prefix (this is only
appropriate for host libraries not referring to files in their
configured prefix at runtime). A --target configure option is
passed unless target is None. Any configure options from the
configure_opts variable and component hook are added
automatically. If make_target is not None, it is the target
passed to make for the main build step; install_target likewise
specifies the target for the install step.
If additional steps are required after installation, the caller
should add a postinstall task or tasks to the group returned.
"""
build = relcfg.build.get().build_cfg
if name is None:
name = component.copy_name
if srcdir is None:
srcdir = component.vars.srcdir.get()
objdir = relcfg.objdir_path(host, name)
instdir = relcfg.install_tree_path(host, name)
if prefix is None:
cfg_prefix = instdir
destdir = None
else:
cfg_prefix = prefix
destdir = instdir
task_group = BuildTask(relcfg, parent, name)
task_group.provide_install(host, name)
init_task = BuildTask(relcfg, task_group, 'init')
init_task.add_empty_dir(objdir)
init_task.add_empty_dir(instdir)
cfg_task = BuildTask(relcfg, task_group, 'configure')
cfg_cmd = [os.path.join(srcdir, 'configure'),
'--build=%s' % build.triplet,
'--host=%s' % host.triplet,
'--prefix=%s' % cfg_prefix]
if target is not None:
cfg_cmd.append('--target=%s' % target)
cfg_cmd.extend(pkg_cfg_opts)
cfg_cmd.extend(component.vars.configure_opts.get())
cfg_cmd.extend(component.cls.configure_opts(relcfg, host))
cfg_cmd.extend(host.configure_vars())
cfg_cmd.extend(['CC_FOR_BUILD=%s'
% ' '.join(build.tool('c-compiler')),
'CXX_FOR_BUILD=%s'
% ' '.join(build.tool('c++-compiler'))])
cfg_task.add_command(cfg_cmd, cwd=objdir)
build_task = BuildTask(relcfg, task_group, 'build')
if parallel:
build_cmd = []
else:
build_cmd = ['-j1']
if make_target is not None:
build_cmd.append(make_target)
build_task.add_make(build_cmd, objdir)
install_task = BuildTask(relcfg, task_group, 'install')
install_cmd = ['-j1', install_target]
if destdir is not None:
install_cmd.append('DESTDIR=%s' % destdir)
install_task.add_make(install_cmd, objdir)
return task_group
def add_build_tasks_for_first_host(cfg, host, component, host_group):
task = BuildTask(cfg, host_group, 'first-host')
task.add_command(['true'], cwd='/dev/null')
def add_build_tasks_for_first_host(cfg, host, component, host_group):
task = BuildTask(cfg, host_group, 'first-host')
host_b = host.build_cfg
objdir = cfg.objdir_path(host_b, '%s-first' % component.copy_name)
objdir2 = cfg.objdir_path(host_b, '%s-first2' % component.copy_name)
task.add_empty_dir(objdir)
# Test add_empty_dir_parent with and without the directory
# existing.
task.add_empty_dir_parent(os.path.join(objdir, 'x', 'y', 'z'))
os.makedirs(os.path.join(objdir2, 'x', 'y', 'z'))
task.add_empty_dir_parent(os.path.join(objdir2, 'x', 'y', 'z'))
task.add_command(['sh', '-c',
'echo "all:; echo first-host \\$(X) > out" > '
'%s/GNUmakefile' % objdir])
task.add_make(['all', 'X=Y'], objdir)
def add_build_tasks_for_first_host(cfg, host, component, host_group):
task = BuildTask(cfg, host_group, 'first-host')
host_b = host.build_cfg
objdir = cfg.objdir_path(host_b, '%s-first' % component.copy_name)
task.add_empty_dir(objdir)
instdir_1 = cfg.install_tree_path(host_b, 'first-inst-1')
task.add_empty_dir(instdir_1)
task.provide_install(host_b, 'first-inst-1')
instdir_2 = cfg.install_tree_path(host_b, 'first-inst-2')
task.add_empty_dir(instdir_2)
task.provide_install(host_b, 'first-inst-2')
instdir_3 = cfg.install_tree_path(host_b, 'first-inst-3')
task.add_empty_dir(instdir_3)
task.provide_install(host_b, 'first-inst-3')
task.add_command(['sh', '-c', 'echo a > %s/a' % instdir_1])
task.add_command(['sh', '-c', 'echo b > %s/b' % instdir_2])
task.add_command(['sh', '-c', 'echo c > %s/c' % instdir_3])
tree_1 = cfg.install_tree_fstree(host_b, 'first-inst-1')
tree_1 = FSTreeMove(tree_1, 'q')
tree_2 = cfg.install_tree_fstree(host_b, 'first-inst-2')
tree_3 = cfg.install_tree_fstree(host_b, 'first-inst-3')
task.define_implicit_install(host_b, 'impl-def', tree_1)
task.declare_implicit_install(host_b, 'impl-empty')
task.declare_implicit_install(host_b, 'impl-one')
task.contribute_implicit_install(host_b, 'impl-one', tree_2)
task.declare_implicit_install(host_b, 'impl-two')
task.contribute_implicit_install(host_b, 'impl-two', tree_2)
task.contribute_implicit_install(host_b, 'impl-two', tree_3)
def add_build_tasks_fini(cfg, component, fini_group):
task = BuildTask(cfg, fini_group, 'fini')
objdir = cfg.objdir_path(None, '%s-fini' % component.copy_name)
task.add_empty_dir(objdir)
task.add_command(['sh', '-c', 'echo fini > %s/out' % objdir])
def add_build_tasks_host_indep(cfg, component, host_indep_group):
task = BuildTask(cfg, host_indep_group, 'host-indep')
objdir = cfg.objdir_path(None, '%s-host-indep' % component.copy_name)
task.add_empty_dir(objdir)
task.add_command(['sh', '-c', 'echo host-indep > %s/out' % objdir])
def add_build_tasks_init(cfg, component, init_group):
task = BuildTask(cfg, init_group, 'init')
objdir = cfg.objdir_path(None, '%s-init' % component.copy_name)
task.add_empty_dir(objdir)
task.add_command(['sh', '-c', 'echo init > %s/out' % objdir])