def _fetch_tags(path, url=TORVALDS_GIT_URL):
shell_cmd("""
set -e
cd {path}
git fetch --tags {url}
""".format(path=path, url=url))
def _make_defconfig(defconfig, kwargs, extras, verbose, log_file):
kdir, output_path = (kwargs.get(k) for k in ('kdir', 'output'))
result = True
defconfig_kwargs = dict(kwargs)
defconfig_opts = dict(defconfig_kwargs['opts'])
defconfig_kwargs['opts'] = defconfig_opts
tmpfile_fd, tmpfile_path = tempfile.mkstemp(prefix='kconfig-')
tmpfile = os.fdopen(tmpfile_fd, 'w')
tmpfile_used = False
defs = defconfig.split('+')
target = defs.pop(0)
for d in defs:
if d.startswith('KCONFIG_'):
config, value = d.split('=')
defconfig_opts[config] = value
extras.append(d)
elif d.startswith('CONFIG_'):
tmpfile.write(d + '\n')
extras.append(d)
tmpfile_used = True
else:
frag_path = os.path.join(kdir, d)
if os.path.exists(frag_path):
with open(frag_path) as frag:
tmpfile.write("\n# fragment from : {}\n".format(d))
tmpfile.writelines(frag)
tmpfile_used = True
extras.append(os.path.basename(os.path.splitext(d)[0]))
else:
print_flush("Fragment not found: {}".format(frag_path))
result = False
tmpfile.flush()
if not _run_make(target=target, **defconfig_kwargs):
result = False
if result and tmpfile_used:
kconfig_frag_name = 'frag.config'
kconfig_frag = os.path.join(output_path, kconfig_frag_name)
shutil.copy(tmpfile_path, kconfig_frag)
os.chmod(kconfig_frag,
stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH)
rel_path = os.path.relpath(output_path, kdir)
cmd = """
set -e
cd {kdir}
export ARCH={arch}
export HOSTCC={cc}
export CC={cc}
export CROSS_COMPILE={cross}
scripts/kconfig/merge_config.sh -O {output} '{base}' '{frag}' {redir}
""".format(kdir=kdir, arch=kwargs['arch'], cc=kwargs['cc'],
cross=kwargs['cross_compile'], output=rel_path,
base=os.path.join(rel_path, '.config'),
frag=os.path.join(rel_path, kconfig_frag_name),
redir='> /dev/null' if not verbose else '')
print_flush(cmd.strip())
if log_file:
cmd = _output_to_file(cmd, log_file, kdir)
result = shell_cmd(cmd, True)
tmpfile.close()
os.unlink(tmpfile_path)
return result
def build_kernel(build_env,
kdir,
arch,
defconfig=None,
jopt=None,
verbose=False,
output_path=None,
mod_path=None):
"""Build a linux kernel
*build_env* is a BuildEnvironment object
*kdir* is the path to the kernel source directory
*defconfig* is the name of the kernel defconfig
*jopt* is the -j option to pass to make for parallel builds
*verbose* is whether to print all the output of the make commands
*output_path* is the path to the directory where the binaries are made
*mod_path* is the path to where the modules are installed
The returned value is True if the build was successful or False if there
was any build error.
"""
cc = build_env.cc
cross_compile = build_env.get_cross_compile(arch) or ''
cross_compile_compat = build_env.get_cross_compile_compat(arch) or ''
use_ccache = shell_cmd("which ccache > /dev/null", True)
if jopt is None:
jopt = int(shell_cmd("nproc")) + 2
if not output_path:
output_path = os.path.join(kdir, 'build')
if not os.path.exists(output_path):
os.mkdir(output_path)
if not mod_path:
mod_path = os.path.join(output_path, '_modules_')
build_log = 'build.log'
log_file = os.path.join(output_path, build_log)
dot_config = os.path.join(output_path, '.config')
if os.path.exists(log_file):
os.unlink(log_file)
opts = {
'KBUILD_BUILD_USER': '******',
}
kwargs = {
'kdir': kdir,
'arch': arch,
'cc': cc,
'cross_compile': cross_compile,
'cross_compile_compat': cross_compile_compat,
'use_ccache': use_ccache,
'output': output_path,
'silent': not verbose,
'log_file': log_file,
'opts': opts,
}
start_time = time.time()
defconfig_extras = []
if defconfig:
result = _make_defconfig(defconfig, kwargs, defconfig_extras, verbose,
log_file)
elif os.path.exists(dot_config):
print_flush("Re-using {}".format(dot_config))
result = True
else:
print_flush("ERROR: Missing kernel config")
result = False
if result:
if _kernel_config_enabled(dot_config, 'XIP_KERNEL'):
target = 'xipImage'
elif _kernel_config_enabled(dot_config, 'SYS_SUPPORTS_ZBOOT'):
target = 'vmlinuz'
else:
target = MAKE_TARGETS.get(arch)
result = _run_make(jopt=jopt, target=target, **kwargs)
mods = _kernel_config_enabled(dot_config, 'MODULES')
if result and mods:
result = _run_make(jopt=jopt, target='modules', **kwargs)
if result and _kernel_config_enabled(dot_config, 'OF_FLATTREE'):
dts_tree = os.path.join(kdir, 'arch/{}/boot/dts'.format(arch))
if os.path.exists(dts_tree):
result = _run_make(target='dtbs', **kwargs)
build_time = time.time() - start_time
if result and mods:
if os.path.exists(mod_path):
shutil.rmtree(mod_path)
os.makedirs(mod_path)
opts.update({
'INSTALL_MOD_PATH': mod_path,
'INSTALL_MOD_STRIP': '1',
'STRIP': "{}strip".format(cross_compile),
})
result = _run_make(target='modules_install', **kwargs)
# kselftest
if result and "kselftest" in defconfig_extras:
kselftest_install_path = os.path.join(output_path, '_kselftest_')
if os.path.exists(kselftest_install_path):
shutil.rmtree(kselftest_install_path)
opts.update({
'INSTALL_PATH': kselftest_install_path,
})
#
# Ideally this should just be a 'make kselftest-install', but
# due to bugs with O= in kselftest Makefile, this has to be
# 'make -C tools/testing/selftests install'
#
kwargs.update({'kdir': os.path.join(kdir, 'tools/testing/selftests')})
opts.update({
'FORMAT': '.xz',
})
# 'gen_tar' target does 'make install' and creates tarball
result = _run_make(target='gen_tar', **kwargs)
cc_version_cmd = "{}{} --version 2>&1".format(
cross_compile if cross_compile and cc == 'gcc' else '', cc)
cc_version_full = shell_cmd(cc_version_cmd).splitlines()[0]
bmeta = {
'build_threads': jopt,
'build_time': round(build_time, 2),
'status': 'PASS' if result is True else 'FAIL',
'arch': arch,
'cross_compile': cross_compile,
'compiler': cc,
'compiler_version': build_env.cc_version,
'compiler_version_full': cc_version_full,
'build_environment': build_env.name,
'build_log': build_log,
'build_platform': platform.uname(),
}
if defconfig:
defconfig_target = defconfig.split('+')[0]
bmeta.update({
'defconfig':
defconfig_target,
'defconfig_full':
'+'.join([defconfig_target] + defconfig_extras),
})
else:
bmeta.update({
'defconfig': 'none',
'defconfig_full': 'none',
})
vmlinux_file = os.path.join(output_path, 'vmlinux')
if os.path.isfile(vmlinux_file):
vmlinux_meta = kernelci.elf.read(vmlinux_file)
bmeta.update(vmlinux_meta)
bmeta['vmlinux_file_size'] = os.stat(vmlinux_file).st_size
with open(os.path.join(output_path, 'bmeta.json'), 'w') as json_file:
json.dump(bmeta, json_file, indent=4, sort_keys=True)
return result
def install_kernel(kdir,
tree_name,
tree_url,
git_branch,
git_commit=None,
describe=None,
describe_v=None,
output_path=None,
publish_path=None,
install_path=None,
mod_path=None):
"""Install the kernel binaries in a directory for a given built revision
Installing the kernel binaries into a new directory consists of creating a
"bmeta.json" file with all the meta-data for the kernel build, copying the
System.map file, the kernel .config, the build log, the frag.config file,
all the dtbs and a tarball with all the modules. This is an intermediate
step between building a kernel and publishing it via the KernelCI backend.
*kdir* is the path to the kernel source directory
*tree_name* is the name of the tree from a build configuration
*git_branch* is the name of the git branch in the tree
*git_commit* is the git commit SHA
*describe* is the "git describe" for the commit
*describe_v* is the verbose "git describe" for the commit
*output_path" is the path to the directory where the kernel was built
*install_path* is the path where to install the kernel
*mod_path* is the path where the modules were installed
The returned value is True if it was done successfully or False if an error
occurred.
"""
if not install_path:
install_path = os.path.join(kdir, '_install_')
if not output_path:
output_path = os.path.join(kdir, 'build')
if not mod_path:
mod_path = os.path.join(output_path, '_modules_')
if not git_commit:
git_commit = head_commit(kdir)
if not describe:
describe = git_describe(tree_name, kdir)
if not describe_v:
describe_v = git_describe_verbose(kdir)
if os.path.exists(install_path):
shutil.rmtree(install_path)
os.makedirs(install_path)
with open(os.path.join(output_path, 'bmeta.json')) as json_file:
bmeta = json.load(json_file)
system_map = os.path.join(output_path, 'System.map')
if os.path.exists(system_map):
virt_text = shell_cmd('grep " _text" {}'.format(system_map)).split()[0]
text_offset = int(virt_text, 16) & (1 << 30) - 1 # phys: cap at 1G
shutil.copy(system_map, install_path)
else:
text_offset = None
dot_config = os.path.join(output_path, '.config')
dot_config_installed = os.path.join(install_path, 'kernel.config')
shutil.copy(dot_config, dot_config_installed)
build_log = os.path.join(output_path, 'build.log')
shutil.copy(build_log, install_path)
frags = os.path.join(output_path, 'frag.config')
if os.path.exists(frags):
shutil.copy(frags, install_path)
arch = bmeta['arch']
boot_dir = os.path.join(output_path, 'arch', arch, 'boot')
kimage_names = KERNEL_IMAGE_NAMES[arch]
kimages = []
kimage_file = None
for root, _, files in os.walk(boot_dir):
for name in kimage_names:
if name in files:
kimages.append(name)
image_path = os.path.join(root, name)
shutil.copy(image_path, install_path)
for files in os.listdir(output_path):
for name in kimage_names:
if name == files:
kimages.append(name)
image_path = os.path.join(output_path, name)
shutil.copy(image_path, install_path)
if kimages:
for name in kimage_names:
if name in kimages:
kimage_file = name
break
if not kimage_file:
print_flush("Warning: no kernel image found")
dts_dir = os.path.join(boot_dir, 'dts')
dtbs = os.path.join(install_path, 'dtbs')
dtb_list = []
for root, _, files in os.walk(dts_dir):
for f in fnmatch.filter(files, '*.dtb'):
dtb_path = os.path.join(root, f)
dtb_rel = os.path.relpath(dtb_path, dts_dir)
dtb_list.append(dtb_rel)
dest_path = os.path.join(dtbs, dtb_rel)
dest_dir = os.path.dirname(dest_path)
if not os.path.exists(dest_dir):
os.makedirs(dest_dir)
shutil.copy(dtb_path, dest_path)
with open(os.path.join(install_path, 'dtbs.json'), 'w') as json_file:
json.dump({'dtbs': sorted(dtb_list)}, json_file, indent=4)
modules_tarball = None
if os.path.exists(mod_path):
modules_tarball = 'modules.tar.xz'
modules_tarball_path = os.path.join(install_path, modules_tarball)
shell_cmd("tar -C{path} -cJf {tarball} .".format(
path=mod_path, tarball=modules_tarball_path))
# 'make gen_tar' creates this tarball path
kselftest_tarball = 'kselftest-packages/kselftest.tar.xz'
kselftest_tarball_path = os.path.join(output_path, '_kselftest_',
kselftest_tarball)
if os.path.exists(kselftest_tarball_path):
kselftest_tarball = os.path.basename(kselftest_tarball_path)
shutil.copy(kselftest_tarball_path,
os.path.join(install_path, kselftest_tarball))
else:
kselftest_tarball = kselftest_tarball_path = None
build_env = bmeta['build_environment']
defconfig_full = bmeta['defconfig_full']
if not publish_path:
publish_path = '/'.join(
item.replace('/', '-') for item in [
tree_name,
git_branch,
describe,
arch,
defconfig_full,
build_env,
])
bmeta.update({
'kconfig_fragments':
'frag.config' if os.path.exists(frags) else '',
'kernel_image':
kimage_file,
'kernel_config':
os.path.basename(dot_config_installed),
'system_map':
'System.map' if os.path.exists(system_map) else None,
'text_offset':
'0x{:08x}'.format(text_offset) if text_offset else None,
'dtb_dir':
'dtbs' if os.path.exists(dtbs) else None,
'modules':
modules_tarball,
'job':
tree_name,
'git_url':
tree_url,
'git_branch':
git_branch,
'git_describe':
describe,
'git_describe_v':
describe_v,
'git_commit':
git_commit,
'file_server_resource':
publish_path,
'kselftests':
kselftest_tarball,
})
with open(os.path.join(install_path, 'bmeta.json'), 'w') as json_file:
json.dump(bmeta, json_file, indent=4, sort_keys=True)
return True
def _kernel_config_enabled(dot_config, name):
return shell_cmd('grep -cq CONFIG_{}=y {}'.format(name, dot_config), True)
def install_kernel(kdir,
tree_name,
tree_url,
git_branch,
git_commit=None,
describe=None,
describe_v=None,
output_path=None,
install='_install_',
mod_path='_modules_'):
install_path = os.path.join(kdir, install)
if not output_path:
output_path = os.path.join(kdir, 'build')
if not git_commit:
git_commit = head_commit(kdir)
if not describe:
describe = git_describe(tree_name, kdir)
if not describe_v:
describe_v = git_describe_verbose(kdir)
if os.path.exists(install_path):
shutil.rmtree(install_path)
os.makedirs(install_path)
with open(os.path.join(output_path, 'bmeta.json')) as json_file:
bmeta = json.load(json_file)
system_map = os.path.join(output_path, 'System.map')
if os.path.exists(system_map):
virt_text = shell_cmd('grep " _text" {}'.format(system_map)).split()[0]
text_offset = int(virt_text, 16) & (1 << 30) - 1 # phys: cap at 1G
shutil.copy(system_map, install_path)
else:
text_offset = None
dot_config = os.path.join(output_path, '.config')
dot_config_installed = os.path.join(install_path, 'kernel.config')
shutil.copy(dot_config, dot_config_installed)
build_log = os.path.join(output_path, 'build.log')
shutil.copy(build_log, install_path)
frags = os.path.join(output_path, 'frag.config')
if os.path.exists(frags):
shutil.copy(frags, install_path)
arch = bmeta['arch']
boot_dir = os.path.join(output_path, 'arch', arch, 'boot')
kimage_names = KERNEL_IMAGE_NAMES[arch]
kimages = []
kimage_file = None
for root, _, files in os.walk(boot_dir):
for name in kimage_names:
if name in files:
kimages.append(name)
image_path = os.path.join(root, name)
shutil.copy(image_path, install_path)
if kimages:
for name in kimage_names:
if name in kimages:
kimage_file = name
break
if not kimage_file:
print("Warning: no kernel image found")
dts_dir = os.path.join(boot_dir, 'dts')
dtbs = os.path.join(install_path, 'dtbs')
for root, _, files in os.walk(dts_dir):
for f in fnmatch.filter(files, '*.dtb'):
dtb_path = os.path.join(root, f)
dtb_rel = os.path.relpath(dtb_path, dts_dir)
dest_path = os.path.join(dtbs, dtb_rel)
dest_dir = os.path.dirname(dest_path)
if not os.path.exists(dest_dir):
os.makedirs(dest_dir)
shutil.copy(dtb_path, dest_path)
modules_tarball = None
if mod_path:
abs_mod_path = os.path.join(output_path, mod_path)
if os.path.exists(abs_mod_path):
modules_tarball = 'modules.tar.xz'
modules_tarball_path = os.path.join(install_path, modules_tarball)
shell_cmd("tar -C{path} -cJf {tarball} .".format(
path=abs_mod_path, tarball=modules_tarball_path))
build_env = bmeta['build_environment']
defconfig_full = bmeta['defconfig_full']
publish_path = '/'.join([
tree_name,
git_branch,
describe,
arch,
defconfig_full,
build_env,
])
bmeta.update({
'kconfig_fragments':
'frag.config' if os.path.exists(frags) else '',
'kernel_image':
kimage_file,
'kernel_config':
os.path.basename(dot_config_installed),
'system_map':
'System.map' if os.path.exists(system_map) else None,
'text_offset':
'0x{:08x}'.format(text_offset) if text_offset else None,
'dtb_dir':
'dtbs' if os.path.exists(dtbs) else None,
'modules':
modules_tarball,
'job':
tree_name,
'git_url':
tree_url,
'git_branch':
git_branch,
'git_describe':
describe,
'git_describe_verbose':
describe_v,
'git_commit':
git_commit,
'file_server_resource':
publish_path,
})
with open(os.path.join(install_path, 'build.json'), 'w') as json_file:
json.dump(bmeta, json_file, indent=4, sort_keys=True)
return True
def build_kernel(build_env,
kdir,
arch,
defconfig=None,
jopt=None,
verbose=False,
output_path=None,
mod_path='_modules_'):
cc = build_env.cc
cross_compile = build_env.get_cross_compile(arch) or None
use_ccache = shell_cmd("which ccache > /dev/null", True)
if jopt is None:
jopt = int(shell_cmd("nproc")) + 2
if not output_path:
output_path = os.path.join(kdir, 'build')
if not os.path.exists(output_path):
os.mkdir(output_path)
build_log = 'build.log'
log_file = os.path.join(output_path, build_log)
dot_config = os.path.join(output_path, '.config')
if os.path.exists(log_file):
os.unlink(log_file)
opts = {
'KBUILD_BUILD_USER': '******',
}
kwargs = {
'kdir': kdir,
'arch': arch,
'cc': cc,
'cross_compile': cross_compile,
'use_ccache': use_ccache,
'output': output_path,
'silent': not verbose,
'log_file': log_file,
'opts': opts,
}
start_time = time.time()
fragments = []
if defconfig:
result = _make_defconfig(defconfig, kwargs, fragments)
elif os.path.exists(dot_config):
print("Re-using {}".format(dot_config))
result = True
else:
print("ERROR: Missing kernel config")
result = False
if result:
target = ('xipImage' if _kernel_config_enabled(
dot_config, 'XIP_KERNEL') else MAKE_TARGETS.get(arch))
result = _run_make(jopt=jopt, target=target, **kwargs)
mods = _kernel_config_enabled(dot_config, 'MODULES')
if result and mods:
result = _run_make(jopt=jopt, target='modules', **kwargs)
if result and _kernel_config_enabled(dot_config, 'OF_FLATTREE'):
dts_tree = os.path.join(kdir, 'arch/{}/boot/dts'.format(arch))
if os.path.exists(dts_tree):
result = _run_make(target='dtbs', **kwargs)
build_time = time.time() - start_time
if result and mods and mod_path:
abs_mod_path = os.path.join(output_path, mod_path)
if os.path.exists(abs_mod_path):
shutil.rmtree(abs_mod_path)
os.makedirs(abs_mod_path)
opts.update({
'INSTALL_MOD_PATH': mod_path,
'INSTALL_MOD_STRIP': '1',
'STRIP': "{}strip".format(cross_compile or ''),
})
result = _run_make(target='modules_install', **kwargs)
cc_version_cmd = "{}{} --version 2>&1".format(
cross_compile if cross_compile and cc == 'gcc' else '', cc)
cc_version_full = shell_cmd(cc_version_cmd).splitlines()[0]
bmeta = {
'build_threads': jopt,
'build_time': round(build_time, 2),
'build_result': 'PASS' if result is True else 'FAIL',
'arch': arch,
'cross_compile': cross_compile,
'compiler': cc,
'compiler_version': build_env.cc_version,
'compiler_version_full': cc_version_full,
'build_environment': build_env.name,
'build_log': build_log,
'build_platform': platform.uname(),
}
if defconfig:
defconfig_target = defconfig.split('+')[0]
bmeta.update({
'defconfig':
defconfig_target,
'defconfig_full':
'+'.join([defconfig_target] + fragments),
})
vmlinux_file = os.path.join(output_path, 'vmlinux')
if os.path.isfile(vmlinux_file):
vmlinux_meta = kernelci.elf.read(vmlinux_file)
bmeta.update(vmlinux_meta)
bmeta['vmlinux_file_size'] = os.stat(vmlinux_file).st_size
with open(os.path.join(output_path, 'bmeta.json'), 'w') as json_file:
json.dump(bmeta, json_file, indent=4, sort_keys=True)
return result
def add_kselftest_fragment(path, frag_path='kernel/configs/kselftest.config'):
shell_cmd("""
cd {path} &&
find tools/testing/selftests -name config -printf "#\n# %h/%f\n#\n" -exec cat {{}} \; > {frag_path}
""".format(path=path, frag_path=frag_path))
def do_rebase(path, origin='origin', origin_branch='main'):
shell_cmd("""\
cd {path}
git pull --rebase {origin} {origin_branch}
""".format(path=path, origin=origin, origin_branch=origin_branch))
