def push_kernel(kdir, api, token, install_path=None):
"""Push the kernel binaries to the storage server
Push the kernel image, the modules tarball, the dtbs and the build.json
meta-data to the storage server via the KernelCI backend API.
*kdir* is the path to the kernel source directory
*api* is the URL of the KernelCI backend API
*token* is the token to use with the KernelCI backend API
*install_path* is the path to the installation directory
The returned value is True if it was done successfully or False if an error
occurred.
"""
if not install_path:
install_path = os.path.join(kdir, '_install_')
with open(os.path.join(install_path, 'bmeta.json')) as f:
bmeta = json.load(f)
artifacts = {}
for root, _, files in os.walk(install_path):
for f in files:
px = os.path.relpath(root, install_path)
artifacts[os.path.join(px, f)] = open(os.path.join(root, f), "rb")
upload_path = bmeta['file_server_resource']
print_flush("Upload path: {}".format(upload_path))
upload_files(api, token, upload_path, artifacts)
return True
def _run_make(kdir,
arch,
target=None,
jopt=None,
silent=True,
cc='gcc',
cross_compile=None,
use_ccache=None,
output=None,
log_file=None,
opts=None,
cross_compile_compat=None):
args = ['make']
if opts:
args += ['='.join([k, v]) for k, v in opts.items()]
args += ['-C{}'.format(kdir)]
if jopt:
args.append('-j{}'.format(jopt))
if silent:
args.append('-s')
args.append('ARCH={}'.format(arch))
if cross_compile:
args.append('CROSS_COMPILE={}'.format(cross_compile))
if cross_compile_compat:
args.append('CROSS_COMPILE_COMPAT={}'.format(cross_compile_compat))
if cc.startswith('clang'):
args.append('LLVM=1')
else:
args.append('HOSTCC={}'.format(cc))
if use_ccache:
px = cross_compile if cc == 'gcc' and cross_compile else ''
args.append('CC="ccache {}{}"'.format(px, cc))
ccache_dir = '-'.join(['.ccache', arch, cc])
os.environ.setdefault('CCACHE_DIR', ccache_dir)
elif cc != 'gcc':
args.append('CC={}'.format(cc))
if output != kdir:
# due to kselftest Makefile issues, O= cannot be a relative path
args.append('O={}'.format(os.path.abspath(output)))
if target:
args.append(target)
cmd = ' '.join(args)
print_flush(cmd)
if log_file:
cmd = _output_to_file(cmd, log_file)
return shell_cmd(cmd, True)
def install_kernel(kdir, tree_name, tree_url, git_branch, git_commit=None,
describe=None, describe_v=None, output_path=None,
publish_path=None, install_path=None, mod_path=None):
"""Install the kernel binaries in a directory for a given built revision
Installing the kernel binaries into a new directory consists of creating a
"bmeta.json" file with all the meta-data for the kernel build, copying the
System.map file, the kernel .config, the build log, the frag.config file,
all the dtbs and a tarball with all the modules. This is an intermediate
step between building a kernel and publishing it via the KernelCI backend.
*kdir* is the path to the kernel source directory
*tree_name* is the name of the tree from a build configuration
*git_branch* is the name of the git branch in the tree
*git_commit* is the git commit SHA
*describe* is the "git describe" for the commit
*describe_v* is the verbose "git describe" for the commit
*output_path" is the path to the directory where the kernel was built
*install_path* is the path where to install the kernel
*mod_path* is the path where the modules were installed
The returned value is True if it was done successfully or False if an error
occurred.
"""
if not install_path:
install_path = os.path.join(kdir, '_install_')
if not output_path:
output_path = os.path.join(kdir, 'build')
if not mod_path:
mod_path = os.path.join(output_path, '_modules_')
if not git_commit:
git_commit = head_commit(kdir)
if not describe:
describe = git_describe(tree_name, kdir)
if not describe_v:
describe_v = git_describe_verbose(kdir)
if os.path.exists(install_path):
shutil.rmtree(install_path)
os.makedirs(install_path)
with open(os.path.join(output_path, 'bmeta.json')) as json_file:
bmeta = json.load(json_file)
system_map = os.path.join(output_path, 'System.map')
if os.path.exists(system_map):
virt_text = shell_cmd('grep " _text" {}'.format(system_map)).split()[0]
text_offset = int(virt_text, 16) & (1 << 30)-1 # phys: cap at 1G
shutil.copy(system_map, install_path)
else:
text_offset = None
dot_config = os.path.join(output_path, '.config')
dot_config_installed = os.path.join(install_path, 'kernel.config')
shutil.copy(dot_config, dot_config_installed)
build_log = os.path.join(output_path, 'build.log')
shutil.copy(build_log, install_path)
frags = os.path.join(output_path, 'frag.config')
if os.path.exists(frags):
shutil.copy(frags, install_path)
arch = bmeta['arch']
boot_dir = os.path.join(output_path, 'arch', arch, 'boot')
kimage_names = KERNEL_IMAGE_NAMES[arch]
kimages = []
kimage_file = None
for root, _, files in os.walk(boot_dir):
for name in kimage_names:
if name in files:
kimages.append(name)
image_path = os.path.join(root, name)
shutil.copy(image_path, install_path)
for files in os.listdir(output_path):
for name in kimage_names:
if name == files:
kimages.append(name)
image_path = os.path.join(output_path, name)
shutil.copy(image_path, install_path)
if kimages:
for name in kimage_names:
if name in kimages:
kimage_file = name
break
if not kimage_file:
print_flush("Warning: no kernel image found")
dts_dir = os.path.join(boot_dir, 'dts')
dtbs = os.path.join(install_path, 'dtbs')
dtb_list = []
for root, _, files in os.walk(dts_dir):
for f in fnmatch.filter(files, '*.dtb'):
dtb_path = os.path.join(root, f)
dtb_rel = os.path.relpath(dtb_path, dts_dir)
dtb_list.append(dtb_rel)
dest_path = os.path.join(dtbs, dtb_rel)
dest_dir = os.path.dirname(dest_path)
if not os.path.exists(dest_dir):
os.makedirs(dest_dir)
shutil.copy(dtb_path, dest_path)
with open(os.path.join(install_path, 'dtbs.json'), 'w') as json_file:
json.dump({'dtbs': sorted(dtb_list)}, json_file, indent=4)
modules_tarball = None
if os.path.exists(mod_path):
modules_tarball = 'modules.tar.xz'
modules_tarball_path = os.path.join(install_path, modules_tarball)
shell_cmd("tar -C{path} -cJf {tarball} .".format(
path=mod_path, tarball=modules_tarball_path))
# 'make gen_tar' creates this tarball path
kselftest_tarball = 'kselftest-packages/kselftest.tar.xz'
kselftest_tarball_path = os.path.join(output_path, '_kselftest_',
kselftest_tarball)
if os.path.exists(kselftest_tarball_path):
kselftest_tarball = os.path.basename(kselftest_tarball_path)
shutil.copy(kselftest_tarball_path,
os.path.join(install_path, kselftest_tarball))
else:
kselftest_tarball = kselftest_tarball_path = None
build_env = bmeta['build_environment']
defconfig_full = bmeta['defconfig_full']
if not publish_path:
publish_path = '/'.join(item.replace('/', '-') for item in [
tree_name, git_branch, describe, arch, defconfig_full, build_env,
])
bmeta.update({
'kconfig_fragments': 'frag.config' if os.path.exists(frags) else '',
'kernel_image': kimage_file,
'kernel_config': os.path.basename(dot_config_installed),
'system_map': 'System.map' if os.path.exists(system_map) else None,
'text_offset': '0x{:08x}'.format(text_offset) if text_offset else None,
'dtb_dir': 'dtbs' if os.path.exists(dtbs) else None,
'modules': modules_tarball,
'job': tree_name,
'git_url': tree_url,
'git_branch': git_branch,
'git_describe': describe,
'git_describe_v': describe_v,
'git_commit': git_commit,
'file_server_resource': publish_path,
'kselftests': kselftest_tarball,
})
with open(os.path.join(install_path, 'bmeta.json'), 'w') as json_file:
json.dump(bmeta, json_file, indent=4, sort_keys=True)
return True
def build_kernel(build_env, kdir, arch, defconfig=None, jopt=None,
verbose=False, output_path=None, mod_path=None):
"""Build a linux kernel
*build_env* is a BuildEnvironment object
*kdir* is the path to the kernel source directory
*defconfig* is the name of the kernel defconfig
*jopt* is the -j option to pass to make for parallel builds
*verbose* is whether to print all the output of the make commands
*output_path* is the path to the directory where the binaries are made
*mod_path* is the path to where the modules are installed
The returned value is True if the build was successful or False if there
was any build error.
"""
cc = build_env.cc
cross_compile = build_env.get_cross_compile(arch) or ''
cross_compile_compat = build_env.get_cross_compile_compat(arch) or ''
use_ccache = shell_cmd("which ccache > /dev/null", True)
if jopt is None:
jopt = int(shell_cmd("nproc")) + 2
if not output_path:
output_path = os.path.join(kdir, 'build')
if not os.path.exists(output_path):
os.mkdir(output_path)
if not mod_path:
mod_path = os.path.join(output_path, '_modules_')
build_log = 'build.log'
log_file = os.path.join(output_path, build_log)
dot_config = os.path.join(output_path, '.config')
if os.path.exists(log_file):
os.unlink(log_file)
opts = {
'KBUILD_BUILD_USER': '******',
}
kwargs = {
'kdir': kdir,
'arch': arch,
'cc': cc,
'cross_compile': cross_compile,
'cross_compile_compat': cross_compile_compat,
'use_ccache': use_ccache,
'output': output_path,
'silent': not verbose,
'log_file': log_file,
'opts': opts,
}
start_time = time.time()
defconfig_extras = []
if defconfig:
result = _make_defconfig(
defconfig, kwargs, defconfig_extras, verbose, log_file)
elif os.path.exists(dot_config):
print_flush("Re-using {}".format(dot_config))
result = True
else:
print_flush("ERROR: Missing kernel config")
result = False
if result:
if _kernel_config_enabled(dot_config, 'XIP_KERNEL'):
target = 'xipImage'
elif _kernel_config_enabled(dot_config, 'SYS_SUPPORTS_ZBOOT'):
target = 'vmlinuz'
else:
target = MAKE_TARGETS.get(arch)
result = _run_make(jopt=jopt, target=target, **kwargs)
mods = _kernel_config_enabled(dot_config, 'MODULES')
if result and mods:
result = _run_make(jopt=jopt, target='modules', **kwargs)
if result and _kernel_config_enabled(dot_config, 'OF_FLATTREE'):
dts_tree = os.path.join(kdir, 'arch/{}/boot/dts'.format(arch))
if os.path.exists(dts_tree):
result = _run_make(target='dtbs', **kwargs)
build_time = time.time() - start_time
if result and mods:
if os.path.exists(mod_path):
shutil.rmtree(mod_path)
os.makedirs(mod_path)
opts.update({
'INSTALL_MOD_PATH': mod_path,
'INSTALL_MOD_STRIP': '1',
'STRIP': "{}strip".format(cross_compile),
})
result = _run_make(target='modules_install', **kwargs)
# kselftest
if result and "kselftest" in defconfig_extras:
kselftest_install_path = os.path.join(output_path, '_kselftest_')
if os.path.exists(kselftest_install_path):
shutil.rmtree(kselftest_install_path)
opts.update({
'INSTALL_PATH': kselftest_install_path,
})
#
# Ideally this should just be a 'make kselftest-install', but
# due to bugs with O= in kselftest Makefile, this has to be
# 'make -C tools/testing/selftests install'
#
kwargs.update({
'kdir': os.path.join(kdir, 'tools/testing/selftests')
})
opts.update({
'FORMAT': '.xz',
})
# 'gen_tar' target does 'make install' and creates tarball
result = _run_make(target='gen_tar', **kwargs)
cc_version_cmd = "{}{} --version 2>&1".format(
cross_compile if cross_compile and cc == 'gcc' else '', cc)
cc_version_full = shell_cmd(cc_version_cmd).splitlines()[0]
bmeta = {
'build_threads': jopt,
'build_time': round(build_time, 2),
'status': 'PASS' if result is True else 'FAIL',
'arch': arch,
'cross_compile': cross_compile,
'compiler': cc,
'compiler_version': build_env.cc_version,
'compiler_version_full': cc_version_full,
'build_environment': build_env.name,
'build_log': build_log,
'build_platform': platform.uname(),
}
if defconfig:
defconfig_target = defconfig.split('+')[0]
bmeta.update({
'defconfig': defconfig_target,
'defconfig_full': '+'.join([defconfig_target] + defconfig_extras),
})
else:
bmeta.update({
'defconfig': 'none',
'defconfig_full': 'none',
})
vmlinux_file = os.path.join(output_path, 'vmlinux')
if os.path.isfile(vmlinux_file):
vmlinux_meta = kernelci.elf.read(vmlinux_file)
bmeta.update(vmlinux_meta)
bmeta['vmlinux_file_size'] = os.stat(vmlinux_file).st_size
with open(os.path.join(output_path, 'bmeta.json'), 'w') as json_file:
json.dump(bmeta, json_file, indent=4, sort_keys=True)
return result
def _make_defconfig(defconfig, kwargs, extras, verbose, log_file):
kdir, output_path = (kwargs.get(k) for k in ('kdir', 'output'))
result = True
defconfig_kwargs = dict(kwargs)
defconfig_opts = dict(defconfig_kwargs['opts'])
defconfig_kwargs['opts'] = defconfig_opts
tmpfile_fd, tmpfile_path = tempfile.mkstemp(prefix='kconfig-')
tmpfile = os.fdopen(tmpfile_fd, 'w')
tmpfile_used = False
defs = defconfig.split('+')
target = defs.pop(0)
for d in defs:
if d.startswith('KCONFIG_'):
config, value = d.split('=')
defconfig_opts[config] = value
extras.append(d)
elif d.startswith('CONFIG_'):
tmpfile.write(d + '\n')
extras.append(d)
tmpfile_used = True
else:
frag_path = os.path.join(kdir, d)
if os.path.exists(frag_path):
with open(frag_path) as frag:
tmpfile.write("\n# fragment from : {}\n".format(d))
tmpfile.writelines(frag)
tmpfile_used = True
extras.append(os.path.basename(os.path.splitext(d)[0]))
else:
print_flush("Fragment not found: {}".format(frag_path))
result = False
tmpfile.flush()
if not _run_make(target=target, **defconfig_kwargs):
result = False
if result and tmpfile_used:
kconfig_frag_name = 'frag.config'
kconfig_frag = os.path.join(output_path, kconfig_frag_name)
shutil.copy(tmpfile_path, kconfig_frag)
os.chmod(kconfig_frag,
stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH)
rel_path = os.path.relpath(output_path, kdir)
cc = kwargs['cc']
cc_env = (
"export LLVM=1" if cc.startswith('clang') else
"export HOSTCC={cc}\nexport CC={cc}".format(cc=cc)
)
cmd = """
set -e
cd {kdir}
{cc_env}
export ARCH={arch}
export CROSS_COMPILE={cross}
export CROSS_COMPILE_COMPAT={cross_compat}
scripts/kconfig/merge_config.sh -O {output} '{base}' '{frag}' {redir}
""".format(kdir=kdir, arch=kwargs['arch'], cc_env=cc_env,
cross=kwargs['cross_compile'], output=rel_path,
cross_compat=kwargs['cross_compile_compat'],
base=os.path.join(rel_path, '.config'),
frag=os.path.join(rel_path, kconfig_frag_name),
redir='> /dev/null' if not verbose else '')
print_flush(cmd.strip())
if log_file:
cmd = _output_to_file(cmd, log_file, kdir)
result = shell_cmd(cmd, True)
tmpfile.close()
os.unlink(tmpfile_path)
return result
