def push_kernel(kdir, api, token, install_path=None): """Push the kernel binaries to the storage server Push the kernel image, the modules tarball, the dtbs and the build.json meta-data to the storage server via the KernelCI backend API. *kdir* is the path to the kernel source directory *api* is the URL of the KernelCI backend API *token* is the token to use with the KernelCI backend API *install_path* is the path to the installation directory The returned value is True if it was done successfully or False if an error occurred. """ if not install_path: install_path = os.path.join(kdir, '_install_') with open(os.path.join(install_path, 'bmeta.json')) as f: bmeta = json.load(f) artifacts = {} for root, _, files in os.walk(install_path): for f in files: px = os.path.relpath(root, install_path) artifacts[os.path.join(px, f)] = open(os.path.join(root, f), "rb") upload_path = bmeta['file_server_resource'] print_flush("Upload path: {}".format(upload_path)) upload_files(api, token, upload_path, artifacts) return True
def _run_make(kdir, arch, target=None, jopt=None, silent=True, cc='gcc', cross_compile=None, use_ccache=None, output=None, log_file=None, opts=None, cross_compile_compat=None): args = ['make'] if opts: args += ['='.join([k, v]) for k, v in opts.items()] args += ['-C{}'.format(kdir)] if jopt: args.append('-j{}'.format(jopt)) if silent: args.append('-s') args.append('ARCH={}'.format(arch)) if cross_compile: args.append('CROSS_COMPILE={}'.format(cross_compile)) if cross_compile_compat: args.append('CROSS_COMPILE_COMPAT={}'.format(cross_compile_compat)) if cc.startswith('clang'): args.append('LLVM=1') else: args.append('HOSTCC={}'.format(cc)) if use_ccache: px = cross_compile if cc == 'gcc' and cross_compile else '' args.append('CC="ccache {}{}"'.format(px, cc)) ccache_dir = '-'.join(['.ccache', arch, cc]) os.environ.setdefault('CCACHE_DIR', ccache_dir) elif cc != 'gcc': args.append('CC={}'.format(cc)) if output != kdir: # due to kselftest Makefile issues, O= cannot be a relative path args.append('O={}'.format(os.path.abspath(output))) if target: args.append(target) cmd = ' '.join(args) print_flush(cmd) if log_file: cmd = _output_to_file(cmd, log_file) return shell_cmd(cmd, True)
def install_kernel(kdir, tree_name, tree_url, git_branch, git_commit=None, describe=None, describe_v=None, output_path=None, publish_path=None, install_path=None, mod_path=None): """Install the kernel binaries in a directory for a given built revision Installing the kernel binaries into a new directory consists of creating a "bmeta.json" file with all the meta-data for the kernel build, copying the System.map file, the kernel .config, the build log, the frag.config file, all the dtbs and a tarball with all the modules. This is an intermediate step between building a kernel and publishing it via the KernelCI backend. *kdir* is the path to the kernel source directory *tree_name* is the name of the tree from a build configuration *git_branch* is the name of the git branch in the tree *git_commit* is the git commit SHA *describe* is the "git describe" for the commit *describe_v* is the verbose "git describe" for the commit *output_path" is the path to the directory where the kernel was built *install_path* is the path where to install the kernel *mod_path* is the path where the modules were installed The returned value is True if it was done successfully or False if an error occurred. """ if not install_path: install_path = os.path.join(kdir, '_install_') if not output_path: output_path = os.path.join(kdir, 'build') if not mod_path: mod_path = os.path.join(output_path, '_modules_') if not git_commit: git_commit = head_commit(kdir) if not describe: describe = git_describe(tree_name, kdir) if not describe_v: describe_v = git_describe_verbose(kdir) if os.path.exists(install_path): shutil.rmtree(install_path) os.makedirs(install_path) with open(os.path.join(output_path, 'bmeta.json')) as json_file: bmeta = json.load(json_file) system_map = os.path.join(output_path, 'System.map') if os.path.exists(system_map): virt_text = shell_cmd('grep " _text" {}'.format(system_map)).split()[0] text_offset = int(virt_text, 16) & (1 << 30)-1 # phys: cap at 1G shutil.copy(system_map, install_path) else: text_offset = None dot_config = os.path.join(output_path, '.config') dot_config_installed = os.path.join(install_path, 'kernel.config') shutil.copy(dot_config, dot_config_installed) build_log = os.path.join(output_path, 'build.log') shutil.copy(build_log, install_path) frags = os.path.join(output_path, 'frag.config') if os.path.exists(frags): shutil.copy(frags, install_path) arch = bmeta['arch'] boot_dir = os.path.join(output_path, 'arch', arch, 'boot') kimage_names = KERNEL_IMAGE_NAMES[arch] kimages = [] kimage_file = None for root, _, files in os.walk(boot_dir): for name in kimage_names: if name in files: kimages.append(name) image_path = os.path.join(root, name) shutil.copy(image_path, install_path) for files in os.listdir(output_path): for name in kimage_names: if name == files: kimages.append(name) image_path = os.path.join(output_path, name) shutil.copy(image_path, install_path) if kimages: for name in kimage_names: if name in kimages: kimage_file = name break if not kimage_file: print_flush("Warning: no kernel image found") dts_dir = os.path.join(boot_dir, 'dts') dtbs = os.path.join(install_path, 'dtbs') dtb_list = [] for root, _, files in os.walk(dts_dir): for f in fnmatch.filter(files, '*.dtb'): dtb_path = os.path.join(root, f) dtb_rel = os.path.relpath(dtb_path, dts_dir) dtb_list.append(dtb_rel) dest_path = os.path.join(dtbs, dtb_rel) dest_dir = os.path.dirname(dest_path) if not os.path.exists(dest_dir): os.makedirs(dest_dir) shutil.copy(dtb_path, dest_path) with open(os.path.join(install_path, 'dtbs.json'), 'w') as json_file: json.dump({'dtbs': sorted(dtb_list)}, json_file, indent=4) modules_tarball = None if os.path.exists(mod_path): modules_tarball = 'modules.tar.xz' modules_tarball_path = os.path.join(install_path, modules_tarball) shell_cmd("tar -C{path} -cJf {tarball} .".format( path=mod_path, tarball=modules_tarball_path)) # 'make gen_tar' creates this tarball path kselftest_tarball = 'kselftest-packages/kselftest.tar.xz' kselftest_tarball_path = os.path.join(output_path, '_kselftest_', kselftest_tarball) if os.path.exists(kselftest_tarball_path): kselftest_tarball = os.path.basename(kselftest_tarball_path) shutil.copy(kselftest_tarball_path, os.path.join(install_path, kselftest_tarball)) else: kselftest_tarball = kselftest_tarball_path = None build_env = bmeta['build_environment'] defconfig_full = bmeta['defconfig_full'] if not publish_path: publish_path = '/'.join(item.replace('/', '-') for item in [ tree_name, git_branch, describe, arch, defconfig_full, build_env, ]) bmeta.update({ 'kconfig_fragments': 'frag.config' if os.path.exists(frags) else '', 'kernel_image': kimage_file, 'kernel_config': os.path.basename(dot_config_installed), 'system_map': 'System.map' if os.path.exists(system_map) else None, 'text_offset': '0x{:08x}'.format(text_offset) if text_offset else None, 'dtb_dir': 'dtbs' if os.path.exists(dtbs) else None, 'modules': modules_tarball, 'job': tree_name, 'git_url': tree_url, 'git_branch': git_branch, 'git_describe': describe, 'git_describe_v': describe_v, 'git_commit': git_commit, 'file_server_resource': publish_path, 'kselftests': kselftest_tarball, }) with open(os.path.join(install_path, 'bmeta.json'), 'w') as json_file: json.dump(bmeta, json_file, indent=4, sort_keys=True) return True
def build_kernel(build_env, kdir, arch, defconfig=None, jopt=None, verbose=False, output_path=None, mod_path=None): """Build a linux kernel *build_env* is a BuildEnvironment object *kdir* is the path to the kernel source directory *defconfig* is the name of the kernel defconfig *jopt* is the -j option to pass to make for parallel builds *verbose* is whether to print all the output of the make commands *output_path* is the path to the directory where the binaries are made *mod_path* is the path to where the modules are installed The returned value is True if the build was successful or False if there was any build error. """ cc = build_env.cc cross_compile = build_env.get_cross_compile(arch) or '' cross_compile_compat = build_env.get_cross_compile_compat(arch) or '' use_ccache = shell_cmd("which ccache > /dev/null", True) if jopt is None: jopt = int(shell_cmd("nproc")) + 2 if not output_path: output_path = os.path.join(kdir, 'build') if not os.path.exists(output_path): os.mkdir(output_path) if not mod_path: mod_path = os.path.join(output_path, '_modules_') build_log = 'build.log' log_file = os.path.join(output_path, build_log) dot_config = os.path.join(output_path, '.config') if os.path.exists(log_file): os.unlink(log_file) opts = { 'KBUILD_BUILD_USER': '******', } kwargs = { 'kdir': kdir, 'arch': arch, 'cc': cc, 'cross_compile': cross_compile, 'cross_compile_compat': cross_compile_compat, 'use_ccache': use_ccache, 'output': output_path, 'silent': not verbose, 'log_file': log_file, 'opts': opts, } start_time = time.time() defconfig_extras = [] if defconfig: result = _make_defconfig( defconfig, kwargs, defconfig_extras, verbose, log_file) elif os.path.exists(dot_config): print_flush("Re-using {}".format(dot_config)) result = True else: print_flush("ERROR: Missing kernel config") result = False if result: if _kernel_config_enabled(dot_config, 'XIP_KERNEL'): target = 'xipImage' elif _kernel_config_enabled(dot_config, 'SYS_SUPPORTS_ZBOOT'): target = 'vmlinuz' else: target = MAKE_TARGETS.get(arch) result = _run_make(jopt=jopt, target=target, **kwargs) mods = _kernel_config_enabled(dot_config, 'MODULES') if result and mods: result = _run_make(jopt=jopt, target='modules', **kwargs) if result and _kernel_config_enabled(dot_config, 'OF_FLATTREE'): dts_tree = os.path.join(kdir, 'arch/{}/boot/dts'.format(arch)) if os.path.exists(dts_tree): result = _run_make(target='dtbs', **kwargs) build_time = time.time() - start_time if result and mods: if os.path.exists(mod_path): shutil.rmtree(mod_path) os.makedirs(mod_path) opts.update({ 'INSTALL_MOD_PATH': mod_path, 'INSTALL_MOD_STRIP': '1', 'STRIP': "{}strip".format(cross_compile), }) result = _run_make(target='modules_install', **kwargs) # kselftest if result and "kselftest" in defconfig_extras: kselftest_install_path = os.path.join(output_path, '_kselftest_') if os.path.exists(kselftest_install_path): shutil.rmtree(kselftest_install_path) opts.update({ 'INSTALL_PATH': kselftest_install_path, }) # # Ideally this should just be a 'make kselftest-install', but # due to bugs with O= in kselftest Makefile, this has to be # 'make -C tools/testing/selftests install' # kwargs.update({ 'kdir': os.path.join(kdir, 'tools/testing/selftests') }) opts.update({ 'FORMAT': '.xz', }) # 'gen_tar' target does 'make install' and creates tarball result = _run_make(target='gen_tar', **kwargs) cc_version_cmd = "{}{} --version 2>&1".format( cross_compile if cross_compile and cc == 'gcc' else '', cc) cc_version_full = shell_cmd(cc_version_cmd).splitlines()[0] bmeta = { 'build_threads': jopt, 'build_time': round(build_time, 2), 'status': 'PASS' if result is True else 'FAIL', 'arch': arch, 'cross_compile': cross_compile, 'compiler': cc, 'compiler_version': build_env.cc_version, 'compiler_version_full': cc_version_full, 'build_environment': build_env.name, 'build_log': build_log, 'build_platform': platform.uname(), } if defconfig: defconfig_target = defconfig.split('+')[0] bmeta.update({ 'defconfig': defconfig_target, 'defconfig_full': '+'.join([defconfig_target] + defconfig_extras), }) else: bmeta.update({ 'defconfig': 'none', 'defconfig_full': 'none', }) vmlinux_file = os.path.join(output_path, 'vmlinux') if os.path.isfile(vmlinux_file): vmlinux_meta = kernelci.elf.read(vmlinux_file) bmeta.update(vmlinux_meta) bmeta['vmlinux_file_size'] = os.stat(vmlinux_file).st_size with open(os.path.join(output_path, 'bmeta.json'), 'w') as json_file: json.dump(bmeta, json_file, indent=4, sort_keys=True) return result
def _make_defconfig(defconfig, kwargs, extras, verbose, log_file): kdir, output_path = (kwargs.get(k) for k in ('kdir', 'output')) result = True defconfig_kwargs = dict(kwargs) defconfig_opts = dict(defconfig_kwargs['opts']) defconfig_kwargs['opts'] = defconfig_opts tmpfile_fd, tmpfile_path = tempfile.mkstemp(prefix='kconfig-') tmpfile = os.fdopen(tmpfile_fd, 'w') tmpfile_used = False defs = defconfig.split('+') target = defs.pop(0) for d in defs: if d.startswith('KCONFIG_'): config, value = d.split('=') defconfig_opts[config] = value extras.append(d) elif d.startswith('CONFIG_'): tmpfile.write(d + '\n') extras.append(d) tmpfile_used = True else: frag_path = os.path.join(kdir, d) if os.path.exists(frag_path): with open(frag_path) as frag: tmpfile.write("\n# fragment from : {}\n".format(d)) tmpfile.writelines(frag) tmpfile_used = True extras.append(os.path.basename(os.path.splitext(d)[0])) else: print_flush("Fragment not found: {}".format(frag_path)) result = False tmpfile.flush() if not _run_make(target=target, **defconfig_kwargs): result = False if result and tmpfile_used: kconfig_frag_name = 'frag.config' kconfig_frag = os.path.join(output_path, kconfig_frag_name) shutil.copy(tmpfile_path, kconfig_frag) os.chmod(kconfig_frag, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH) rel_path = os.path.relpath(output_path, kdir) cc = kwargs['cc'] cc_env = ( "export LLVM=1" if cc.startswith('clang') else "export HOSTCC={cc}\nexport CC={cc}".format(cc=cc) ) cmd = """ set -e cd {kdir} {cc_env} export ARCH={arch} export CROSS_COMPILE={cross} export CROSS_COMPILE_COMPAT={cross_compat} scripts/kconfig/merge_config.sh -O {output} '{base}' '{frag}' {redir} """.format(kdir=kdir, arch=kwargs['arch'], cc_env=cc_env, cross=kwargs['cross_compile'], output=rel_path, cross_compat=kwargs['cross_compile_compat'], base=os.path.join(rel_path, '.config'), frag=os.path.join(rel_path, kconfig_frag_name), redir='> /dev/null' if not verbose else '') print_flush(cmd.strip()) if log_file: cmd = _output_to_file(cmd, log_file, kdir) result = shell_cmd(cmd, True) tmpfile.close() os.unlink(tmpfile_path) return result