def add_automatic_backports(args): disable_list = [] export = re.compile(r'^EXPORT_SYMBOL(_GPL)?\((?P<sym>[^\)]*)\)') bpi = kconfig.get_backport_info(os.path.join(args.bpid.target_dir, 'compat', 'Kconfig')) configtree = kconfig.ConfigTree(os.path.join(args.bpid.target_dir, 'Kconfig'), args.bpid) ignore=['Kconfig.kernel', 'Kconfig.versions'] configtree.verify_sources(ignore=ignore) git_debug_snapshot(args, "verify sources for automatic backports") all_selects = configtree.all_selects() for sym, vals in bpi.items(): if sym.startswith('BPAUTO_BUILD_'): if not sym[13:] in all_selects: disable_list.append(sym) continue symtype, module_name, c_files, h_files = vals # first copy files files = [] for f in c_files: files.append((f, os.path.join('compat', automatic_backport_mangle_c_file(f)))) for f in h_files: files.append((os.path.join('include', f), os.path.join('include', os.path.dirname(f), 'backport-' + os.path.basename(f)))) if args.git_revision: copy_git_files(args.kerneldir, files, args.git_revision, args.bpid.target_dir) else: copy_files(args.kerneldir, files, args.bpid.target_dir) # now add the Makefile line mf = open(os.path.join(args.bpid.target_dir, 'compat', 'Makefile'), 'a+') o_files = [automatic_backport_mangle_c_file(f)[:-1] + 'o' for f in c_files] if symtype == 'tristate': if not module_name: raise Exception('backporting a module requires a #module-name') for of in o_files: mf.write('%s-objs += %s\n' % (module_name, of)) mf.write('obj-$(%s%s) += %s.o\n' % (args.bpid.full_prefix, sym, module_name)) elif symtype == 'bool': mf.write('compat-$(%s%s) += %s\n' % (args.bpid.full_prefix, sym, ' '.join(o_files))) # finally create the include file syms = [] for f in c_files: for l in open(os.path.join(args.bpid.target_dir, 'compat', automatic_backport_mangle_c_file(f)), 'r'): m = export.match(l) if m: syms.append(m.group('sym')) for f in h_files: outf = open(os.path.join(args.bpid.target_dir, 'include', f), 'w') outf.write('/* Automatically created during backport process */\n') outf.write('#ifndef %s%s\n' % (args.bpid.full_prefix, sym)) outf.write('#include_next <%s>\n' % f) outf.write('#else\n'); for s in syms: outf.write('#undef %s\n' % s) outf.write('#define %s LINUX_BACKPORT(%s)\n' % (s, s)) outf.write('#include <%s>\n' % (os.path.dirname(f) + '/backport-' + os.path.basename(f), )) outf.write('#endif /* %s%s */\n' % (args.bpid.full_prefix, sym)) return disable_list
def process(kerneldir, copy_list_file, git_revision=None, bpid=None, clean=False, refresh=False, base_name="Linux", gitdebug=False, verbose=False, extra_driver=[], kup=False, kup_test=False, test_cocci=None, profile_cocci=None, logwrite=lambda x:None, git_tracked_version=False): class Args(object): def __init__(self, kerneldir, copy_list_file, git_revision, bpid, clean, refresh, base_name, gitdebug, verbose, extra_driver, kup, kup_test, test_cocci, profile_cocci): self.kerneldir = kerneldir self.copy_list = copy_list_file self.git_revision = git_revision self.bpid = bpid self.clean = clean self.refresh = refresh self.base_name = base_name self.gitdebug = gitdebug self.verbose = verbose self.extra_driver = extra_driver self.kup = kup self.kup_test = kup_test self.test_cocci = test_cocci self.profile_cocci = profile_cocci if self.test_cocci or self.profile_cocci: self.gitdebug = True def git_paranoia(tree=None, logwrite=lambda x:None): data = git.paranoia(tree) if (data['r'] != 0): logwrite('Cannot use %s' % tree) logwrite('%s' % data['output']) sys.exit(data['r']) else: logwrite('Validated tree: %s' % tree) args = Args(kerneldir, copy_list_file, git_revision, bpid, clean, refresh, base_name, gitdebug, verbose, extra_driver, kup, kup_test, test_cocci, profile_cocci) rel_prep = None if bpid.integrate: if args.kup_test or args.test_cocci or args.profile_cocci or args.refresh: logwrite('Cannot use integration with:\n\tkup_test\n\ttest_cocci\n\tprofile_cocci\n\trefresh\n'); sys.exit(1) # start processing ... if (args.kup or args.kup_test): git_paranoia(source_dir, logwrite) git_paranoia(kerneldir, logwrite) rel_describe = git.describe(rev=None, tree=source_dir, extra_args=['--dirty']) release = os.path.basename(bpid.target_dir) version = release.replace("backports-", "") rel_prep = get_rel_prep(version) if (not rel_prep): logwrite('Invalid backports release name: %s' % release) logwrite('For rules on the release name see upload_release()') sys.exit(1) rel_type = "linux-stable" if (not rel_prep['stable']): rel_type = "linux-next" if (rel_prep['expected_tag'] != rel_describe): logwrite('Unexpected %s based backports release tag on' % rel_type) logwrite('the backports tree tree: %s\n' % rel_describe) logwrite('You asked to make a release with this ') logwrite('directory name: %s' % release) logwrite('The actual expected tag we should find on') logwrite('the backports tree then is: %s\n' % rel_prep['expected_tag']) logwrite('For rules on the release name see upload_release()') sys.exit(1) copy_list = read_copy_list(args.copy_list) deplist = read_dependencies(os.path.join(source_dir, 'dependencies')) # validate output directory check_output_dir(bpid.target_dir, args.clean) # do the copy backport_integrate_files = [ ('Makefile.kernel', 'Makefile'), ('Kconfig.integrate', 'Kconfig'), ] backport_package_files = [(x, x) for x in [ 'Makefile', 'kconf/', 'Makefile.real', 'Makefile.kernel', 'Kconfig.package.hacks', 'scripts/', '.blacklist.map', '.gitignore', 'Makefile.build'] ] backport_package_files += [ ('Kconfig.package', 'Kconfig'), ] backport_files = [(x, x) for x in [ 'Kconfig.sources', 'compat/', 'backport-include/', ]] if not bpid.integrate: backport_files += backport_package_files else: backport_files += backport_integrate_files if not args.git_revision: logwrite('Copy original source files ...') else: logwrite('Get original source files from git ...') copy_files(os.path.join(source_dir, 'backport'), backport_files, bpid.target_dir) git_debug_init(args) if not args.git_revision: copy_files(args.kerneldir, copy_list, bpid.target_dir) else: copy_git_files(args.kerneldir, copy_list, args.git_revision, bpid.target_dir) # FIXME: should we add a git version of this (e.g. --git-extra-driver)? for src, copy_list in args.extra_driver: if (args.kup or args.kup_test): git_paranoia(src) copy_files(src, read_copy_list(open(copy_list, 'r')), bpid.target_dir) git_debug_snapshot(args, 'Add driver sources') disable_list = add_automatic_backports(args) if git_tracked_version: backports_version = "(see git)" kernel_version = "(see git)" else: backports_version = git.describe(tree=source_dir, extra_args=['--long']) kernel_version = git.describe(rev=args.git_revision or 'HEAD', tree=args.kerneldir, extra_args=['--long']) if not bpid.integrate: f = open(os.path.join(bpid.target_dir, 'versions'), 'w') f.write('BACKPORTS_VERSION="%s"\n' % backports_version) f.write('BACKPORTED_KERNEL_VERSION="%s"\n' % kernel_version) f.write('BACKPORTED_KERNEL_NAME="%s"\n' % args.base_name) if git_tracked_version: f.write('BACKPORTS_GIT_TRACKED="backport tracker ID: $(shell git rev-parse HEAD 2>/dev/null || echo \'not built in git tree\')"\n') f.close() git_debug_snapshot(args, "add versions files") else: kconf_regexes = [ (re.compile(r'.*(?P<key>%%BACKPORT_DIR%%)'), '%%BACKPORT_DIR%%', 'backports/'), (re.compile(r'.*(?P<key>%%BACKPORTS_VERSION%%).*'), '%%BACKPORTS_VERSION%%', backports_version), (re.compile(r'.*(?P<key>%%BACKPORTED_KERNEL_VERSION%%).*'), '%%BACKPORTED_KERNEL_VERSION%%', kernel_version), (re.compile(r'.*(?P<key>%%BACKPORTED_KERNEL_NAME%%).*'), '%%BACKPORTED_KERNEL_NAME%%', args.base_name), ] out = '' for l in open(os.path.join(bpid.target_dir, 'Kconfig'), 'r'): for r in kconf_regexes: m = r[0].match(l) if m: l = re.sub(r'(' + r[1] + ')', r'' + r[2] + '', l) out += l outf = open(os.path.join(bpid.target_dir, 'Kconfig'), 'w') outf.write(out) outf.close() git_debug_snapshot(args, "modify top level backports/Kconfig with backports identity") if disable_list: # No need to verify_sources() as compat's Kconfig has no 'source' call bpcfg = kconfig.ConfigTree(os.path.join(bpid.target_dir, 'compat', 'Kconfig'), bpid) bpcfg.disable_symbols(disable_list) git_debug_snapshot(args, 'Add automatic backports') failure = apply_patches(args, "backport", source_dir, 'patches', bpid.target_dir, logwrite) if failure: return failure # Kernel integration requires Kconfig.versions already generated for you, # we cannot do this for a package as we have no idea what kernel folks # will be using. if bpid.integrate: kver = gen_version.kernelversion(bpid.project_dir) rel_specs = gen_version.get_rel_spec_stable(kver) if not rel_specs: logwrite('Cannot parse source kernel version, update parser') sys.exit(1) data = gen_version.genkconfig_versions(rel_specs) fo = open(os.path.join(bpid.target_dir, 'Kconfig.versions'), 'w') fo.write(data) fo.close() git_debug_snapshot(args, "generate kernel version requirement Kconfig file") # some post-processing is required configtree = kconfig.ConfigTree(os.path.join(bpid.target_dir, 'Kconfig'), bpid) ignore=['Kconfig.kernel', 'Kconfig.versions', 'Kconfig.local'] configtree.verify_sources(ignore=ignore) git_debug_snapshot(args, "verify sources on top level backports Kconfig") orig_symbols = configtree.symbols() logwrite('Modify Kconfig tree ...') configtree.prune_sources(ignore=ignore) git_debug_snapshot(args, "prune Kconfig tree") if not bpid.integrate: configtree.force_tristate_modular() git_debug_snapshot(args, "force tristate options modular") ignore = [os.path.join(bpid.target_dir, x) for x in [ 'Kconfig.package.hacks', 'Kconfig.versions', 'Kconfig.local', 'Kconfig', ] ] configtree.adjust_backported_configs(ignore=ignore, orig_symbols=orig_symbols) git_debug_snapshot(args, "adjust backports config symbols we port") configtree.modify_selects() git_debug_snapshot(args, "convert select to depends on") symbols = configtree.symbols() # write local symbol list -- needed during packaging build if not bpid.integrate: f = open(os.path.join(bpid.target_dir, 'local-symbols'), 'w') for sym in symbols: f.write('%s=\n' % sym) f.close() git_debug_snapshot(args, "add symbols files") # also write Kconfig.local, representing all local symbols # with a BACKPORTED_ prefix f = open(os.path.join(bpid.target_dir, 'Kconfig.local'), 'w') for sym in symbols: f.write('config BACKPORTED_%s\n' % sym) f.write('\ttristate\n') f.write('\tdefault %s\n' % sym) f.close() git_debug_snapshot(args, "add Kconfig.local") # add defconfigs that we want defconfigs_dir = os.path.join(source_dir, 'backport', 'defconfigs') os.mkdir(os.path.join(bpid.target_dir, 'defconfigs')) for dfbase in os.listdir(defconfigs_dir): copy_defconfig = True dfsrc = os.path.join(defconfigs_dir, dfbase) for line in open(dfsrc, 'r'): if not '=' in line: continue line_ok = False for sym in symbols: if sym + '=' in line: line_ok = True break if not line_ok: copy_defconfig = False break if copy_defconfig: shutil.copy(dfsrc, os.path.join(bpid.target_dir, 'defconfigs', dfbase)) git_debug_snapshot(args, "add (useful) defconfig files") logwrite('Rewrite Makefiles and Kconfig files ...') # rewrite Makefile and source symbols # symbols we know only we can provide under the backport project prefix # for which we need an exception. skip_orig_syms = [ bpid.project_prefix + x for x in [ 'INTEGRATE', ] ] parse_orig_syms = [x for x in orig_symbols if x not in skip_orig_syms ] regexes = [] for some_symbols in [parse_orig_syms[i:i + 50] for i in range(0, len(parse_orig_syms), 50)]: r = 'CONFIG_((' + '|'.join([s + '(_MODULE)?' for s in some_symbols]) + ')([^A-Za-z0-9_]|$))' regexes.append(re.compile(r, re.MULTILINE)) for root, dirs, files in os.walk(bpid.target_dir): # don't go into .git dir (possible debug thing) if '.git' in dirs: dirs.remove('.git') for f in files: data = open(os.path.join(root, f), 'r').read() for r in regexes: data = r.sub(r'' + bpid.full_prefix + '\\1', data) data = re.sub(r'\$\(srctree\)', '$(backport_srctree)', data) data = re.sub(r'-Idrivers', '-I$(backport_srctree)/drivers', data) if bpid.integrate: data = re.sub(r'CPTCFG_', bpid.full_prefix, data) fo = open(os.path.join(root, f), 'w') fo.write(data) fo.close() git_debug_snapshot(args, "rename config symbol / srctree usage") # disable unbuildable Kconfig symbols and stuff Makefiles that doesn't exist if bpid.integrate: maketree = make.MakeTree(os.path.join(bpid.target_dir, 'Makefile')) else: maketree = make.MakeTree(os.path.join(bpid.target_dir, 'Makefile.kernel')) disable_kconfig = [] disable_makefile = [] for sym in maketree.get_impossible_symbols(): disable_kconfig.append(sym[7:]) disable_makefile.append(sym[7:]) configtree.disable_symbols(disable_kconfig) git_debug_snapshot(args, "disable impossible kconfig symbols") # add kernel version dependencies to Kconfig, from the dependency list # we read previously for sym in tuple(deplist.keys()): new = [] for dep in deplist[sym]: if "kconfig:" in dep: kconfig_expr = dep.replace('kconfig: ', '') new.append(kconfig_expr) elif (dep == "DISABLE"): new.append('BACKPORT_DISABLED_KCONFIG_OPTION') else: new.append('!KERNEL_%s' % dep.replace('.', '_')) if bpid.integrate: deplist[sym] = ["BACKPORT_" + x for x in new] else: deplist[sym] = new configtree.add_dependencies(deplist) git_debug_snapshot(args, "add kernel version dependencies") # disable things in makefiles that can't be selected and that the # build shouldn't recurse into because they don't exist -- if we # don't do that then a symbol from the kernel could cause the build # to attempt to recurse and fail # # Note that we split the regex after 50 symbols, this is because of a # limitation in the regex implementation (it only supports 100 nested # groups -- 50 seemed safer and is still fast) regexes = [] for some_symbols in [disable_makefile[i:i + 50] for i in range(0, len(disable_makefile), 50)]: r = '^(([^#].*((' + bpid.full_prefix_resafe + '|CONFIG_)(' + '|'.join([s for s in some_symbols]) + ')))\W)' regexes.append(re.compile(r, re.MULTILINE)) for f in maketree.get_makefiles(): data = open(f, 'r').read() for r in regexes: data = r.sub(r'#\1', data) fo = open(f, 'w') fo.write(data) fo.close() git_debug_snapshot(args, "disable unsatisfied Makefile parts") if bpid.integrate: f = open(os.path.join(bpid.project_dir, 'Kconfig'), 'a') f.write('source "backports/Kconfig"\n') f.close() git_debug_snapshot(args, "hooked backport to top level Kconfig") failure = apply_patches(args, "integration", source_dir, 'integration-patches/', bpid.project_dir, logwrite) if failure: return failure if (args.kup or args.kup_test): req = reqs.Req() req.kup() if not req.reqs_match(): sys.exit(1) upload_release(args, rel_prep, logwrite=logwrite) logwrite('Done!') return 0
def process(kerneldir, outdir, copy_list_file, git_revision=None, clean=False, refresh=False, base_name="Linux", gitdebug=False, verbose=False, extra_driver=[], kup=False, kup_test=False, logwrite=lambda x: None, git_tracked_version=False): class Args(object): def __init__(self, kerneldir, outdir, copy_list_file, git_revision, clean, refresh, base_name, gitdebug, verbose, extra_driver, kup, kup_test): self.kerneldir = kerneldir self.outdir = outdir self.copy_list = copy_list_file self.git_revision = git_revision self.clean = clean self.refresh = refresh self.base_name = base_name self.gitdebug = gitdebug self.verbose = verbose self.extra_driver = extra_driver self.kup = kup self.kup_test = kup_test def git_paranoia(tree=None, logwrite=lambda x: None): data = git.paranoia(tree) if (data['r'] != 0): logwrite('Cannot use %s' % tree) logwrite('%s' % data['output']) sys.exit(data['r']) else: logwrite('Validated tree: %s' % tree) args = Args(kerneldir, outdir, copy_list_file, git_revision, clean, refresh, base_name, gitdebug, verbose, extra_driver, kup, kup_test) rel_prep = None # start processing ... if (args.kup or args.kup_test): git_paranoia(source_dir, logwrite) git_paranoia(kerneldir, logwrite) rel_describe = git.describe(rev=None, tree=source_dir, extra_args=['--dirty']) release = os.path.basename(args.outdir) version = release.replace("backports-", "") rel_prep = get_rel_prep(version) if (not rel_prep): logwrite('Invalid backports release name: %s' % release) logwrite('For rules on the release name see upload_release()') sys.exit(1) rel_type = "linux-stable" if (not rel_prep['stable']): rel_type = "linux-next" if (rel_prep['expected_tag'] != rel_describe): logwrite('Unexpected %s based backports release tag on' % rel_type) logwrite('the backports tree tree: %s\n' % rel_describe) logwrite('You asked to make a release with this ') logwrite('directory name: %s' % release) logwrite('The actual expected tag we should find on') logwrite('the backports tree then is: %s\n' % rel_prep['expected_tag']) logwrite('For rules on the release name see upload_release()') sys.exit(1) copy_list = read_copy_list(args.copy_list) deplist = read_dependencies(os.path.join(source_dir, 'dependencies')) # validate output directory check_output_dir(args.outdir, args.clean) # do the copy backport_files = [(x, x) for x in [ 'Kconfig', 'Makefile', 'Makefile.build', 'Makefile.kernel', '.gitignore', 'Makefile.real', 'compat/', 'backport-include/', 'kconf/', 'defconfigs/', 'scripts/', '.blacklist.map', 'udev/', ]] if not args.git_revision: logwrite('Copy original source files ...') else: logwrite('Get original source files from git ...') copy_files(os.path.join(source_dir, 'backport'), backport_files, args.outdir) git_debug_init(args) if not args.git_revision: copy_files(args.kerneldir, copy_list, args.outdir) else: copy_git_files(args.kerneldir, copy_list, args.git_revision, args.outdir) # FIXME: should we add a git version of this (e.g. --git-extra-driver)? for src, copy_list in args.extra_driver: if (args.kup or args.kup_test): git_paranoia(src) copy_files(src, read_copy_list(open(copy_list, 'r')), args.outdir) git_debug_snapshot(args, 'Add driver sources') disable_list = add_automatic_backports(args) if disable_list: bpcfg = kconfig.ConfigTree( os.path.join(args.outdir, 'compat', 'Kconfig')) bpcfg.disable_symbols(disable_list) git_debug_snapshot(args, 'Add automatic backports') logwrite('Apply patches ...') patches = [] sempatches = [] for root, dirs, files in os.walk(os.path.join(source_dir, 'patches')): for f in files: if f.endswith('.patch'): patches.append(os.path.join(root, f)) if f.endswith('.cocci'): sempatches.append(os.path.join(root, f)) patches.sort() prefix_len = len(os.path.join(source_dir, 'patches')) + 1 for pfile in patches: print_name = pfile[prefix_len:] # read the patch file p = patch.fromfile(pfile) # complain if it's not a patch if not p: raise Exception('No patch content found in %s' % print_name) # leading / seems to be stripped? if 'dev/null' in p.items[0].source: raise Exception( 'Patches creating files are not supported (in %s)' % print_name) # check if the first file the patch touches exists, if so # assume the patch needs to be applied -- otherwise continue patched_file = '/'.join(p.items[0].source.split('/')[1:]) fullfn = os.path.join(args.outdir, patched_file) if not os.path.exists(fullfn): if args.verbose: logwrite("Not applying %s, not needed" % print_name) continue if args.verbose: logwrite("Applying patch %s" % print_name) if args.refresh: # but for refresh, of course look at all files the patch touches for patchitem in p.items: patched_file = '/'.join(patchitem.source.split('/')[1:]) fullfn = os.path.join(args.outdir, patched_file) shutil.copyfile(fullfn, fullfn + '.orig_file') process = subprocess.Popen(['patch', '-p1'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, stdin=subprocess.PIPE, close_fds=True, universal_newlines=True, cwd=args.outdir) output = process.communicate(input=open(pfile, 'r').read())[0] output = output.split('\n') if output[-1] == '': output = output[:-1] if args.verbose: for line in output: logwrite('> %s' % line) if process.returncode != 0: if not args.verbose: logwrite("Failed to apply changes from %s" % print_name) for line in output: logwrite('> %s' % line) return 2 if args.refresh: pfilef = open(pfile + '.tmp', 'a') pfilef.write(p.top_header) pfilef.flush() for patchitem in p.items: patched_file = '/'.join(patchitem.source.split('/')[1:]) fullfn = os.path.join(args.outdir, patched_file) process = subprocess.Popen([ 'diff', '-p', '-u', patched_file + '.orig_file', patched_file, '--label', 'a/' + patched_file, '--label', 'b/' + patched_file ], stdout=pfilef, close_fds=True, universal_newlines=True, cwd=args.outdir) process.wait() os.unlink(fullfn + '.orig_file') if not process.returncode in (0, 1): logwrite("Failed to diff to refresh %s" % print_name) pfilef.close() os.unlink(pfile + '.tmp') return 3 pfilef.close() os.rename(pfile + '.tmp', pfile) # remove orig/rej files that patch sometimes creates for root, dirs, files in os.walk(args.outdir): for f in files: if f[-5:] == '.orig' or f[-4:] == '.rej': os.unlink(os.path.join(root, f)) git_debug_snapshot(args, "apply backport patch %s" % print_name) sempatches.sort() with tempdir() as t: if not args.gitdebug: # combine all spatches fn = os.path.join(t, 'combined.cocci') f = open(fn, 'w') for cocci_file in sempatches: for l in open(cocci_file, 'r'): f.write(l) f.write('\n') f.close() sempatches = [fn] prefix_len = 0 else: prefix_len = len(os.path.join(source_dir, 'patches')) + 1 for cocci_file in sempatches: print_name = cocci_file[prefix_len:] if args.verbose: logwrite("Applying patch %s" % print_name) process = subprocess.Popen([ 'spatch', '--sp-file', cocci_file, '--in-place', '--backup-suffix', '.cocci_backup', '--dir', '.' ], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True, universal_newlines=True, cwd=args.outdir) output = process.communicate()[0] output = output.split('\n') if output[-1] == '': output = output[:-1] if args.verbose: for line in output: logwrite('> %s' % line) if process.returncode != 0: if not args.verbose: logwrite("Failed to apply changes from %s" % print_name) for line in output: logwrite('> %s' % line) return 2 # remove cocci_backup files for root, dirs, files in os.walk(args.outdir): for f in files: if f.endswith('.cocci_backup'): os.unlink(os.path.join(root, f)) git_debug_snapshot(args, "apply backport patch %s" % print_name) # some post-processing is required configtree = kconfig.ConfigTree(os.path.join(args.outdir, 'Kconfig')) logwrite('Modify Kconfig tree ...') configtree.prune_sources(ignore=['Kconfig.kernel', 'Kconfig.versions']) git_debug_snapshot(args, "prune Kconfig tree") configtree.force_tristate_modular() git_debug_snapshot(args, "force tristate options modular") configtree.modify_selects() git_debug_snapshot(args, "convert select to depends on") # write the versioning file if git_tracked_version: backports_version = "(see git)" kernel_version = "(see git)" else: backports_version = git.describe(tree=source_dir, extra_args=['--long']) kernel_version = git.describe(rev=args.git_revision or 'HEAD', tree=args.kerneldir, extra_args=['--long']) f = open(os.path.join(args.outdir, 'versions'), 'w') f.write('BACKPORTS_VERSION="%s"\n' % backports_version) f.write('BACKPORTED_KERNEL_VERSION="%s"\n' % kernel_version) f.write('BACKPORTED_KERNEL_NAME="%s"\n' % args.base_name) if git_tracked_version: f.write( 'BACKPORTS_GIT_TRACKED="backport tracker ID: $(shell git rev-parse HEAD 2>/dev/null || echo \'not built in git tree\')"\n' ) f.close() symbols = configtree.symbols() # write local symbol list -- needed during build f = open(os.path.join(args.outdir, '.local-symbols'), 'w') for sym in symbols: f.write('%s=\n' % sym) f.close() git_debug_snapshot(args, "add versions/symbols files") logwrite('Rewrite Makefiles and Kconfig files ...') # rewrite Makefile and source symbols regexes = [] for some_symbols in [ symbols[i:i + 50] for i in range(0, len(symbols), 50) ]: r = 'CONFIG_((' + '|'.join([s + '(_MODULE)?' for s in some_symbols ]) + ')([^A-Za-z0-9_]|$))' regexes.append(re.compile(r, re.MULTILINE)) for root, dirs, files in os.walk(args.outdir): # don't go into .git dir (possible debug thing) if '.git' in dirs: dirs.remove('.git') for f in files: data = open(os.path.join(root, f), 'r').read() for r in regexes: data = r.sub(r'CPTCFG_\1', data) data = re.sub(r'\$\(srctree\)', '$(backport_srctree)', data) data = re.sub(r'-Idrivers', '-I$(backport_srctree)/drivers', data) fo = open(os.path.join(root, f), 'w') fo.write(data) fo.close() git_debug_snapshot(args, "rename config symbol / srctree usage") # disable unbuildable Kconfig symbols and stuff Makefiles that doesn't exist maketree = make.MakeTree(os.path.join(args.outdir, 'Makefile.kernel')) disable_kconfig = [] disable_makefile = [] for sym in maketree.get_impossible_symbols(): disable_kconfig.append(sym[7:]) disable_makefile.append(sym[7:]) configtree.disable_symbols(disable_kconfig) git_debug_snapshot(args, "disable impossible kconfig symbols") # add kernel version dependencies to Kconfig, from the dependency list # we read previously for sym in tuple(deplist.keys()): new = [] for dep in deplist[sym]: if "kconfig:" in dep: kconfig_expr = dep.replace('kconfig: ', '') new.append(kconfig_expr) elif (dep == "DISABLE"): new.append('BACKPORT_DISABLED_KCONFIG_OPTION') else: new.append('!BACKPORT_KERNEL_%s' % dep.replace('.', '_')) deplist[sym] = new configtree.add_dependencies(deplist) git_debug_snapshot(args, "add kernel version dependencies") # disable things in makefiles that can't be selected and that the # build shouldn't recurse into because they don't exist -- if we # don't do that then a symbol from the kernel could cause the build # to attempt to recurse and fail # # Note that we split the regex after 50 symbols, this is because of a # limitation in the regex implementation (it only supports 100 nested # groups -- 50 seemed safer and is still fast) regexes = [] for some_symbols in [ disable_makefile[i:i + 50] for i in range(0, len(disable_makefile), 50) ]: r = '^([^#].*((CPTCFG|CONFIG)_(' + '|'.join([s for s in some_symbols ]) + ')))' regexes.append(re.compile(r, re.MULTILINE)) for f in maketree.get_makefiles(): data = open(f, 'r').read() for r in regexes: data = r.sub(r'#\1', data) fo = open(f, 'w') fo.write(data) fo.close() git_debug_snapshot(args, "disable unsatisfied Makefile parts") if (args.kup or args.kup_test): upload_release(args, rel_prep, logwrite=logwrite) logwrite('Done!') return 0
def prune(defconfig, verify, basedir): ''' do the entire prune for the given defconfig, with or without verification ''' allowed_unset_syms = [ x.strip() for x in open('intc-scripts/publishable-options') ] # also use publishable-options-<defconfig> but don't fail if it doesn't exist try: allowed_unset_syms.extend([ x.strip() for x in open('intc-scripts/publishable-options-%s' % defconfig) ]) except IOError: pass en, dis = parse_defconfig(os.path.join(basedir, 'defconfigs', defconfig)) # get the symbols that can be found starting from iwlwifi's Kconfig # (follows the files source'd into this one, which is Kconfig.noupstream) ct = kconfig.ConfigTree('drivers/net/wireless/intel/iwlwifi/Kconfig', basedir=basedir) syms = ct.symbols() # verify the symbols are enabled or disabled or allowed to be unset for sym in syms: if not sym in en and not sym in dis and not sym in allowed_unset_syms: if verify: assert False, '%s state not known!' % sym else: print(('%s state not known, assuming allowed!' % sym)) drv_dis = [s for s in dis if s in syms] ct.remove_symbols(drv_dis) # do the same thing over again for mac80211 ct = kconfig.ConfigTree('net/mac80211/Kconfig', basedir=basedir) syms = ct.symbols() for sym in syms: if not sym in en and not sym in dis and not sym in allowed_unset_syms: if verify: assert False, '%s state not known!' % sym else: print(('%s state not known, assuming allowed!' % sym)) mac_dis = [s for s in dis if s in syms] ct.remove_symbols(mac_dis) dis = drv_dis + mac_dis run_extras(basedir, dis) includes = _prune_src( os.path.join(basedir, 'drivers', 'net', 'wireless', 'intel', 'iwlwifi'), dis) inc = _prune_src(os.path.join(basedir, 'net', 'mac80211'), dis) includes.extend(inc) inc = _prune_src(os.path.join(basedir, 'include', 'net'), dis) includes.extend(inc) local = '' for l in open(os.path.join(basedir, 'local-symbols'), 'r'): if not l[:-2] in dis: local += l lf = open(os.path.join(basedir, 'local-symbols'), 'w') lf.write(local) lf.close() if verify: verify_files_ok = [ l.strip() for l in open( os.path.join(basedir, 'intc-scripts', 'publishable-files')) ] try: verify_files_ok.extend([ l.strip() for l in open( os.path.join(basedir, 'intc-scripts', 'publishable-files-%s' % defconfig)) ]) except: pass # strip or remove defconfigs try: publishable_defconfigs = [ l.strip() for l in open( os.path.join(basedir, 'intc-scripts', 'publishable-defconfigs-%s' % defconfig)) ] except: publishable_defconfigs = [] defconfigs = os.path.join(basedir, 'defconfigs') for f in os.listdir(defconfigs): if f in publishable_defconfigs: strip_defconfig(os.path.join(defconfigs, f), dis) continue if f == defconfig: continue os.unlink(os.path.join(defconfigs, f)) # point the default defconfig to the prune one os.unlink(os.path.join(basedir, 'defconfig')) os.symlink('defconfigs/' + defconfig, 'defconfig') # clean up scripts (other than the ones we should publish) publishable_scripts = [ l.strip() for l in open( os.path.join(basedir, 'intc-scripts', 'publishable-files')) ] publishable_scripts = [ x[13:] for x in publishable_scripts if x.startswith('intc-scripts/') ] intc_scripts_dir = os.path.join(basedir, 'intc-scripts') for f in os.listdir(intc_scripts_dir): if f in publishable_scripts: continue rmf = os.path.join(intc_scripts_dir, f) if os.path.isdir(rmf): shutil.rmtree(rmf) else: os.unlink(rmf) os.unlink(os.path.join(basedir, 'shell.nix')) shutil.rmtree(os.path.join(basedir, 'nix')) shutil.rmtree(os.path.join(basedir, 'Documentation')) for root, dirs, files in os.walk( os.path.join(basedir, 'drivers/net/wireless/intel/iwlwifi')): for file in files: if file[-2:] == '.h': if not file in includes: os.unlink(os.path.join(root, file)) for root, dirs, files in os.walk(os.path.join( basedir, 'drivers/net/wireless/intel/iwlwifi'), topdown=False): if not files: os.rmdir(root) if verify: for root, dirs, files in os.walk(basedir): for d in list(dirs): full = os.path.join(root, d, '*') if full[:2] == './': full = full[2:] if full in verify_files_ok: dirs.remove(d) for e in itertools.chain(dirs, files): full = os.path.join(root, e) if full[:2] == './': full = full[2:] assert full in verify_files_ok, "Wrong file: %s" % full