def process(kerneldir, copy_list_file, git_revision=None,
bpid=None,
clean=False, refresh=False, base_name="Linux", gitdebug=False,
verbose=False, extra_driver=[], kup=False,
kup_test=False,
test_cocci=None,
profile_cocci=None,
logwrite=lambda x:None,
git_tracked_version=False):
class Args(object):
def __init__(self, kerneldir, copy_list_file,
git_revision, bpid, clean, refresh, base_name,
gitdebug, verbose, extra_driver, kup,
kup_test,
test_cocci,
profile_cocci):
self.kerneldir = kerneldir
self.copy_list = copy_list_file
self.git_revision = git_revision
self.bpid = bpid
self.clean = clean
self.refresh = refresh
self.base_name = base_name
self.gitdebug = gitdebug
self.verbose = verbose
self.extra_driver = extra_driver
self.kup = kup
self.kup_test = kup_test
self.test_cocci = test_cocci
self.profile_cocci = profile_cocci
if self.test_cocci or self.profile_cocci:
self.gitdebug = True
def git_paranoia(tree=None, logwrite=lambda x:None):
data = git.paranoia(tree)
if (data['r'] != 0):
logwrite('Cannot use %s' % tree)
logwrite('%s' % data['output'])
sys.exit(data['r'])
else:
logwrite('Validated tree: %s' % tree)
args = Args(kerneldir, copy_list_file,
git_revision, bpid, clean, refresh, base_name,
gitdebug, verbose, extra_driver, kup, kup_test,
test_cocci, profile_cocci)
rel_prep = None
if bpid.integrate:
if args.kup_test or args.test_cocci or args.profile_cocci or args.refresh:
logwrite('Cannot use integration with:\n\tkup_test\n\ttest_cocci\n\tprofile_cocci\n\trefresh\n');
sys.exit(1)
# start processing ...
if (args.kup or args.kup_test):
git_paranoia(source_dir, logwrite)
git_paranoia(kerneldir, logwrite)
rel_describe = git.describe(rev=None, tree=source_dir, extra_args=['--dirty'])
release = os.path.basename(bpid.target_dir)
version = release.replace("backports-", "")
rel_prep = get_rel_prep(version)
if (not rel_prep):
logwrite('Invalid backports release name: %s' % release)
logwrite('For rules on the release name see upload_release()')
sys.exit(1)
rel_type = "linux-stable"
if (not rel_prep['stable']):
rel_type = "linux-next"
if (rel_prep['expected_tag'] != rel_describe):
logwrite('Unexpected %s based backports release tag on' % rel_type)
logwrite('the backports tree tree: %s\n' % rel_describe)
logwrite('You asked to make a release with this ')
logwrite('directory name: %s' % release)
logwrite('The actual expected tag we should find on')
logwrite('the backports tree then is: %s\n' % rel_prep['expected_tag'])
logwrite('For rules on the release name see upload_release()')
sys.exit(1)
copy_list = read_copy_list(args.copy_list)
deplist = read_dependencies(os.path.join(source_dir, 'dependencies'))
# validate output directory
check_output_dir(bpid.target_dir, args.clean)
# do the copy
backport_integrate_files = [
('Makefile.kernel', 'Makefile'),
('Kconfig.integrate', 'Kconfig'),
]
backport_package_files = [(x, x) for x in [
'Makefile',
'kconf/',
'Makefile.real',
'Makefile.kernel',
'Kconfig.package.hacks',
'scripts/',
'.blacklist.map',
'.gitignore',
'Makefile.build'] ]
backport_package_files += [
('Kconfig.package', 'Kconfig'),
]
backport_files = [(x, x) for x in [
'Kconfig.sources',
'compat/',
'backport-include/',
]]
if not bpid.integrate:
backport_files += backport_package_files
else:
backport_files += backport_integrate_files
if not args.git_revision:
logwrite('Copy original source files ...')
else:
logwrite('Get original source files from git ...')
copy_files(os.path.join(source_dir, 'backport'), backport_files, bpid.target_dir)
git_debug_init(args)
if not args.git_revision:
copy_files(args.kerneldir, copy_list, bpid.target_dir)
else:
copy_git_files(args.kerneldir, copy_list, args.git_revision, bpid.target_dir)
# FIXME: should we add a git version of this (e.g. --git-extra-driver)?
for src, copy_list in args.extra_driver:
if (args.kup or args.kup_test):
git_paranoia(src)
copy_files(src, read_copy_list(open(copy_list, 'r')), bpid.target_dir)
git_debug_snapshot(args, 'Add driver sources')
disable_list = add_automatic_backports(args)
if git_tracked_version:
backports_version = "(see git)"
kernel_version = "(see git)"
else:
backports_version = git.describe(tree=source_dir, extra_args=['--long'])
kernel_version = git.describe(rev=args.git_revision or 'HEAD',
tree=args.kerneldir,
extra_args=['--long'])
if not bpid.integrate:
f = open(os.path.join(bpid.target_dir, 'versions'), 'w')
f.write('BACKPORTS_VERSION="%s"\n' % backports_version)
f.write('BACKPORTED_KERNEL_VERSION="%s"\n' % kernel_version)
f.write('BACKPORTED_KERNEL_NAME="%s"\n' % args.base_name)
if git_tracked_version:
f.write('BACKPORTS_GIT_TRACKED="backport tracker ID: $(shell git rev-parse HEAD 2>/dev/null || echo \'not built in git tree\')"\n')
f.close()
git_debug_snapshot(args, "add versions files")
else:
kconf_regexes = [
(re.compile(r'.*(?P<key>%%BACKPORT_DIR%%)'), '%%BACKPORT_DIR%%', 'backports/'),
(re.compile(r'.*(?P<key>%%BACKPORTS_VERSION%%).*'), '%%BACKPORTS_VERSION%%', backports_version),
(re.compile(r'.*(?P<key>%%BACKPORTED_KERNEL_VERSION%%).*'), '%%BACKPORTED_KERNEL_VERSION%%', kernel_version),
(re.compile(r'.*(?P<key>%%BACKPORTED_KERNEL_NAME%%).*'), '%%BACKPORTED_KERNEL_NAME%%', args.base_name),
]
out = ''
for l in open(os.path.join(bpid.target_dir, 'Kconfig'), 'r'):
for r in kconf_regexes:
m = r[0].match(l)
if m:
l = re.sub(r'(' + r[1] + ')', r'' + r[2] + '', l)
out += l
outf = open(os.path.join(bpid.target_dir, 'Kconfig'), 'w')
outf.write(out)
outf.close()
git_debug_snapshot(args, "modify top level backports/Kconfig with backports identity")
if disable_list:
# No need to verify_sources() as compat's Kconfig has no 'source' call
bpcfg = kconfig.ConfigTree(os.path.join(bpid.target_dir, 'compat', 'Kconfig'), bpid)
bpcfg.disable_symbols(disable_list)
git_debug_snapshot(args, 'Add automatic backports')
failure = apply_patches(args, "backport", source_dir, 'patches', bpid.target_dir, logwrite)
if failure:
return failure
# Kernel integration requires Kconfig.versions already generated for you,
# we cannot do this for a package as we have no idea what kernel folks
# will be using.
if bpid.integrate:
kver = gen_version.kernelversion(bpid.project_dir)
rel_specs = gen_version.get_rel_spec_stable(kver)
if not rel_specs:
logwrite('Cannot parse source kernel version, update parser')
sys.exit(1)
data = gen_version.genkconfig_versions(rel_specs)
fo = open(os.path.join(bpid.target_dir, 'Kconfig.versions'), 'w')
fo.write(data)
fo.close()
git_debug_snapshot(args, "generate kernel version requirement Kconfig file")
# some post-processing is required
configtree = kconfig.ConfigTree(os.path.join(bpid.target_dir, 'Kconfig'), bpid)
ignore=['Kconfig.kernel', 'Kconfig.versions', 'Kconfig.local']
configtree.verify_sources(ignore=ignore)
git_debug_snapshot(args, "verify sources on top level backports Kconfig")
orig_symbols = configtree.symbols()
logwrite('Modify Kconfig tree ...')
configtree.prune_sources(ignore=ignore)
git_debug_snapshot(args, "prune Kconfig tree")
if not bpid.integrate:
configtree.force_tristate_modular()
git_debug_snapshot(args, "force tristate options modular")
ignore = [os.path.join(bpid.target_dir, x) for x in [
'Kconfig.package.hacks',
'Kconfig.versions',
'Kconfig.local',
'Kconfig',
]
]
configtree.adjust_backported_configs(ignore=ignore, orig_symbols=orig_symbols)
git_debug_snapshot(args, "adjust backports config symbols we port")
configtree.modify_selects()
git_debug_snapshot(args, "convert select to depends on")
symbols = configtree.symbols()
# write local symbol list -- needed during packaging build
if not bpid.integrate:
f = open(os.path.join(bpid.target_dir, 'local-symbols'), 'w')
for sym in symbols:
f.write('%s=\n' % sym)
f.close()
git_debug_snapshot(args, "add symbols files")
# also write Kconfig.local, representing all local symbols
# with a BACKPORTED_ prefix
f = open(os.path.join(bpid.target_dir, 'Kconfig.local'), 'w')
for sym in symbols:
f.write('config BACKPORTED_%s\n' % sym)
f.write('\ttristate\n')
f.write('\tdefault %s\n' % sym)
f.close()
git_debug_snapshot(args, "add Kconfig.local")
# add defconfigs that we want
defconfigs_dir = os.path.join(source_dir, 'backport', 'defconfigs')
os.mkdir(os.path.join(bpid.target_dir, 'defconfigs'))
for dfbase in os.listdir(defconfigs_dir):
copy_defconfig = True
dfsrc = os.path.join(defconfigs_dir, dfbase)
for line in open(dfsrc, 'r'):
if not '=' in line:
continue
line_ok = False
for sym in symbols:
if sym + '=' in line:
line_ok = True
break
if not line_ok:
copy_defconfig = False
break
if copy_defconfig:
shutil.copy(dfsrc, os.path.join(bpid.target_dir, 'defconfigs', dfbase))
git_debug_snapshot(args, "add (useful) defconfig files")
logwrite('Rewrite Makefiles and Kconfig files ...')
# rewrite Makefile and source symbols
# symbols we know only we can provide under the backport project prefix
# for which we need an exception.
skip_orig_syms = [ bpid.project_prefix + x for x in [
'INTEGRATE',
]
]
parse_orig_syms = [x for x in orig_symbols if x not in skip_orig_syms ]
regexes = []
for some_symbols in [parse_orig_syms[i:i + 50] for i in range(0, len(parse_orig_syms), 50)]:
r = 'CONFIG_((' + '|'.join([s + '(_MODULE)?' for s in some_symbols]) + ')([^A-Za-z0-9_]|$))'
regexes.append(re.compile(r, re.MULTILINE))
for root, dirs, files in os.walk(bpid.target_dir):
# don't go into .git dir (possible debug thing)
if '.git' in dirs:
dirs.remove('.git')
for f in files:
data = open(os.path.join(root, f), 'r').read()
for r in regexes:
data = r.sub(r'' + bpid.full_prefix + '\\1', data)
data = re.sub(r'\$\(srctree\)', '$(backport_srctree)', data)
data = re.sub(r'-Idrivers', '-I$(backport_srctree)/drivers', data)
if bpid.integrate:
data = re.sub(r'CPTCFG_', bpid.full_prefix, data)
fo = open(os.path.join(root, f), 'w')
fo.write(data)
fo.close()
git_debug_snapshot(args, "rename config symbol / srctree usage")
# disable unbuildable Kconfig symbols and stuff Makefiles that doesn't exist
if bpid.integrate:
maketree = make.MakeTree(os.path.join(bpid.target_dir, 'Makefile'))
else:
maketree = make.MakeTree(os.path.join(bpid.target_dir, 'Makefile.kernel'))
disable_kconfig = []
disable_makefile = []
for sym in maketree.get_impossible_symbols():
disable_kconfig.append(sym[7:])
disable_makefile.append(sym[7:])
configtree.disable_symbols(disable_kconfig)
git_debug_snapshot(args, "disable impossible kconfig symbols")
# add kernel version dependencies to Kconfig, from the dependency list
# we read previously
for sym in tuple(deplist.keys()):
new = []
for dep in deplist[sym]:
if "kconfig:" in dep:
kconfig_expr = dep.replace('kconfig: ', '')
new.append(kconfig_expr)
elif (dep == "DISABLE"):
new.append('BACKPORT_DISABLED_KCONFIG_OPTION')
else:
new.append('!KERNEL_%s' % dep.replace('.', '_'))
if bpid.integrate:
deplist[sym] = ["BACKPORT_" + x for x in new]
else:
deplist[sym] = new
configtree.add_dependencies(deplist)
git_debug_snapshot(args, "add kernel version dependencies")
# disable things in makefiles that can't be selected and that the
# build shouldn't recurse into because they don't exist -- if we
# don't do that then a symbol from the kernel could cause the build
# to attempt to recurse and fail
#
# Note that we split the regex after 50 symbols, this is because of a
# limitation in the regex implementation (it only supports 100 nested
# groups -- 50 seemed safer and is still fast)
regexes = []
for some_symbols in [disable_makefile[i:i + 50] for i in range(0, len(disable_makefile), 50)]:
r = '^(([^#].*((' + bpid.full_prefix_resafe + '|CONFIG_)(' + '|'.join([s for s in some_symbols]) + ')))\W)'
regexes.append(re.compile(r, re.MULTILINE))
for f in maketree.get_makefiles():
data = open(f, 'r').read()
for r in regexes:
data = r.sub(r'#\1', data)
fo = open(f, 'w')
fo.write(data)
fo.close()
git_debug_snapshot(args, "disable unsatisfied Makefile parts")
if bpid.integrate:
f = open(os.path.join(bpid.project_dir, 'Kconfig'), 'a')
f.write('source "backports/Kconfig"\n')
f.close()
git_debug_snapshot(args, "hooked backport to top level Kconfig")
failure = apply_patches(args, "integration", source_dir, 'integration-patches/',
bpid.project_dir, logwrite)
if failure:
return failure
if (args.kup or args.kup_test):
req = reqs.Req()
req.kup()
if not req.reqs_match():
sys.exit(1)
upload_release(args, rel_prep, logwrite=logwrite)
logwrite('Done!')
return 0
def process(kerneldir, outdir, copy_list_file, git_revision=None,
clean=False, refresh=False, base_name="Linux", gitdebug=False,
verbose=False, extra_driver=[], kup=False,
kup_test=False,
logwrite=lambda x:None,
git_tracked_version=False):
class Args(object):
def __init__(self, kerneldir, outdir, copy_list_file,
git_revision, clean, refresh, base_name,
gitdebug, verbose, extra_driver, kup,
kup_test):
self.kerneldir = kerneldir
self.outdir = outdir
self.copy_list = copy_list_file
self.git_revision = git_revision
self.clean = clean
self.refresh = refresh
self.base_name = base_name
self.gitdebug = gitdebug
self.verbose = verbose
self.extra_driver = extra_driver
self.kup = kup
self.kup_test = kup_test
def git_paranoia(tree=None, logwrite=lambda x:None):
data = git.paranoia(tree)
if (data['r'] != 0):
logwrite('Cannot use %s' % tree)
logwrite('%s' % data['output'])
sys.exit(data['r'])
else:
logwrite('Validated tree: %s' % tree)
args = Args(kerneldir, outdir, copy_list_file,
git_revision, clean, refresh, base_name,
gitdebug, verbose, extra_driver, kup, kup_test)
rel_prep = None
# start processing ...
if (args.kup or args.kup_test):
git_paranoia(source_dir, logwrite)
git_paranoia(kerneldir, logwrite)
rel_describe = git.describe(rev=None, tree=source_dir, extra_args=['--dirty'])
release = os.path.basename(args.outdir)
version = release.replace("backports-", "")
rel_prep = get_rel_prep(version)
if (not rel_prep):
logwrite('Invalid backports release name: %s' % release)
logwrite('For rules on the release name see upload_release()')
sys.exit(1)
rel_type = "linux-stable"
if (not rel_prep['stable']):
rel_type = "linux-next"
if (rel_prep['expected_tag'] != rel_describe):
logwrite('Unexpected %s based backports release tag on' % rel_type)
logwrite('the backports tree tree: %s\n' % rel_describe)
logwrite('You asked to make a release with this ')
logwrite('directory name: %s' % release)
logwrite('The actual expected tag we should find on')
logwrite('the backports tree then is: %s\n' % rel_prep['expected_tag'])
logwrite('For rules on the release name see upload_release()')
sys.exit(1)
copy_list = read_copy_list(args.copy_list)
deplist = read_dependencies(os.path.join(source_dir, 'dependencies'))
# validate output directory
check_output_dir(args.outdir, args.clean)
# do the copy
backport_files = [(x, x) for x in [
'Kconfig', 'Makefile', 'Makefile.build', 'Makefile.kernel', '.gitignore',
'Makefile.real', 'compat/', 'backport-include/', 'kconf/', 'defconfigs/',
'scripts/', '.blacklist.map', 'udev/',
]]
if not args.git_revision:
logwrite('Copy original source files ...')
else:
logwrite('Get original source files from git ...')
copy_files(os.path.join(source_dir, 'backport'), backport_files, args.outdir)
git_debug_init(args)
if not args.git_revision:
copy_files(args.kerneldir, copy_list, args.outdir)
else:
copy_git_files(args.kerneldir, copy_list, args.git_revision, args.outdir)
# FIXME: should we add a git version of this (e.g. --git-extra-driver)?
for src, copy_list in args.extra_driver:
if (args.kup or args.kup_test):
git_paranoia(src)
copy_files(src, read_copy_list(open(copy_list, 'r')), args.outdir)
git_debug_snapshot(args, 'Add driver sources')
disable_list = add_automatic_backports(args)
if disable_list:
bpcfg = kconfig.ConfigTree(os.path.join(args.outdir, 'compat', 'Kconfig'))
bpcfg.disable_symbols(disable_list)
git_debug_snapshot(args, 'Add automatic backports')
logwrite('Apply patches ...')
patches = []
sempatches = []
for root, dirs, files in os.walk(os.path.join(source_dir, 'patches')):
for f in files:
if f.endswith('.patch'):
patches.append(os.path.join(root, f))
if f.endswith('.cocci'):
sempatches.append(os.path.join(root, f))
patches.sort()
prefix_len = len(os.path.join(source_dir, 'patches')) + 1
for pfile in patches:
print_name = pfile[prefix_len:]
# read the patch file
p = patch.fromfile(pfile)
# complain if it's not a patch
if not p:
raise Exception('No patch content found in %s' % print_name)
# leading / seems to be stripped?
if 'dev/null' in p.items[0].source:
raise Exception('Patches creating files are not supported (in %s)' % print_name)
# check if the first file the patch touches exists, if so
# assume the patch needs to be applied -- otherwise continue
patched_file = '/'.join(p.items[0].source.split('/')[1:])
fullfn = os.path.join(args.outdir, patched_file)
if not os.path.exists(fullfn):
if args.verbose:
logwrite("Not applying %s, not needed" % print_name)
continue
if args.verbose:
logwrite("Applying patch %s" % print_name)
if args.refresh:
# but for refresh, of course look at all files the patch touches
for patchitem in p.items:
patched_file = '/'.join(patchitem.source.split('/')[1:])
fullfn = os.path.join(args.outdir, patched_file)
shutil.copyfile(fullfn, fullfn + '.orig_file')
process = subprocess.Popen(['patch', '-p1'], stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, stdin=subprocess.PIPE,
close_fds=True, universal_newlines=True,
cwd=args.outdir)
output = process.communicate(input=open(pfile, 'r').read())[0]
output = output.split('\n')
if output[-1] == '':
output = output[:-1]
if args.verbose:
for line in output:
logwrite('> %s' % line)
if process.returncode != 0:
if not args.verbose:
logwrite("Failed to apply changes from %s" % print_name)
for line in output:
logwrite('> %s' % line)
# return 2
if args.refresh:
pfilef = open(pfile + '.tmp', 'a')
pfilef.write(p.top_header)
pfilef.flush()
for patchitem in p.items:
patched_file = '/'.join(patchitem.source.split('/')[1:])
fullfn = os.path.join(args.outdir, patched_file)
process = subprocess.Popen(['diff', '-p', '-u', patched_file + '.orig_file', patched_file,
'--label', 'a/' + patched_file,
'--label', 'b/' + patched_file],
stdout=pfilef, close_fds=True,
universal_newlines=True, cwd=args.outdir)
process.wait()
os.unlink(fullfn + '.orig_file')
if not process.returncode in (0, 1):
logwrite("Failed to diff to refresh %s" % print_name)
pfilef.close()
os.unlink(pfile + '.tmp')
return 3
pfilef.close()
os.rename(pfile + '.tmp', pfile)
# remove orig/rej files that patch sometimes creates
for root, dirs, files in os.walk(args.outdir):
for f in files:
if f[-5:] == '.orig' or f[-4:] == '.rej':
os.unlink(os.path.join(root, f))
git_debug_snapshot(args, "apply backport patch %s" % print_name)
sempatches.sort()
with tempdir() as t:
if not args.gitdebug and sempatches:
# combine all spatches
fn = os.path.join(t, 'combined.cocci')
f = open(fn, 'w')
for cocci_file in sempatches:
for l in open(cocci_file, 'r'):
f.write(l)
f.write('\n')
f.close()
sempatches = [fn]
prefix_len = 0
else:
prefix_len = len(os.path.join(source_dir, 'patches')) + 1
for cocci_file in sempatches:
print_name = cocci_file[prefix_len:]
if args.verbose:
logwrite("Applying patch %s" % print_name)
process = subprocess.Popen(['spatch', '--sp-file', cocci_file, '--in-place',
'--backup-suffix', '.cocci_backup', '--dir', '.'],
stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
close_fds=True, universal_newlines=True,
cwd=args.outdir)
output = process.communicate()[0]
output = output.split('\n')
if output[-1] == '':
output = output[:-1]
if args.verbose:
for line in output:
logwrite('> %s' % line)
if process.returncode != 0:
if not args.verbose:
logwrite("Failed to apply changes from %s" % print_name)
for line in output:
logwrite('> %s' % line)
# return 2
# remove cocci_backup files
for root, dirs, files in os.walk(args.outdir):
for f in files:
if f.endswith('.cocci_backup'):
os.unlink(os.path.join(root, f))
git_debug_snapshot(args, "apply backport patch %s" % print_name)
# some post-processing is required
configtree = kconfig.ConfigTree(os.path.join(args.outdir, 'Kconfig'))
logwrite('Modify Kconfig tree ...')
configtree.prune_sources(ignore=['Kconfig.kernel', 'Kconfig.versions'])
git_debug_snapshot(args, "prune Kconfig tree")
configtree.force_tristate_modular()
git_debug_snapshot(args, "force tristate options modular")
configtree.modify_selects()
git_debug_snapshot(args, "convert select to depends on")
# write the versioning file
if git_tracked_version:
backports_version = "(see git)"
kernel_version = "(see git)"
else:
backports_version = git.describe(tree=source_dir, extra_args=['--long'])
kernel_version = git.describe(rev=args.git_revision or 'HEAD',
tree=args.kerneldir,
extra_args=['--long'])
f = open(os.path.join(args.outdir, 'versions'), 'w')
f.write('BACKPORTS_VERSION="%s"\n' % backports_version)
f.write('BACKPORTED_KERNEL_VERSION="%s"\n' % kernel_version)
f.write('BACKPORTED_KERNEL_NAME="%s"\n' % args.base_name)
if git_tracked_version:
f.write('BACKPORTS_GIT_TRACKED="backport tracker ID: $(shell git rev-parse HEAD 2>/dev/null || echo \'not built in git tree\')"\n')
f.close()
symbols = configtree.symbols()
# write local symbol list -- needed during build
f = open(os.path.join(args.outdir, '.local-symbols'), 'w')
for sym in symbols:
f.write('%s=\n' % sym)
f.close()
git_debug_snapshot(args, "add versions/symbols files")
logwrite('Rewrite Makefiles and Kconfig files ...')
# rewrite Makefile and source symbols
regexes = []
for some_symbols in [symbols[i:i + 50] for i in range(0, len(symbols), 50)]:
r = 'CONFIG_((' + '|'.join([s + '(_MODULE)?' for s in some_symbols]) + ')([^A-Za-z0-9_]|$))'
regexes.append(re.compile(r, re.MULTILINE))
for root, dirs, files in os.walk(args.outdir):
# don't go into .git dir (possible debug thing)
if '.git' in dirs:
dirs.remove('.git')
for f in files:
data = open(os.path.join(root, f), 'r').read()
for r in regexes:
data = r.sub(r'CPTCFG_\1', data)
data = re.sub(r'\$\(srctree\)', '$(backport_srctree)', data)
data = re.sub(r'-Idrivers', '-I$(backport_srctree)/drivers', data)
fo = open(os.path.join(root, f), 'w')
fo.write(data)
fo.close()
git_debug_snapshot(args, "rename config symbol / srctree usage")
# disable unbuildable Kconfig symbols and stuff Makefiles that doesn't exist
maketree = make.MakeTree(os.path.join(args.outdir, 'Makefile.kernel'))
disable_kconfig = []
disable_makefile = []
for sym in maketree.get_impossible_symbols():
disable_kconfig.append(sym[7:])
disable_makefile.append(sym[7:])
configtree.disable_symbols(disable_kconfig)
git_debug_snapshot(args, "disable impossible kconfig symbols")
# add kernel version dependencies to Kconfig, from the dependency list
# we read previously
for sym in tuple(deplist.keys()):
new = []
for dep in deplist[sym]:
if "kconfig:" in dep:
kconfig_expr = dep.replace('kconfig: ', '')
new.append(kconfig_expr)
elif (dep == "DISABLE"):
new.append('BACKPORT_DISABLED_KCONFIG_OPTION')
else:
new.append('!BACKPORT_KERNEL_%s' % dep.replace('.', '_'))
deplist[sym] = new
configtree.add_dependencies(deplist)
git_debug_snapshot(args, "add kernel version dependencies")
# disable things in makefiles that can't be selected and that the
# build shouldn't recurse into because they don't exist -- if we
# don't do that then a symbol from the kernel could cause the build
# to attempt to recurse and fail
#
# Note that we split the regex after 50 symbols, this is because of a
# limitation in the regex implementation (it only supports 100 nested
# groups -- 50 seemed safer and is still fast)
regexes = []
for some_symbols in [disable_makefile[i:i + 50] for i in range(0, len(disable_makefile), 50)]:
r = '^([^#].*((CPTCFG|CONFIG)_(' + '|'.join([s for s in some_symbols]) + ')))'
regexes.append(re.compile(r, re.MULTILINE))
for f in maketree.get_makefiles():
data = open(f, 'r').read()
for r in regexes:
data = r.sub(r'#\1', data)
fo = open(f, 'w')
fo.write(data)
fo.close()
git_debug_snapshot(args, "disable unsatisfied Makefile parts")
if (args.kup or args.kup_test):
upload_release(args, rel_prep, logwrite=logwrite)
logwrite('Done!')
return 0
def process(kerneldir,
outdir,
copy_list_file,
git_revision=None,
clean=False,
refresh=False,
base_name="Linux",
gitdebug=False,
verbose=False,
extra_driver=[],
kup=False,
kup_test=False,
logwrite=lambda x: None,
git_tracked_version=False):
class Args(object):
def __init__(self, kerneldir, outdir, copy_list_file, git_revision,
clean, refresh, base_name, gitdebug, verbose,
extra_driver, kup, kup_test):
self.kerneldir = kerneldir
self.outdir = outdir
self.copy_list = copy_list_file
self.git_revision = git_revision
self.clean = clean
self.refresh = refresh
self.base_name = base_name
self.gitdebug = gitdebug
self.verbose = verbose
self.extra_driver = extra_driver
self.kup = kup
self.kup_test = kup_test
def git_paranoia(tree=None, logwrite=lambda x: None):
data = git.paranoia(tree)
if (data['r'] != 0):
logwrite('Cannot use %s' % tree)
logwrite('%s' % data['output'])
sys.exit(data['r'])
else:
logwrite('Validated tree: %s' % tree)
args = Args(kerneldir, outdir, copy_list_file, git_revision, clean,
refresh, base_name, gitdebug, verbose, extra_driver, kup,
kup_test)
rel_prep = None
# start processing ...
if (args.kup or args.kup_test):
git_paranoia(source_dir, logwrite)
git_paranoia(kerneldir, logwrite)
rel_describe = git.describe(rev=None,
tree=source_dir,
extra_args=['--dirty'])
release = os.path.basename(args.outdir)
version = release.replace("backports-", "")
rel_prep = get_rel_prep(version)
if (not rel_prep):
logwrite('Invalid backports release name: %s' % release)
logwrite('For rules on the release name see upload_release()')
sys.exit(1)
rel_type = "linux-stable"
if (not rel_prep['stable']):
rel_type = "linux-next"
if (rel_prep['expected_tag'] != rel_describe):
logwrite('Unexpected %s based backports release tag on' % rel_type)
logwrite('the backports tree tree: %s\n' % rel_describe)
logwrite('You asked to make a release with this ')
logwrite('directory name: %s' % release)
logwrite('The actual expected tag we should find on')
logwrite('the backports tree then is: %s\n' %
rel_prep['expected_tag'])
logwrite('For rules on the release name see upload_release()')
sys.exit(1)
copy_list = read_copy_list(args.copy_list)
deplist = read_dependencies(os.path.join(source_dir, 'dependencies'))
# validate output directory
check_output_dir(args.outdir, args.clean)
# do the copy
backport_files = [(x, x) for x in [
'Kconfig',
'Makefile',
'Makefile.build',
'Makefile.kernel',
'.gitignore',
'Makefile.real',
'compat/',
'backport-include/',
'kconf/',
'defconfigs/',
'scripts/',
'.blacklist.map',
'udev/',
]]
if not args.git_revision:
logwrite('Copy original source files ...')
else:
logwrite('Get original source files from git ...')
copy_files(os.path.join(source_dir, 'backport'), backport_files,
args.outdir)
git_debug_init(args)
if not args.git_revision:
copy_files(args.kerneldir, copy_list, args.outdir)
else:
copy_git_files(args.kerneldir, copy_list, args.git_revision,
args.outdir)
# FIXME: should we add a git version of this (e.g. --git-extra-driver)?
for src, copy_list in args.extra_driver:
if (args.kup or args.kup_test):
git_paranoia(src)
copy_files(src, read_copy_list(open(copy_list, 'r')), args.outdir)
git_debug_snapshot(args, 'Add driver sources')
disable_list = add_automatic_backports(args)
if disable_list:
bpcfg = kconfig.ConfigTree(
os.path.join(args.outdir, 'compat', 'Kconfig'))
bpcfg.disable_symbols(disable_list)
git_debug_snapshot(args, 'Add automatic backports')
logwrite('Apply patches ...')
patches = []
sempatches = []
for root, dirs, files in os.walk(os.path.join(source_dir, 'patches')):
for f in files:
if f.endswith('.patch'):
patches.append(os.path.join(root, f))
if f.endswith('.cocci'):
sempatches.append(os.path.join(root, f))
patches.sort()
prefix_len = len(os.path.join(source_dir, 'patches')) + 1
for pfile in patches:
print_name = pfile[prefix_len:]
# read the patch file
p = patch.fromfile(pfile)
# complain if it's not a patch
if not p:
raise Exception('No patch content found in %s' % print_name)
# leading / seems to be stripped?
if 'dev/null' in p.items[0].source:
raise Exception(
'Patches creating files are not supported (in %s)' %
print_name)
# check if the first file the patch touches exists, if so
# assume the patch needs to be applied -- otherwise continue
patched_file = '/'.join(p.items[0].source.split('/')[1:])
fullfn = os.path.join(args.outdir, patched_file)
if not os.path.exists(fullfn):
if args.verbose:
logwrite("Not applying %s, not needed" % print_name)
continue
if args.verbose:
logwrite("Applying patch %s" % print_name)
if args.refresh:
# but for refresh, of course look at all files the patch touches
for patchitem in p.items:
patched_file = '/'.join(patchitem.source.split('/')[1:])
fullfn = os.path.join(args.outdir, patched_file)
shutil.copyfile(fullfn, fullfn + '.orig_file')
process = subprocess.Popen(['patch', '-p1'],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
stdin=subprocess.PIPE,
close_fds=True,
universal_newlines=True,
cwd=args.outdir)
output = process.communicate(input=open(pfile, 'r').read())[0]
output = output.split('\n')
if output[-1] == '':
output = output[:-1]
if args.verbose:
for line in output:
logwrite('> %s' % line)
if process.returncode != 0:
if not args.verbose:
logwrite("Failed to apply changes from %s" % print_name)
for line in output:
logwrite('> %s' % line)
return 2
if args.refresh:
pfilef = open(pfile + '.tmp', 'a')
pfilef.write(p.top_header)
pfilef.flush()
for patchitem in p.items:
patched_file = '/'.join(patchitem.source.split('/')[1:])
fullfn = os.path.join(args.outdir, patched_file)
process = subprocess.Popen([
'diff', '-p', '-u', patched_file + '.orig_file',
patched_file, '--label', 'a/' + patched_file, '--label',
'b/' + patched_file
],
stdout=pfilef,
close_fds=True,
universal_newlines=True,
cwd=args.outdir)
process.wait()
os.unlink(fullfn + '.orig_file')
if not process.returncode in (0, 1):
logwrite("Failed to diff to refresh %s" % print_name)
pfilef.close()
os.unlink(pfile + '.tmp')
return 3
pfilef.close()
os.rename(pfile + '.tmp', pfile)
# remove orig/rej files that patch sometimes creates
for root, dirs, files in os.walk(args.outdir):
for f in files:
if f[-5:] == '.orig' or f[-4:] == '.rej':
os.unlink(os.path.join(root, f))
git_debug_snapshot(args, "apply backport patch %s" % print_name)
sempatches.sort()
with tempdir() as t:
if not args.gitdebug:
# combine all spatches
fn = os.path.join(t, 'combined.cocci')
f = open(fn, 'w')
for cocci_file in sempatches:
for l in open(cocci_file, 'r'):
f.write(l)
f.write('\n')
f.close()
sempatches = [fn]
prefix_len = 0
else:
prefix_len = len(os.path.join(source_dir, 'patches')) + 1
for cocci_file in sempatches:
print_name = cocci_file[prefix_len:]
if args.verbose:
logwrite("Applying patch %s" % print_name)
process = subprocess.Popen([
'spatch', '--sp-file', cocci_file, '--in-place',
'--backup-suffix', '.cocci_backup', '--dir', '.'
],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
close_fds=True,
universal_newlines=True,
cwd=args.outdir)
output = process.communicate()[0]
output = output.split('\n')
if output[-1] == '':
output = output[:-1]
if args.verbose:
for line in output:
logwrite('> %s' % line)
if process.returncode != 0:
if not args.verbose:
logwrite("Failed to apply changes from %s" % print_name)
for line in output:
logwrite('> %s' % line)
return 2
# remove cocci_backup files
for root, dirs, files in os.walk(args.outdir):
for f in files:
if f.endswith('.cocci_backup'):
os.unlink(os.path.join(root, f))
git_debug_snapshot(args, "apply backport patch %s" % print_name)
# some post-processing is required
configtree = kconfig.ConfigTree(os.path.join(args.outdir, 'Kconfig'))
logwrite('Modify Kconfig tree ...')
configtree.prune_sources(ignore=['Kconfig.kernel', 'Kconfig.versions'])
git_debug_snapshot(args, "prune Kconfig tree")
configtree.force_tristate_modular()
git_debug_snapshot(args, "force tristate options modular")
configtree.modify_selects()
git_debug_snapshot(args, "convert select to depends on")
# write the versioning file
if git_tracked_version:
backports_version = "(see git)"
kernel_version = "(see git)"
else:
backports_version = git.describe(tree=source_dir,
extra_args=['--long'])
kernel_version = git.describe(rev=args.git_revision or 'HEAD',
tree=args.kerneldir,
extra_args=['--long'])
f = open(os.path.join(args.outdir, 'versions'), 'w')
f.write('BACKPORTS_VERSION="%s"\n' % backports_version)
f.write('BACKPORTED_KERNEL_VERSION="%s"\n' % kernel_version)
f.write('BACKPORTED_KERNEL_NAME="%s"\n' % args.base_name)
if git_tracked_version:
f.write(
'BACKPORTS_GIT_TRACKED="backport tracker ID: $(shell git rev-parse HEAD 2>/dev/null || echo \'not built in git tree\')"\n'
)
f.close()
symbols = configtree.symbols()
# write local symbol list -- needed during build
f = open(os.path.join(args.outdir, '.local-symbols'), 'w')
for sym in symbols:
f.write('%s=\n' % sym)
f.close()
git_debug_snapshot(args, "add versions/symbols files")
logwrite('Rewrite Makefiles and Kconfig files ...')
# rewrite Makefile and source symbols
regexes = []
for some_symbols in [
symbols[i:i + 50] for i in range(0, len(symbols), 50)
]:
r = 'CONFIG_((' + '|'.join([s + '(_MODULE)?' for s in some_symbols
]) + ')([^A-Za-z0-9_]|$))'
regexes.append(re.compile(r, re.MULTILINE))
for root, dirs, files in os.walk(args.outdir):
# don't go into .git dir (possible debug thing)
if '.git' in dirs:
dirs.remove('.git')
for f in files:
data = open(os.path.join(root, f), 'r').read()
for r in regexes:
data = r.sub(r'CPTCFG_\1', data)
data = re.sub(r'\$\(srctree\)', '$(backport_srctree)', data)
data = re.sub(r'-Idrivers', '-I$(backport_srctree)/drivers', data)
fo = open(os.path.join(root, f), 'w')
fo.write(data)
fo.close()
git_debug_snapshot(args, "rename config symbol / srctree usage")
# disable unbuildable Kconfig symbols and stuff Makefiles that doesn't exist
maketree = make.MakeTree(os.path.join(args.outdir, 'Makefile.kernel'))
disable_kconfig = []
disable_makefile = []
for sym in maketree.get_impossible_symbols():
disable_kconfig.append(sym[7:])
disable_makefile.append(sym[7:])
configtree.disable_symbols(disable_kconfig)
git_debug_snapshot(args, "disable impossible kconfig symbols")
# add kernel version dependencies to Kconfig, from the dependency list
# we read previously
for sym in tuple(deplist.keys()):
new = []
for dep in deplist[sym]:
if "kconfig:" in dep:
kconfig_expr = dep.replace('kconfig: ', '')
new.append(kconfig_expr)
elif (dep == "DISABLE"):
new.append('BACKPORT_DISABLED_KCONFIG_OPTION')
else:
new.append('!BACKPORT_KERNEL_%s' % dep.replace('.', '_'))
deplist[sym] = new
configtree.add_dependencies(deplist)
git_debug_snapshot(args, "add kernel version dependencies")
# disable things in makefiles that can't be selected and that the
# build shouldn't recurse into because they don't exist -- if we
# don't do that then a symbol from the kernel could cause the build
# to attempt to recurse and fail
#
# Note that we split the regex after 50 symbols, this is because of a
# limitation in the regex implementation (it only supports 100 nested
# groups -- 50 seemed safer and is still fast)
regexes = []
for some_symbols in [
disable_makefile[i:i + 50]
for i in range(0, len(disable_makefile), 50)
]:
r = '^([^#].*((CPTCFG|CONFIG)_(' + '|'.join([s for s in some_symbols
]) + ')))'
regexes.append(re.compile(r, re.MULTILINE))
for f in maketree.get_makefiles():
data = open(f, 'r').read()
for r in regexes:
data = r.sub(r'#\1', data)
fo = open(f, 'w')
fo.write(data)
fo.close()
git_debug_snapshot(args, "disable unsatisfied Makefile parts")
if (args.kup or args.kup_test):
upload_release(args, rel_prep, logwrite=logwrite)
logwrite('Done!')
return 0
def process(kerneldir, copy_list_file, git_revision=None,
bpid=None,
clean=False, refresh=False, base_name="Linux", gitdebug=False,
verbose=False, extra_driver=[], kup=False,
kup_test=False,
test_cocci=None,
profile_cocci=None,
logwrite=lambda x:None,
git_tracked_version=False):
class Args(object):
def __init__(self, kerneldir, copy_list_file,
git_revision, bpid, clean, refresh, base_name,
gitdebug, verbose, extra_driver, kup,
kup_test,
test_cocci,
profile_cocci):
self.kerneldir = kerneldir
self.copy_list = copy_list_file
self.git_revision = git_revision
self.bpid = bpid
self.clean = clean
self.refresh = refresh
self.base_name = base_name
self.gitdebug = gitdebug
self.verbose = verbose
self.extra_driver = extra_driver
self.kup = kup
self.kup_test = kup_test
self.test_cocci = test_cocci
self.profile_cocci = profile_cocci
if self.test_cocci or self.profile_cocci:
self.gitdebug = True
def git_paranoia(tree=None, logwrite=lambda x:None):
data = git.paranoia(tree)
if (data['r'] != 0):
logwrite('Cannot use %s' % tree)
logwrite('%s' % data['output'])
sys.exit(data['r'])
else:
logwrite('Validated tree: %s' % tree)
args = Args(kerneldir, copy_list_file,
git_revision, bpid, clean, refresh, base_name,
gitdebug, verbose, extra_driver, kup, kup_test,
test_cocci, profile_cocci)
rel_prep = None
if bpid.integrate:
if args.kup_test or args.test_cocci or args.profile_cocci or args.refresh:
logwrite('Cannot use integration with:\n\tkup_test\n\ttest_cocci\n\tprofile_cocci\n\trefresh\n');
sys.exit(1)
# start processing ...
if (args.kup or args.kup_test):
git_paranoia(source_dir, logwrite)
git_paranoia(kerneldir, logwrite)
rel_describe = git.describe(rev=None, tree=source_dir, extra_args=['--dirty'])
release = os.path.basename(bpid.target_dir)
version = release.replace("backports-", "")
rel_prep = get_rel_prep(version)
if (not rel_prep):
logwrite('Invalid backports release name: %s' % release)
logwrite('For rules on the release name see upload_release()')
sys.exit(1)
rel_type = "linux-stable"
if (not rel_prep['stable']):
rel_type = "linux-next"
if (rel_prep['expected_tag'] != rel_describe):
logwrite('Unexpected %s based backports release tag on' % rel_type)
logwrite('the backports tree tree: %s\n' % rel_describe)
logwrite('You asked to make a release with this ')
logwrite('directory name: %s' % release)
logwrite('The actual expected tag we should find on')
logwrite('the backports tree then is: %s\n' % rel_prep['expected_tag'])
logwrite('For rules on the release name see upload_release()')
sys.exit(1)
copy_list = read_copy_list(args.copy_list)
deplist = read_dependencies(os.path.join(source_dir, 'dependencies'))
# validate output directory
check_output_dir(bpid.target_dir, args.clean)
# do the copy
backport_integrate_files = [
('Makefile.kernel', 'Makefile'),
('Kconfig.integrate', 'Kconfig'),
]
backport_package_files = [(x, x) for x in [
'Makefile',
'kconf/',
'Makefile.real',
'Makefile.kernel',
'Kconfig.package.hacks',
'scripts/',
'.blacklist.map',
'.gitignore',
'Makefile.build'] ]
backport_package_files += [
('Kconfig.package', 'Kconfig'),
]
backport_files = [(x, x) for x in [
'Kconfig.sources',
'compat/',
'backport-include/',
]]
if not bpid.integrate:
backport_files += backport_package_files
else:
backport_files += backport_integrate_files
if not args.git_revision:
logwrite('Copy original source files ...')
else:
logwrite('Get original source files from git ...')
copy_files(os.path.join(source_dir, 'backport'), backport_files, bpid.target_dir)
git_debug_init(args)
if not args.git_revision:
copy_files(args.kerneldir, copy_list, bpid.target_dir)
else:
copy_git_files(args.kerneldir, copy_list, args.git_revision, bpid.target_dir)
# FIXME: should we add a git version of this (e.g. --git-extra-driver)?
for src, copy_list in args.extra_driver:
if (args.kup or args.kup_test):
git_paranoia(src)
copy_files(src, read_copy_list(open(copy_list, 'r')), bpid.target_dir)
git_debug_snapshot(args, 'Add driver sources')
disable_list = add_automatic_backports(args)
if git_tracked_version:
backports_version = "(see git)"
kernel_version = "(see git)"
else:
backports_version = git.describe(tree=source_dir, extra_args=['--long'])
kernel_version = git.describe(rev=args.git_revision or 'HEAD',
tree=args.kerneldir,
extra_args=['--long'])
if not bpid.integrate:
f = open(os.path.join(bpid.target_dir, 'versions'), 'w')
f.write('BACKPORTS_VERSION="%s"\n' % backports_version)
f.write('BACKPORTED_KERNEL_VERSION="%s"\n' % kernel_version)
f.write('BACKPORTED_KERNEL_NAME="%s"\n' % args.base_name)
if git_tracked_version:
f.write('BACKPORTS_GIT_TRACKED="backport tracker ID: $(shell git rev-parse HEAD 2>/dev/null || echo \'not built in git tree\')"\n')
f.close()
git_debug_snapshot(args, "add versions files")
else:
kconf_regexes = [
(re.compile(r'.*(?P<key>%%BACKPORT_DIR%%)'), '%%BACKPORT_DIR%%', 'backports/'),
(re.compile(r'.*(?P<key>%%BACKPORTS_VERSION%%).*'), '%%BACKPORTS_VERSION%%', backports_version),
(re.compile(r'.*(?P<key>%%BACKPORTED_KERNEL_VERSION%%).*'), '%%BACKPORTED_KERNEL_VERSION%%', kernel_version),
(re.compile(r'.*(?P<key>%%BACKPORTED_KERNEL_NAME%%).*'), '%%BACKPORTED_KERNEL_NAME%%', args.base_name),
]
out = ''
for l in open(os.path.join(bpid.target_dir, 'Kconfig'), 'r'):
for r in kconf_regexes:
m = r[0].match(l)
if m:
l = re.sub(r'(' + r[1] + ')', r'' + r[2] + '', l)
out += l
outf = open(os.path.join(bpid.target_dir, 'Kconfig'), 'w')
outf.write(out)
outf.close()
git_debug_snapshot(args, "modify top level backports/Kconfig with backports identity")
if disable_list:
# No need to verify_sources() as compat's Kconfig has no 'source' call
bpcfg = kconfig.ConfigTree(os.path.join(bpid.target_dir, 'compat', 'Kconfig'), bpid)
bpcfg.disable_symbols(disable_list)
git_debug_snapshot(args, 'Add automatic backports')
apply_patches(args, "backport", source_dir, 'patches', bpid.target_dir, logwrite)
# Kernel integration requires Kconfig.versions already generated for you,
# we cannot do this for a package as we have no idea what kernel folks
# will be using.
if bpid.integrate:
kver = gen_version.kernelversion(bpid.project_dir)
rel_specs = gen_version.get_rel_spec_stable(kver)
if not rel_specs:
logwrite('Cannot parse source kernel version, update parser')
sys.exit(1)
data = gen_version.genkconfig_versions(rel_specs)
fo = open(os.path.join(bpid.target_dir, 'Kconfig.versions'), 'w')
fo.write(data)
fo.close()
git_debug_snapshot(args, "generate kernel version requirement Kconfig file")
# some post-processing is required
configtree = kconfig.ConfigTree(os.path.join(bpid.target_dir, 'Kconfig'), bpid)
ignore=['Kconfig.kernel', 'Kconfig.versions']
configtree.verify_sources(ignore=ignore)
git_debug_snapshot(args, "verify sources on top level backports Kconfig")
orig_symbols = configtree.symbols()
logwrite('Modify Kconfig tree ...')
configtree.prune_sources(ignore=ignore)
git_debug_snapshot(args, "prune Kconfig tree")
if not bpid.integrate:
configtree.force_tristate_modular()
git_debug_snapshot(args, "force tristate options modular")
ignore = [os.path.join(bpid.target_dir, x) for x in [
'Kconfig.package.hacks',
'Kconfig.versions',
'Kconfig',
]
]
configtree.adjust_backported_configs(ignore=ignore, orig_symbols=orig_symbols)
git_debug_snapshot(args, "adjust backports config symbols we port")
configtree.modify_selects()
git_debug_snapshot(args, "convert select to depends on")
symbols = configtree.symbols()
# write local symbol list -- needed during packaging build
if not bpid.integrate:
f = open(os.path.join(bpid.target_dir, '.local-symbols'), 'w')
for sym in symbols:
f.write('%s=\n' % sym)
f.close()
git_debug_snapshot(args, "add symbols files")
# add defconfigs that we want
defconfigs_dir = os.path.join(source_dir, 'backport', 'defconfigs')
os.mkdir(os.path.join(bpid.target_dir, 'defconfigs'))
for dfbase in os.listdir(defconfigs_dir):
copy_defconfig = True
dfsrc = os.path.join(defconfigs_dir, dfbase)
for line in open(dfsrc, 'r'):
if not '=' in line:
continue
line_ok = False
for sym in symbols:
if sym + '=' in line:
line_ok = True
break
if not line_ok:
copy_defconfig = False
break
if copy_defconfig:
shutil.copy(dfsrc, os.path.join(bpid.target_dir, 'defconfigs', dfbase))
git_debug_snapshot(args, "add (useful) defconfig files")
logwrite('Rewrite Makefiles and Kconfig files ...')
# rewrite Makefile and source symbols
# symbols we know only we can provide under the backport project prefix
# for which we need an exception.
skip_orig_syms = [ bpid.project_prefix + x for x in [
'INTEGRATE',
]
]
parse_orig_syms = [x for x in orig_symbols if x not in skip_orig_syms ]
regexes = []
for some_symbols in [parse_orig_syms[i:i + 50] for i in range(0, len(parse_orig_syms), 50)]:
r = 'CONFIG_((' + '|'.join([s + '(_MODULE)?' for s in some_symbols]) + ')([^A-Za-z0-9_]|$))'
regexes.append(re.compile(r, re.MULTILINE))
for root, dirs, files in os.walk(bpid.target_dir):
# don't go into .git dir (possible debug thing)
if '.git' in dirs:
dirs.remove('.git')
for f in files:
data = open(os.path.join(root, f), 'r').read()
for r in regexes:
data = r.sub(r'' + bpid.full_prefix + '\\1', data)
data = re.sub(r'\$\(srctree\)', '$(backport_srctree)', data)
data = re.sub(r'-Idrivers', '-I$(backport_srctree)/drivers', data)
if bpid.integrate:
data = re.sub(r'CPTCFG_', bpid.full_prefix, data)
fo = open(os.path.join(root, f), 'w')
fo.write(data)
fo.close()
git_debug_snapshot(args, "rename config symbol / srctree usage")
# disable unbuildable Kconfig symbols and stuff Makefiles that doesn't exist
if bpid.integrate:
maketree = make.MakeTree(os.path.join(bpid.target_dir, 'Makefile'))
else:
maketree = make.MakeTree(os.path.join(bpid.target_dir, 'Makefile.kernel'))
disable_kconfig = []
disable_makefile = []
for sym in maketree.get_impossible_symbols():
disable_kconfig.append(sym[7:])
disable_makefile.append(sym[7:])
configtree.disable_symbols(disable_kconfig)
git_debug_snapshot(args, "disable impossible kconfig symbols")
# add kernel version dependencies to Kconfig, from the dependency list
# we read previously
for sym in tuple(deplist.keys()):
new = []
for dep in deplist[sym]:
if "kconfig:" in dep:
kconfig_expr = dep.replace('kconfig: ', '')
new.append(kconfig_expr)
elif (dep == "DISABLE"):
new.append('BACKPORT_DISABLED_KCONFIG_OPTION')
else:
new.append('!KERNEL_%s' % dep.replace('.', '_'))
if bpid.integrate:
deplist[sym] = ["BACKPORT_" + x for x in new]
else:
deplist[sym] = new
configtree.add_dependencies(deplist)
git_debug_snapshot(args, "add kernel version dependencies")
# disable things in makefiles that can't be selected and that the
# build shouldn't recurse into because they don't exist -- if we
# don't do that then a symbol from the kernel could cause the build
# to attempt to recurse and fail
#
# Note that we split the regex after 50 symbols, this is because of a
# limitation in the regex implementation (it only supports 100 nested
# groups -- 50 seemed safer and is still fast)
regexes = []
for some_symbols in [disable_makefile[i:i + 50] for i in range(0, len(disable_makefile), 50)]:
r = '^([^#].*((' + bpid.full_prefix_resafe + '|CONFIG_)(' + '|'.join([s for s in some_symbols]) + ')))'
regexes.append(re.compile(r, re.MULTILINE))
for f in maketree.get_makefiles():
data = open(f, 'r').read()
for r in regexes:
data = r.sub(r'#\1', data)
fo = open(f, 'w')
fo.write(data)
fo.close()
git_debug_snapshot(args, "disable unsatisfied Makefile parts")
if bpid.integrate:
f = open(os.path.join(bpid.project_dir, 'Kconfig'), 'a')
f.write('source "backports/Kconfig"\n')
f.close()
git_debug_snapshot(args, "hooked backport to top level Kconfig")
apply_patches(args, "integration", source_dir, 'integration-patches/',
bpid.project_dir, logwrite)
if (args.kup or args.kup_test):
req = reqs.Req()
req.kup()
if not req.reqs_match():
sys.exit(1)
upload_release(args, rel_prep, logwrite=logwrite)
logwrite('Done!')
return 0