def process(kerneldir, copy_list_file, git_revision=None,
bpid=None,
clean=False, refresh=False, base_name="Linux", gitdebug=False,
verbose=False, extra_driver=[], kup=False,
kup_test=False,
test_cocci=None,
profile_cocci=None,
logwrite=lambda x:None,
git_tracked_version=False):
class Args(object):
def __init__(self, kerneldir, copy_list_file,
git_revision, bpid, clean, refresh, base_name,
gitdebug, verbose, extra_driver, kup,
kup_test,
test_cocci,
profile_cocci):
self.kerneldir = kerneldir
self.copy_list = copy_list_file
self.git_revision = git_revision
self.bpid = bpid
self.clean = clean
self.refresh = refresh
self.base_name = base_name
self.gitdebug = gitdebug
self.verbose = verbose
self.extra_driver = extra_driver
self.kup = kup
self.kup_test = kup_test
self.test_cocci = test_cocci
self.profile_cocci = profile_cocci
if self.test_cocci or self.profile_cocci:
self.gitdebug = True
def git_paranoia(tree=None, logwrite=lambda x:None):
data = git.paranoia(tree)
if (data['r'] != 0):
logwrite('Cannot use %s' % tree)
logwrite('%s' % data['output'])
sys.exit(data['r'])
else:
logwrite('Validated tree: %s' % tree)
args = Args(kerneldir, copy_list_file,
git_revision, bpid, clean, refresh, base_name,
gitdebug, verbose, extra_driver, kup, kup_test,
test_cocci, profile_cocci)
rel_prep = None
if bpid.integrate:
if args.kup_test or args.test_cocci or args.profile_cocci or args.refresh:
logwrite('Cannot use integration with:\n\tkup_test\n\ttest_cocci\n\tprofile_cocci\n\trefresh\n');
sys.exit(1)
# start processing ...
if (args.kup or args.kup_test):
git_paranoia(source_dir, logwrite)
git_paranoia(kerneldir, logwrite)
rel_describe = git.describe(rev=None, tree=source_dir, extra_args=['--dirty'])
release = os.path.basename(bpid.target_dir)
version = release.replace("backports-", "")
rel_prep = get_rel_prep(version)
if (not rel_prep):
logwrite('Invalid backports release name: %s' % release)
logwrite('For rules on the release name see upload_release()')
sys.exit(1)
rel_type = "linux-stable"
if (not rel_prep['stable']):
rel_type = "linux-next"
if (rel_prep['expected_tag'] != rel_describe):
logwrite('Unexpected %s based backports release tag on' % rel_type)
logwrite('the backports tree tree: %s\n' % rel_describe)
logwrite('You asked to make a release with this ')
logwrite('directory name: %s' % release)
logwrite('The actual expected tag we should find on')
logwrite('the backports tree then is: %s\n' % rel_prep['expected_tag'])
logwrite('For rules on the release name see upload_release()')
sys.exit(1)
copy_list = read_copy_list(args.copy_list)
deplist = read_dependencies(os.path.join(source_dir, 'dependencies'))
# validate output directory
check_output_dir(bpid.target_dir, args.clean)
# do the copy
backport_integrate_files = [
('Makefile.kernel', 'Makefile'),
('Kconfig.integrate', 'Kconfig'),
]
backport_package_files = [(x, x) for x in [
'Makefile',
'kconf/',
'Makefile.real',
'Makefile.kernel',
'Kconfig.package.hacks',
'scripts/',
'.blacklist.map',
'.gitignore',
'Makefile.build'] ]
backport_package_files += [
('Kconfig.package', 'Kconfig'),
]
backport_files = [(x, x) for x in [
'Kconfig.sources',
'compat/',
'backport-include/',
]]
if not bpid.integrate:
backport_files += backport_package_files
else:
backport_files += backport_integrate_files
if not args.git_revision:
logwrite('Copy original source files ...')
else:
logwrite('Get original source files from git ...')
copy_files(os.path.join(source_dir, 'backport'), backport_files, bpid.target_dir)
git_debug_init(args)
if not args.git_revision:
copy_files(args.kerneldir, copy_list, bpid.target_dir)
else:
copy_git_files(args.kerneldir, copy_list, args.git_revision, bpid.target_dir)
# FIXME: should we add a git version of this (e.g. --git-extra-driver)?
for src, copy_list in args.extra_driver:
if (args.kup or args.kup_test):
git_paranoia(src)
copy_files(src, read_copy_list(open(copy_list, 'r')), bpid.target_dir)
git_debug_snapshot(args, 'Add driver sources')
disable_list = add_automatic_backports(args)
if git_tracked_version:
backports_version = "(see git)"
kernel_version = "(see git)"
else:
backports_version = git.describe(tree=source_dir, extra_args=['--long'])
kernel_version = git.describe(rev=args.git_revision or 'HEAD',
tree=args.kerneldir,
extra_args=['--long'])
if not bpid.integrate:
f = open(os.path.join(bpid.target_dir, 'versions'), 'w')
f.write('BACKPORTS_VERSION="%s"\n' % backports_version)
f.write('BACKPORTED_KERNEL_VERSION="%s"\n' % kernel_version)
f.write('BACKPORTED_KERNEL_NAME="%s"\n' % args.base_name)
if git_tracked_version:
f.write('BACKPORTS_GIT_TRACKED="backport tracker ID: $(shell git rev-parse HEAD 2>/dev/null || echo \'not built in git tree\')"\n')
f.close()
git_debug_snapshot(args, "add versions files")
else:
kconf_regexes = [
(re.compile(r'.*(?P<key>%%BACKPORT_DIR%%)'), '%%BACKPORT_DIR%%', 'backports/'),
(re.compile(r'.*(?P<key>%%BACKPORTS_VERSION%%).*'), '%%BACKPORTS_VERSION%%', backports_version),
(re.compile(r'.*(?P<key>%%BACKPORTED_KERNEL_VERSION%%).*'), '%%BACKPORTED_KERNEL_VERSION%%', kernel_version),
(re.compile(r'.*(?P<key>%%BACKPORTED_KERNEL_NAME%%).*'), '%%BACKPORTED_KERNEL_NAME%%', args.base_name),
]
out = ''
for l in open(os.path.join(bpid.target_dir, 'Kconfig'), 'r'):
for r in kconf_regexes:
m = r[0].match(l)
if m:
l = re.sub(r'(' + r[1] + ')', r'' + r[2] + '', l)
out += l
outf = open(os.path.join(bpid.target_dir, 'Kconfig'), 'w')
outf.write(out)
outf.close()
git_debug_snapshot(args, "modify top level backports/Kconfig with backports identity")
if disable_list:
# No need to verify_sources() as compat's Kconfig has no 'source' call
bpcfg = kconfig.ConfigTree(os.path.join(bpid.target_dir, 'compat', 'Kconfig'), bpid)
bpcfg.disable_symbols(disable_list)
git_debug_snapshot(args, 'Add automatic backports')
failure = apply_patches(args, "backport", source_dir, 'patches', bpid.target_dir, logwrite)
if failure:
return failure
# Kernel integration requires Kconfig.versions already generated for you,
# we cannot do this for a package as we have no idea what kernel folks
# will be using.
if bpid.integrate:
kver = gen_version.kernelversion(bpid.project_dir)
rel_specs = gen_version.get_rel_spec_stable(kver)
if not rel_specs:
logwrite('Cannot parse source kernel version, update parser')
sys.exit(1)
data = gen_version.genkconfig_versions(rel_specs)
fo = open(os.path.join(bpid.target_dir, 'Kconfig.versions'), 'w')
fo.write(data)
fo.close()
git_debug_snapshot(args, "generate kernel version requirement Kconfig file")
# some post-processing is required
configtree = kconfig.ConfigTree(os.path.join(bpid.target_dir, 'Kconfig'), bpid)
ignore=['Kconfig.kernel', 'Kconfig.versions', 'Kconfig.local']
configtree.verify_sources(ignore=ignore)
git_debug_snapshot(args, "verify sources on top level backports Kconfig")
orig_symbols = configtree.symbols()
logwrite('Modify Kconfig tree ...')
configtree.prune_sources(ignore=ignore)
git_debug_snapshot(args, "prune Kconfig tree")
if not bpid.integrate:
configtree.force_tristate_modular()
git_debug_snapshot(args, "force tristate options modular")
ignore = [os.path.join(bpid.target_dir, x) for x in [
'Kconfig.package.hacks',
'Kconfig.versions',
'Kconfig.local',
'Kconfig',
]
]
configtree.adjust_backported_configs(ignore=ignore, orig_symbols=orig_symbols)
git_debug_snapshot(args, "adjust backports config symbols we port")
configtree.modify_selects()
git_debug_snapshot(args, "convert select to depends on")
symbols = configtree.symbols()
# write local symbol list -- needed during packaging build
if not bpid.integrate:
f = open(os.path.join(bpid.target_dir, 'local-symbols'), 'w')
for sym in symbols:
f.write('%s=\n' % sym)
f.close()
git_debug_snapshot(args, "add symbols files")
# also write Kconfig.local, representing all local symbols
# with a BACKPORTED_ prefix
f = open(os.path.join(bpid.target_dir, 'Kconfig.local'), 'w')
for sym in symbols:
f.write('config BACKPORTED_%s\n' % sym)
f.write('\ttristate\n')
f.write('\tdefault %s\n' % sym)
f.close()
git_debug_snapshot(args, "add Kconfig.local")
# add defconfigs that we want
defconfigs_dir = os.path.join(source_dir, 'backport', 'defconfigs')
os.mkdir(os.path.join(bpid.target_dir, 'defconfigs'))
for dfbase in os.listdir(defconfigs_dir):
copy_defconfig = True
dfsrc = os.path.join(defconfigs_dir, dfbase)
for line in open(dfsrc, 'r'):
if not '=' in line:
continue
line_ok = False
for sym in symbols:
if sym + '=' in line:
line_ok = True
break
if not line_ok:
copy_defconfig = False
break
if copy_defconfig:
shutil.copy(dfsrc, os.path.join(bpid.target_dir, 'defconfigs', dfbase))
git_debug_snapshot(args, "add (useful) defconfig files")
logwrite('Rewrite Makefiles and Kconfig files ...')
# rewrite Makefile and source symbols
# symbols we know only we can provide under the backport project prefix
# for which we need an exception.
skip_orig_syms = [ bpid.project_prefix + x for x in [
'INTEGRATE',
]
]
parse_orig_syms = [x for x in orig_symbols if x not in skip_orig_syms ]
regexes = []
for some_symbols in [parse_orig_syms[i:i + 50] for i in range(0, len(parse_orig_syms), 50)]:
r = 'CONFIG_((' + '|'.join([s + '(_MODULE)?' for s in some_symbols]) + ')([^A-Za-z0-9_]|$))'
regexes.append(re.compile(r, re.MULTILINE))
for root, dirs, files in os.walk(bpid.target_dir):
# don't go into .git dir (possible debug thing)
if '.git' in dirs:
dirs.remove('.git')
for f in files:
data = open(os.path.join(root, f), 'r').read()
for r in regexes:
data = r.sub(r'' + bpid.full_prefix + '\\1', data)
data = re.sub(r'\$\(srctree\)', '$(backport_srctree)', data)
data = re.sub(r'-Idrivers', '-I$(backport_srctree)/drivers', data)
if bpid.integrate:
data = re.sub(r'CPTCFG_', bpid.full_prefix, data)
fo = open(os.path.join(root, f), 'w')
fo.write(data)
fo.close()
git_debug_snapshot(args, "rename config symbol / srctree usage")
# disable unbuildable Kconfig symbols and stuff Makefiles that doesn't exist
if bpid.integrate:
maketree = make.MakeTree(os.path.join(bpid.target_dir, 'Makefile'))
else:
maketree = make.MakeTree(os.path.join(bpid.target_dir, 'Makefile.kernel'))
disable_kconfig = []
disable_makefile = []
for sym in maketree.get_impossible_symbols():
disable_kconfig.append(sym[7:])
disable_makefile.append(sym[7:])
configtree.disable_symbols(disable_kconfig)
git_debug_snapshot(args, "disable impossible kconfig symbols")
# add kernel version dependencies to Kconfig, from the dependency list
# we read previously
for sym in tuple(deplist.keys()):
new = []
for dep in deplist[sym]:
if "kconfig:" in dep:
kconfig_expr = dep.replace('kconfig: ', '')
new.append(kconfig_expr)
elif (dep == "DISABLE"):
new.append('BACKPORT_DISABLED_KCONFIG_OPTION')
else:
new.append('!KERNEL_%s' % dep.replace('.', '_'))
if bpid.integrate:
deplist[sym] = ["BACKPORT_" + x for x in new]
else:
deplist[sym] = new
configtree.add_dependencies(deplist)
git_debug_snapshot(args, "add kernel version dependencies")
# disable things in makefiles that can't be selected and that the
# build shouldn't recurse into because they don't exist -- if we
# don't do that then a symbol from the kernel could cause the build
# to attempt to recurse and fail
#
# Note that we split the regex after 50 symbols, this is because of a
# limitation in the regex implementation (it only supports 100 nested
# groups -- 50 seemed safer and is still fast)
regexes = []
for some_symbols in [disable_makefile[i:i + 50] for i in range(0, len(disable_makefile), 50)]:
r = '^(([^#].*((' + bpid.full_prefix_resafe + '|CONFIG_)(' + '|'.join([s for s in some_symbols]) + ')))\W)'
regexes.append(re.compile(r, re.MULTILINE))
for f in maketree.get_makefiles():
data = open(f, 'r').read()
for r in regexes:
data = r.sub(r'#\1', data)
fo = open(f, 'w')
fo.write(data)
fo.close()
git_debug_snapshot(args, "disable unsatisfied Makefile parts")
if bpid.integrate:
f = open(os.path.join(bpid.project_dir, 'Kconfig'), 'a')
f.write('source "backports/Kconfig"\n')
f.close()
git_debug_snapshot(args, "hooked backport to top level Kconfig")
failure = apply_patches(args, "integration", source_dir, 'integration-patches/',
bpid.project_dir, logwrite)
if failure:
return failure
if (args.kup or args.kup_test):
req = reqs.Req()
req.kup()
if not req.reqs_match():
sys.exit(1)
upload_release(args, rel_prep, logwrite=logwrite)
logwrite('Done!')
return 0
def process(kerneldir,
outdir,
copy_list_file,
git_revision=None,
clean=False,
refresh=False,
base_name="Linux",
gitdebug=False,
verbose=False,
extra_driver=[],
kup=False,
kup_test=False,
test_cocci=None,
profile_cocci=None,
logwrite=lambda x: None,
git_tracked_version=False):
class Args(object):
def __init__(self, kerneldir, outdir, copy_list_file, git_revision,
clean, refresh, base_name, gitdebug, verbose,
extra_driver, kup, kup_test, test_cocci, profile_cocci):
self.kerneldir = kerneldir
self.outdir = outdir
self.copy_list = copy_list_file
self.git_revision = git_revision
self.clean = clean
self.refresh = refresh
self.base_name = base_name
self.gitdebug = gitdebug
self.verbose = verbose
self.extra_driver = extra_driver
self.kup = kup
self.kup_test = kup_test
self.test_cocci = test_cocci
self.profile_cocci = profile_cocci
if self.test_cocci or self.profile_cocci:
self.gitdebug = True
def git_paranoia(tree=None, logwrite=lambda x: None):
data = git.paranoia(tree)
if (data['r'] != 0):
logwrite('Cannot use %s' % tree)
logwrite('%s' % data['output'])
sys.exit(data['r'])
else:
logwrite('Validated tree: %s' % tree)
args = Args(kerneldir, outdir, copy_list_file, git_revision, clean,
refresh, base_name, gitdebug, verbose, extra_driver, kup,
kup_test, test_cocci, profile_cocci)
rel_prep = None
# start processing ...
if (args.kup or args.kup_test):
git_paranoia(source_dir, logwrite)
git_paranoia(kerneldir, logwrite)
rel_describe = git.describe(rev=None,
tree=source_dir,
extra_args=['--dirty'])
release = os.path.basename(args.outdir)
version = release.replace("backports-", "")
rel_prep = get_rel_prep(version)
if (not rel_prep):
logwrite('Invalid backports release name: %s' % release)
logwrite('For rules on the release name see upload_release()')
sys.exit(1)
rel_type = "linux-stable"
if (not rel_prep['stable']):
rel_type = "linux-next"
if (rel_prep['expected_tag'] != rel_describe):
logwrite('Unexpected %s based backports release tag on' % rel_type)
logwrite('the backports tree tree: %s\n' % rel_describe)
logwrite('You asked to make a release with this ')
logwrite('directory name: %s' % release)
logwrite('The actual expected tag we should find on')
logwrite('the backports tree then is: %s\n' %
rel_prep['expected_tag'])
logwrite('For rules on the release name see upload_release()')
sys.exit(1)
copy_list = read_copy_list(args.copy_list)
deplist = read_dependencies(os.path.join(source_dir, 'dependencies'))
# validate output directory
check_output_dir(args.outdir, args.clean)
# do the copy
backport_files = [(x, x) for x in [
'Kconfig',
'Makefile',
'Makefile.build',
'Makefile.kernel',
'.gitignore',
'Makefile.real',
'compat/',
'backport-include/',
'kconf/',
'scripts/',
'.blacklist.map',
]]
if not args.git_revision:
logwrite('Copy original source files ...')
else:
logwrite('Get original source files from git ...')
copy_files(os.path.join(source_dir, 'backport'), backport_files,
args.outdir)
git_debug_init(args)
if not args.git_revision:
copy_files(args.kerneldir, copy_list, args.outdir)
else:
copy_git_files(args.kerneldir, copy_list, args.git_revision,
args.outdir)
# FIXME: should we add a git version of this (e.g. --git-extra-driver)?
for src, copy_list in args.extra_driver:
if (args.kup or args.kup_test):
git_paranoia(src)
copy_files(src, read_copy_list(open(copy_list, 'r')), args.outdir)
git_debug_snapshot(args, 'Add driver sources')
disable_list = add_automatic_backports(args)
if disable_list:
bpcfg = kconfig.ConfigTree(
os.path.join(args.outdir, 'compat', 'Kconfig'))
bpcfg.disable_symbols(disable_list)
git_debug_snapshot(args, 'Add automatic backports')
test_cocci = args.test_cocci or args.profile_cocci
logwrite('Apply patches ...')
patches = []
sempatches = []
for root, dirs, files in os.walk(os.path.join(source_dir, 'patches')):
for f in files:
if not test_cocci and f.endswith('.patch'):
patches.append(os.path.join(root, f))
if f.endswith('.cocci'):
if test_cocci:
if f not in test_cocci:
continue
if args.test_cocci:
logwrite("Testing Coccinelle SmPL patch: %s" %
test_cocci)
elif args.profile_cocci:
logwrite("Profiling Coccinelle SmPL patch: %s" %
test_cocci)
sempatches.append(os.path.join(root, f))
patches.sort()
prefix_len = len(os.path.join(source_dir, 'patches')) + 1
for pfile in patches:
print_name = pfile[prefix_len:]
# read the patch file
p = patch.fromfile(pfile)
# complain if it's not a patch
if not p:
raise Exception('No patch content found in %s' % print_name)
# leading / seems to be stripped?
if 'dev/null' in p.items[0].source:
raise Exception(
'Patches creating files are not supported (in %s)' %
print_name)
# check if the first file the patch touches exists, if so
# assume the patch needs to be applied -- otherwise continue
patched_file = '/'.join(p.items[0].source.split('/')[1:])
fullfn = os.path.join(args.outdir, patched_file)
if not os.path.exists(fullfn):
if args.verbose:
logwrite("Not applying %s, not needed" % print_name)
continue
if args.verbose:
logwrite("Applying patch %s" % print_name)
if args.refresh:
# but for refresh, of course look at all files the patch touches
for patchitem in p.items:
patched_file = '/'.join(patchitem.source.split('/')[1:])
fullfn = os.path.join(args.outdir, patched_file)
shutil.copyfile(fullfn, fullfn + '.orig_file')
process = subprocess.Popen(['patch', '-p1'],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
stdin=subprocess.PIPE,
close_fds=True,
universal_newlines=True,
cwd=args.outdir)
output = process.communicate(input=open(pfile, 'r').read())[0]
output = output.split('\n')
if output[-1] == '':
output = output[:-1]
if args.verbose:
for line in output:
logwrite('> %s' % line)
if process.returncode != 0:
if not args.verbose:
logwrite("Failed to apply changes from %s" % print_name)
for line in output:
logwrite('> %s' % line)
return 2
if args.refresh:
pfilef = open(pfile + '.tmp', 'a')
pfilef.write(p.top_header)
pfilef.flush()
for patchitem in p.items:
patched_file = '/'.join(patchitem.source.split('/')[1:])
fullfn = os.path.join(args.outdir, patched_file)
process = subprocess.Popen([
'diff', '-p', '-u', patched_file + '.orig_file',
patched_file, '--label', 'a/' + patched_file, '--label',
'b/' + patched_file
],
stdout=pfilef,
close_fds=True,
universal_newlines=True,
cwd=args.outdir)
process.wait()
os.unlink(fullfn + '.orig_file')
if not process.returncode in (0, 1):
logwrite("Failed to diff to refresh %s" % print_name)
pfilef.close()
os.unlink(pfile + '.tmp')
return 3
pfilef.close()
os.rename(pfile + '.tmp', pfile)
# remove orig/rej files that patch sometimes creates
for root, dirs, files in os.walk(args.outdir):
for f in files:
if f[-5:] == '.orig' or f[-4:] == '.rej':
os.unlink(os.path.join(root, f))
git_debug_snapshot(args, "apply backport patch %s" % print_name)
sempatches.sort()
prefix_len = len(os.path.join(source_dir, 'patches')) + 1
for cocci_file in sempatches:
# Until Coccinelle picks this up
pycocci = os.path.join(source_dir, 'devel/pycocci')
cmd = [pycocci, cocci_file]
extra_spatch_args = []
if args.profile_cocci:
cmd.append('--profile-cocci')
cmd.append(args.outdir)
print_name = cocci_file[prefix_len:]
if args.verbose:
logwrite("Applying SmPL patch %s" % print_name)
sprocess = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
close_fds=True,
universal_newlines=True,
cwd=args.outdir)
output = sprocess.communicate()[0]
sprocess.wait()
if sprocess.returncode != 0:
logwrite("Failed to process SmPL patch %s" % print_name)
return 3
output = output.split('\n')
if output[-1] == '':
output = output[:-1]
if args.verbose:
for line in output:
logwrite('> %s' % line)
# remove cocci_backup files
for root, dirs, files in os.walk(args.outdir):
for f in files:
if f.endswith('.cocci_backup'):
os.unlink(os.path.join(root, f))
git_debug_snapshot(args, "apply backport SmPL patch %s" % print_name)
if test_cocci:
logwrite('Done!')
return 0
# some post-processing is required
configtree = kconfig.ConfigTree(os.path.join(args.outdir, 'Kconfig'))
logwrite('Modify Kconfig tree ...')
configtree.prune_sources(ignore=['Kconfig.kernel', 'Kconfig.versions'])
git_debug_snapshot(args, "prune Kconfig tree")
configtree.force_tristate_modular()
git_debug_snapshot(args, "force tristate options modular")
configtree.modify_selects()
git_debug_snapshot(args, "convert select to depends on")
# write the versioning file
if git_tracked_version:
backports_version = "(see git)"
kernel_version = "(see git)"
else:
backports_version = git.describe(tree=source_dir,
extra_args=['--long'])
kernel_version = git.describe(rev=args.git_revision or 'HEAD',
tree=args.kerneldir,
extra_args=['--long'])
f = open(os.path.join(args.outdir, 'versions'), 'w')
f.write('BACKPORTS_VERSION="%s"\n' % backports_version)
f.write('BACKPORTED_KERNEL_VERSION="%s"\n' % kernel_version)
f.write('BACKPORTED_KERNEL_NAME="%s"\n' % args.base_name)
if git_tracked_version:
f.write(
'BACKPORTS_GIT_TRACKED="backport tracker ID: $(shell git rev-parse HEAD 2>/dev/null || echo \'not built in git tree\')"\n'
)
f.close()
symbols = configtree.symbols()
# write local symbol list -- needed during build
f = open(os.path.join(args.outdir, '.local-symbols'), 'w')
for sym in symbols:
f.write('%s=\n' % sym)
f.close()
git_debug_snapshot(args, "add versions/symbols files")
# add defconfigs that we want
defconfigs_dir = os.path.join(source_dir, 'backport', 'defconfigs')
os.mkdir(os.path.join(args.outdir, 'defconfigs'))
for dfbase in os.listdir(defconfigs_dir):
copy_defconfig = True
dfsrc = os.path.join(defconfigs_dir, dfbase)
for line in open(dfsrc, 'r'):
if not '=' in line:
continue
line_ok = False
for sym in symbols:
if sym + '=' in line:
line_ok = True
break
if not line_ok:
print dfbase, line
copy_defconfig = False
break
if copy_defconfig:
shutil.copy(dfsrc, os.path.join(args.outdir, 'defconfigs', dfbase))
git_debug_snapshot(args, "add (useful) defconfig files")
logwrite('Rewrite Makefiles and Kconfig files ...')
# rewrite Makefile and source symbols
regexes = []
for some_symbols in [
symbols[i:i + 50] for i in range(0, len(symbols), 50)
]:
r = 'CONFIG_((' + '|'.join([s + '(_MODULE)?' for s in some_symbols
]) + ')([^A-Za-z0-9_]|$))'
regexes.append(re.compile(r, re.MULTILINE))
for root, dirs, files in os.walk(args.outdir):
# don't go into .git dir (possible debug thing)
if '.git' in dirs:
dirs.remove('.git')
for f in files:
data = open(os.path.join(root, f), 'r').read()
for r in regexes:
data = r.sub(r'CPTCFG_\1', data)
data = re.sub(r'\$\(srctree\)', '$(backport_srctree)', data)
data = re.sub(r'-Idrivers', '-I$(backport_srctree)/drivers', data)
fo = open(os.path.join(root, f), 'w')
fo.write(data)
fo.close()
git_debug_snapshot(args, "rename config symbol / srctree usage")
# disable unbuildable Kconfig symbols and stuff Makefiles that doesn't exist
maketree = make.MakeTree(os.path.join(args.outdir, 'Makefile.kernel'))
disable_kconfig = []
disable_makefile = []
for sym in maketree.get_impossible_symbols():
disable_kconfig.append(sym[7:])
disable_makefile.append(sym[7:])
configtree.disable_symbols(disable_kconfig)
git_debug_snapshot(args, "disable impossible kconfig symbols")
# add kernel version dependencies to Kconfig, from the dependency list
# we read previously
for sym in tuple(deplist.keys()):
new = []
for dep in deplist[sym]:
if "kconfig:" in dep:
kconfig_expr = dep.replace('kconfig: ', '')
new.append(kconfig_expr)
elif (dep == "DISABLE"):
new.append('BACKPORT_DISABLED_KCONFIG_OPTION')
else:
new.append('!BACKPORT_KERNEL_%s' % dep.replace('.', '_'))
deplist[sym] = new
configtree.add_dependencies(deplist)
git_debug_snapshot(args, "add kernel version dependencies")
# disable things in makefiles that can't be selected and that the
# build shouldn't recurse into because they don't exist -- if we
# don't do that then a symbol from the kernel could cause the build
# to attempt to recurse and fail
#
# Note that we split the regex after 50 symbols, this is because of a
# limitation in the regex implementation (it only supports 100 nested
# groups -- 50 seemed safer and is still fast)
regexes = []
for some_symbols in [
disable_makefile[i:i + 50]
for i in range(0, len(disable_makefile), 50)
]:
r = '^([^#].*((CPTCFG|CONFIG)_(' + '|'.join([s for s in some_symbols
]) + ')))'
regexes.append(re.compile(r, re.MULTILINE))
for f in maketree.get_makefiles():
data = open(f, 'r').read()
for r in regexes:
data = r.sub(r'#\1', data)
fo = open(f, 'w')
fo.write(data)
fo.close()
git_debug_snapshot(args, "disable unsatisfied Makefile parts")
if (args.kup or args.kup_test):
req = reqs.Req()
req.kup()
if not req.reqs_match():
sys.exit(1)
upload_release(args, rel_prep, logwrite=logwrite)
logwrite('Done!')
return 0
def _main():
# Our binary requirements go here
req = reqs.Req()
req.require('git')
req.coccinelle('1.0.0-rc24')
if not req.reqs_match():
sys.exit(1)
# set up and parse arguments
parser = argparse.ArgumentParser(description='generate backport tree')
parser.add_argument('kerneldir', metavar='<kernel tree>', type=str,
help='Kernel tree to copy drivers from')
parser.add_argument('outdir', metavar='<output directory>', type=str,
help='Directory to write the generated tree to')
parser.add_argument('--copy-list', metavar='<listfile>', type=argparse.FileType('r'),
default='copy-list',
help='File containing list of files/directories to copy, default "copy-list"')
parser.add_argument('--git-revision', metavar='<revision>', type=str,
help='git commit revision (see gitrevisions(7)) to take objects from.' +
'If this is specified, the kernel tree is used as git object storage ' +
'and we use git ls-tree to get the files.')
parser.add_argument('--clean', const=True, default=False, action="store_const",
help='Clean output directory instead of erroring if it isn\'t empty')
parser.add_argument('--integrate', const=True, default=False, action="store_const",
help='Integrate a future backported kernel solution into ' +
'an older kernel tree source directory.')
parser.add_argument('--refresh', const=True, default=False, action="store_const",
help='Refresh patches as they are applied, the source dir will be modified!')
parser.add_argument('--base-name', metavar='<name>', type=str, default='Linux',
help='name of base tree, default just "Linux"')
parser.add_argument('--gitdebug', const=True, default=False, action="store_const",
help='Use git, in the output tree, to debug the various transformation steps ' +
'that the tree generation makes (apply patches, ...)')
parser.add_argument('--verbose', const=True, default=False, action="store_const",
help='Print more verbose information')
parser.add_argument('--extra-driver', nargs=2, metavar=('<source dir>', '<copy-list>'), type=str,
action='append', default=[], help='Extra driver directory/copy-list.')
parser.add_argument('--kup', const=True, default=False, action="store_const",
help='For maintainers: upload a release to kernel.org')
parser.add_argument('--kup-test', const=True, default=False, action="store_const",
help='For maintainers: do all the work as if you were about to ' +
'upload to kernel.org but do not do the final `kup put` ' +
'and also do not run any `kup mkdir` commands. This will ' +
'however run `kup ls` on the target paths so ' +
'at the very least we test your kup configuration. ' +
'If this is your first time uploading use this first!')
parser.add_argument('--test-cocci', metavar='<sp_file>', type=str, default=None,
help='Only use the cocci file passed for Coccinelle, don\'t do anything else, ' +
'also creates a git repo on the target directory for easy inspection ' +
'of changes done by Coccinelle.')
parser.add_argument('--profile-cocci', metavar='<sp_file>', type=str, default=None,
help='Only use the cocci file passed and pass --profile to Coccinelle, ' +
'also creates a git repo on the target directory for easy inspection ' +
'of changes done by Coccinelle.')
args = parser.parse_args()
# When building a package we use CPTCFG as we can rely on the
# fact that kconfig treats CONFIG_ as an environment variable
# requring less changes on code. For kernel integration we use
# the longer CONFIG_BACKPORT given that we'll be sticking to
# the kernel symbol namespace, to address that we do a final
# search / replace. Technically its possible to rely on the
# same prefix for packaging as with kernel integration but
# there are already some users of the CPTCFG prefix.
bpid = None
if args.integrate:
bpid = Bp_Identity(integrate = args.integrate,
kconfig_prefix = 'CONFIG_',
project_prefix = 'BACKPORT_',
project_dir = args.outdir,
target_dir = os.path.join(args.outdir, 'backports/'),
target_dir_name = 'backports/',
kconfig_source_var = '$BACKPORT_DIR',
)
else:
bpid = Bp_Identity(integrate = args.integrate,
kconfig_prefix = 'CPTCFG_',
project_prefix = '',
project_dir = args.outdir,
target_dir = args.outdir,
target_dir_name = '',
kconfig_source_var = '$BACKPORT_DIR',
)
def logwrite(msg):
sys.stdout.write(msg)
sys.stdout.write('\n')
sys.stdout.flush()
return process(args.kerneldir, args.copy_list,
git_revision=args.git_revision,
bpid=bpid,
clean=args.clean,
refresh=args.refresh, base_name=args.base_name,
gitdebug=args.gitdebug, verbose=args.verbose,
extra_driver=args.extra_driver,
kup=args.kup,
kup_test=args.kup_test,
test_cocci=args.test_cocci,
profile_cocci=args.profile_cocci,
logwrite=logwrite)
def _main():
# Our binary requirements go here
req = reqs.Req()
req.require('git')
req.coccinelle('1.0.0-rc21')
if not req.reqs_match():
sys.exit(1)
# set up and parse arguments
parser = argparse.ArgumentParser(description='generate backport tree')
parser.add_argument('kerneldir',
metavar='<kernel tree>',
type=str,
help='Kernel tree to copy drivers from')
parser.add_argument('outdir',
metavar='<output directory>',
type=str,
help='Directory to write the generated tree to')
parser.add_argument(
'--copy-list',
metavar='<listfile>',
type=argparse.FileType('r'),
default='copy-list',
help=
'File containing list of files/directories to copy, default "copy-list"'
)
parser.add_argument(
'--git-revision',
metavar='<revision>',
type=str,
help='git commit revision (see gitrevisions(7)) to take objects from.'
+
'If this is specified, the kernel tree is used as git object storage '
+ 'and we use git ls-tree to get the files.')
parser.add_argument(
'--clean',
const=True,
default=False,
action="store_const",
help='Clean output directory instead of erroring if it isn\'t empty')
parser.add_argument(
'--refresh',
const=True,
default=False,
action="store_const",
help=
'Refresh patches as they are applied, the source dir will be modified!'
)
parser.add_argument('--base-name',
metavar='<name>',
type=str,
default='Linux',
help='name of base tree, default just "Linux"')
parser.add_argument(
'--gitdebug',
const=True,
default=False,
action="store_const",
help=
'Use git, in the output tree, to debug the various transformation steps '
+ 'that the tree generation makes (apply patches, ...)')
parser.add_argument('--verbose',
const=True,
default=False,
action="store_const",
help='Print more verbose information')
parser.add_argument('--extra-driver',
nargs=2,
metavar=('<source dir>', '<copy-list>'),
type=str,
action='append',
default=[],
help='Extra driver directory/copy-list.')
parser.add_argument('--kup',
const=True,
default=False,
action="store_const",
help='For maintainers: upload a release to kernel.org')
parser.add_argument(
'--kup-test',
const=True,
default=False,
action="store_const",
help='For maintainers: do all the work as if you were about to ' +
'upload to kernel.org but do not do the final `kup put` ' +
'and also do not run any `kup mkdir` commands. This will ' +
'however run `kup ls` on the target paths so ' +
'at the very least we test your kup configuration. ' +
'If this is your first time uploading use this first!')
parser.add_argument(
'--test-cocci',
metavar='<sp_file>',
type=str,
default=None,
help=
'Only use the cocci file passed for Coccinelle, don\'t do anything else, '
+
'also creates a git repo on the target directory for easy inspection '
+ 'of changes done by Coccinelle.')
parser.add_argument(
'--profile-cocci',
metavar='<sp_file>',
type=str,
default=None,
help=
'Only use the cocci file passed and pass --profile to Coccinelle, ' +
'also creates a git repo on the target directory for easy inspection '
+ 'of changes done by Coccinelle.')
args = parser.parse_args()
def logwrite(msg):
sys.stdout.write(msg)
sys.stdout.write('\n')
sys.stdout.flush()
return process(args.kerneldir,
args.outdir,
args.copy_list,
git_revision=args.git_revision,
clean=args.clean,
refresh=args.refresh,
base_name=args.base_name,
gitdebug=args.gitdebug,
verbose=args.verbose,
extra_driver=args.extra_driver,
kup=args.kup,
kup_test=args.kup_test,
test_cocci=args.test_cocci,
profile_cocci=args.profile_cocci,
logwrite=logwrite)