def test_whitespace_preservation(self): source = "ifeq ' x' 'x '\n$(error stripping)\nendif" statements = parsestring(source, 'foo.mk') self.assertEqual(statements.to_source(), source) source = 'ifneq (x , x)\n$(error stripping)\nendif' statements = parsestring(source, 'foo.mk') self.assertEqual(statements.to_source(), 'ifneq (x,x)\n$(error stripping)\nendif')
def resolve(self, makefile, variables, fd, setting): if makefile.parsingfinished: # GNU make allows variables to be set by recursive expansion during # command execution. This seems really dumb to me, so I don't! raise errors.DataError( "$(eval) not allowed via recursive expansion after parsing is finished", self.loc) stmts = parser.parsestring( self._arguments[0].resolvestr(makefile, variables, setting), 'evaluation from %s' % self.loc) stmts.execute(makefile)
def _get_test_files(self): ourdir = os.path.dirname(os.path.abspath(__file__)) for makefile in glob.glob(os.path.join(ourdir, '*.mk')): if os.path.basename(makefile) in self._IGNORE_FILES: continue source = None with open(makefile, 'rU') as fh: source = fh.read() try: yield (makefile, source, parsestring(source, makefile)) except SyntaxError: continue
def enter(subdir): if reduce(operator.or_, map(os.path.samefile, skip, repeat(subdir, len(skip)))): # print "SKIPPING ", subdir return kbuild = get_kbuild(subdir) print "kbuild ", kbuild fd = open(kbuild, 'rU') s = fd.read() fd.close() stmts = parser.parsestring(s, kbuild) for s in stmts: if isinstance(s, parserdata.SetVariable): analyze(subdir, s) elif isinstance(s, parserdata.ConditionBlock): # print "NEED TO SUPPORT CONDITION" pass else: # print "UNSUPPORTED STATEMENT TYPE" pass
def test_reparse_consistency(self): for filename, source, statements in self._get_test_files(): reformatted = statements.to_source() # We should be able to parse the reformatted source fine. new_statements = parsestring(reformatted, filename) # If we do the formatting again, the representation shouldn't # change. i.e. the only lossy change should be the original # (whitespace and some semantics aren't preserved). reformatted_again = new_statements.to_source() self.assertEqual(reformatted, reformatted_again, '%s has lossless reformat.' % filename) self.assertEqual(len(statements), len(new_statements)) for i in xrange(0, len(statements)): original = statements[i] formatted = new_statements[i] self.assertEqual(original, formatted, '%s %d: %s != %s' % (filename, i, original, formatted))
def test_extra_statements(self): source = 'ifdef FOO\nF := 1\nifdef BAR\nB += 1\nendif\nC = 1\nendif' statements = parsestring(source, 'foo.mk') self.assertEqual(statements.to_source(), source)
def test_mixed_conditions(self): source = 'ifdef FOO\nifeq ($(FOO),bar)\nvar += $(value)\nendif\nendif' statements = parsestring(source, 'foo.mk') self.assertEqual(statements.to_source(), source)
def test_negation(self): source = 'ifneq (foo,bar)\nhello = world\nendif' statements = parsestring(source, 'foo.mk') self.assertEqual(statements.to_source(), source)
def extract(makefile_path, compilation_units, library_units, composites, hostprog_units, hostprog_composites, unconfigurable_units, extra_targets, clean_files, c_file_targets): debug("processing makefile", makefile_path) if os.path.isdir(makefile_path): subdir = makefile_path makefile_path = os.path.join(subdir, "Kbuild") if not os.path.isfile(makefile_path): makefile_path = os.path.join(subdir, "Makefile") if not os.path.isfile(makefile_path): fatal("not found", makefile_path) exit(1) obj = os.path.dirname(makefile_path) makefile = open(makefile_path, "rU") s = makefile.read() makefile.close() statements = parser.parsestring(s, makefile.name) kbuild = Kbuild() kbuild.add_definitions(args.define) kbuild.process_statements(statements, kbuild.T) # OPTIMIZE: list by combining non-exclusive configurations # OPTIMIZE: find maximal list and combine configurations # TODO: emit list of configurations for the dry-runs # TODO: merge equivalence between CONFIG= and !defined(CONFIG) subdirectories = set() collect_units(kbuild, obj, set([ "obj-y", "obj-m" ]), compilation_units, subdirectories, composites) collect_units(kbuild, obj, set([ "lib-y", "lib-m" ]), library_units, None, None) pending_hostprog_composites = set([]) for v in set([ "hostprogs-y", "hostprogs-m", "host-progs", "always" ]): for u in split_definitions(kbuild, v): composite_name = u + "-objs" unit_name = os.path.join(obj, u) if composite_name in kbuild.variables: pending_hostprog_composites.add(composite_name) hostprog_composites.add(unit_name) else: hostprog_units.add(unit_name) if (len(pending_hostprog_composites) > 0): collect_units(kbuild, obj, pending_hostprog_composites, hostprog_units, None, hostprog_composites) for v in set([ "targets", "extra-y" ]): for u in split_definitions(kbuild, v): unit_name = os.path.join(obj, u) if unit_name.endswith(".o"): extra_targets.add(unit_name) for u in split_definitions(kbuild, "targets"): if u.endswith(".c"): c_file_targets.add(os.path.join(obj, u)) for u in split_definitions(kbuild, "clean-files"): clean_files.add(os.path.join(obj, u)) # look for variables starting with obj-, lib-, hostprogs-, # compositename-, or hostprog-compositename-. doesn't account for # composites in the unconfigurable variables unconfigurable_prefixes = set([ "obj-$", "lib-$", "hostprogs-$" ]) for cset in (composites, hostprog_composites): for c in cset: c = os.path.basename(c) if c.endswith(".o"): c = c[:-2] unconfigurable_prefixes.add(c + "-$") unconfigurable_variables = set([]) for x in kbuild.variables: for p in unconfigurable_prefixes: if x.startswith(p) and \ not x.endswith("-") and \ not x.endswith("-y") and \ not x.endswith("-m") and \ not x.endswith("-objs") and \ x != "host-progs": unconfigurable_variables.add(x) collect_units(kbuild, obj, unconfigurable_variables, unconfigurable_units, None, None) # look for variable expansions or function calls in # compilation_units, subdirectories, and variable names check_unexpanded_variables(compilation_units, "compilation unit") check_unexpanded_variables(subdirectories, "subdirectory") check_unexpanded_variables(kbuild.variables.keys(), "variable name") if args.variable: kbuild.print_variable(args.variable, kbuild.variables[args.variable]) elif args.table: kbuild.print_variables_table(kbuild.variables) _pycudd.delete_DdManager(kbuild.mgr) return subdirectories
def test_nested(self): source = 'ifdef FOO\nifdef BAR\nhello = world\nendif\nendif' statements = parsestring(source, 'foo.mk') self.assertEqual(statements[0].to_source(), source)
def test_simple(self): source = 'ifdef FOO\nbar := $(value)\nendif' statements = parsestring(source, 'foo.mk') self.assertEqual(statements[0].to_source(), source)
def test_multiline_immediate(self): source = 'define FOO :=\nhello\nworld\nendef' statements = parsestring(source, 'foo.mk') self.assertEqual(statements.to_source(), source)
makefiles = [] for f in args.makefiles: if os.path.isdir(f): subdir = f f = os.path.join(subdir, "Kbuild") if not os.path.isfile(f): f = os.path.join(subdir, "Makefile") if not os.path.isfile(f): print f, "not found" else: makefiles.append(open(f, "rU")) for f in makefiles: s = f.read() f.close() stmts = parser.parsestring(s, f.name) process_statements(stmts) for var in reassigned.keys(): if assigned[var] > 1: reassignments += 1 for r in [ reassigned[var][:i] for i in range(1, len(reassigned[var]) + 1) ]: r = [ x for x in r if x != 'y' and x != 'n' and x != 'm' ] if len(r) > 0: foff = ' '.join(r) print 'reassigns', var, foff if not force_off[foff]: force_off[foff] = True for foff in [ x for x in force_off.keys() if force_off[x] ]: print 'force_off', foff
makefiles = [] for f in args.makefiles: if os.path.isdir(f): subdir = f f = os.path.join(subdir, "Kbuild") if not os.path.isfile(f): f = os.path.join(subdir, "Makefile") if not os.path.isfile(f): print f, "not found" else: makefiles.append(open(f, "rU")) def process_statements(stmts): for s in stmts: if isinstance(s, parserdata.ConditionBlock): for c, cstmts in s: print "cond:", c process_statements(s) elif isinstance(s, parserdata.SetVariable): if s.token in ['=', ':=', '?=']: print s.to_source() for f in makefiles: s = f.read() f.close() stmts = parser.parsestring(s, f.name) process_statements(stmts)
def test_negation(self): source = 'ifndef FOO\nbar += value\nendif' statements = parsestring(source, 'foo.mk') self.assertEqual(statements[0].to_source(), source)
import sys from os import path from pymake import parser, parserdata, data, functions import operator from itertools import repeat from collections import defaultdict def go(stmts): for s in stmts: if isinstance(s, parserdata.SetVariable): objs = s.value.split() for o in objs: if o[-2:] == ".o": print o elif isinstance(s, parserdata.ConditionBlock): for c in s._groups: _, l = c go(l) kbuild=sys.argv[1] fd = open(kbuild, 'rU') s = fd.read() fd.close() stmts = parser.parsestring(s, kbuild) go(stmts)
def test_simple(self): source = 'ifeq ($(foo),bar)\nhello = $(world)\nendif' statements = parsestring(source, 'foo.mk') self.assertEqual(statements[0].to_source(), source)
# compilation unit are a disjunction. import sys from os import path from pymake import parser, parserdata, data, functions import operator from itertools import repeat from collections import defaultdict def go(stmts): for s in stmts: if isinstance(s, parserdata.SetVariable): objs = s.value.split() for o in objs: if o[-2:] == ".o": print o elif isinstance(s, parserdata.ConditionBlock): for c in s._groups: _, l = c go(l) kbuild = sys.argv[1] fd = open(kbuild, 'rU') s = fd.read() fd.close() stmts = parser.parsestring(s, kbuild) go(stmts)
def extract(makefile_path, compilation_units, library_units, composites, hostprog_units, hostprog_composites, unconfigurable_units, extra_targets, clean_files, c_file_targets): debug("processing makefile", makefile_path) if os.path.isdir(makefile_path): subdir = makefile_path makefile_path = os.path.join(subdir, "Kbuild") if not os.path.isfile(makefile_path): makefile_path = os.path.join(subdir, "Makefile") if not os.path.isfile(makefile_path): fatal("not found", makefile_path) exit(1) obj = os.path.dirname(makefile_path) makefile = open(makefile_path, "rU") s = makefile.read() makefile.close() statements = parser.parsestring(s, makefile.name) kbuild = Kbuild() kbuild.add_definitions(args.define) kbuild.process_statements(statements, kbuild.T) # OPTIMIZE: list by combining non-exclusive configurations # OPTIMIZE: find maximal list and combine configurations # TODO: emit list of configurations for the dry-runs # TODO: merge equivalence between CONFIG= and !defined(CONFIG) subdirectories = set() collect_units(kbuild, obj, set(["obj-y", "obj-m"]), compilation_units, subdirectories, composites) collect_units(kbuild, obj, set(["lib-y", "lib-m"]), library_units, None, None) pending_hostprog_composites = set([]) for v in set(["hostprogs-y", "hostprogs-m", "host-progs", "always"]): for u in split_definitions(kbuild, v): composite_name = u + "-objs" unit_name = os.path.join(obj, u) if composite_name in kbuild.variables: pending_hostprog_composites.add(composite_name) hostprog_composites.add(unit_name) else: hostprog_units.add(unit_name) if (len(pending_hostprog_composites) > 0): collect_units(kbuild, obj, pending_hostprog_composites, hostprog_units, None, hostprog_composites) for v in set(["targets", "extra-y"]): for u in split_definitions(kbuild, v): unit_name = os.path.join(obj, u) if unit_name.endswith(".o"): extra_targets.add(unit_name) for u in split_definitions(kbuild, "targets"): if u.endswith(".c"): c_file_targets.add(os.path.join(obj, u)) for u in split_definitions(kbuild, "clean-files"): clean_files.add(os.path.join(obj, u)) # look for variables starting with obj-, lib-, hostprogs-, # compositename-, or hostprog-compositename-. doesn't account for # composites in the unconfigurable variables unconfigurable_prefixes = set(["obj-$", "lib-$", "hostprogs-$"]) for cset in (composites, hostprog_composites): for c in cset: c = os.path.basename(c) if c.endswith(".o"): c = c[:-2] unconfigurable_prefixes.add(c + "-$") unconfigurable_variables = set([]) for x in kbuild.variables: for p in unconfigurable_prefixes: if x.startswith(p) and \ not x.endswith("-") and \ not x.endswith("-y") and \ not x.endswith("-m") and \ not x.endswith("-objs") and \ x != "host-progs": unconfigurable_variables.add(x) collect_units(kbuild, obj, unconfigurable_variables, unconfigurable_units, None, None) # look for variable expansions or function calls in # compilation_units, subdirectories, and variable names check_unexpanded_variables(compilation_units, "compilation unit") check_unexpanded_variables(subdirectories, "subdirectory") check_unexpanded_variables(kbuild.variables.keys(), "variable name") if args.variable: kbuild.print_variable(args.variable, kbuild.variables[args.variable]) elif args.table: kbuild.print_variables_table(kbuild.variables) _pycudd.delete_DdManager(kbuild.mgr) return subdirectories