def main(): parser = argparse.ArgumentParser(description='Find duplicate files in directory.') parser.add_argument('--warning', '-w', action='store_true', help='Only warn about duplicates, do not exit with an error') parser.add_argument('--file', '-f', action='append', dest='dupes_files', default=[], help='Add exceptions to the duplicate list from this file') parser.add_argument('-D', action=DefinesAction) parser.add_argument('-U', action='append', default=[]) parser.add_argument('directory', help='The directory to check for duplicates in') args = parser.parse_args() allowed_dupes = [] for filename in args.dupes_files: pp = Preprocessor() pp.context.update(buildconfig.defines['ALLDEFINES']) if args.D: pp.context.update(args.D) for undefine in args.U: if undefine in pp.context: del pp.context[undefine] pp.out = StringIO() pp.do_filter('substitution') pp.do_include(filename) allowed_dupes.extend([line.partition('#')[0].rstrip() for line in pp.out.getvalue().splitlines()]) find_dupes(args.directory, bail=not args.warning, allowed_dupes=allowed_dupes)
def preprocess(path, defines): pp = Preprocessor(defines=defines, marker='%') pp.context.update(defines) pp.out = io.StringIO() pp.do_filter('substitution') pp.do_include(io.open(path, 'r', encoding='latin1')) pp.out.seek(0) return pp.out
def preprocess(path, defines): pp = Preprocessor(defines=defines, marker="%") pp.context.update(defines) pp.out = io.StringIO() pp.do_filter("substitution") pp.do_include(io.open(path, "r", encoding="latin1")) pp.out.seek(0) return pp.out
def preprocess(input, parser, defines={}): ''' Preprocess the file-like input with the given defines, and send the preprocessed output line by line to the given parser. ''' pp = Preprocessor() pp.context.update(defines) pp.do_filter('substitution') pp.out = PreprocessorOutputWrapper(pp, parser) pp.do_include(input)
def emit_code(fd, pref_list_filename): pp = Preprocessor() pp.context.update(buildconfig.defines["ALLDEFINES"]) # A necessary hack until MOZ_DEBUG_FLAGS are part of buildconfig.defines. if buildconfig.substs.get("MOZ_DEBUG"): pp.context["DEBUG"] = "1" if buildconfig.substs.get("CPU_ARCH") == "aarch64": pp.context["MOZ_AARCH64"] = True pp.out = StringIO() pp.do_filter("substitution") pp.do_include(pref_list_filename) try: pref_list = yaml.safe_load(pp.out.getvalue()) input_file = os.path.relpath( pref_list_filename, os.environ.get("GECKO_PATH", os.environ.get("TOPSRCDIR")), ) code = generate_code(pref_list, input_file) except (IOError, ValueError) as e: print("{}: error:\n {}\n".format(pref_list_filename, e)) sys.exit(1) # When generating multiple files from a script, the build system treats the # first named output file (StaticPrefListAll.h in this case) specially -- it # is created elsewhere, and written to via `fd`. fd.write(code["static_pref_list_all_h"]) # We must create the remaining output files ourselves. This requires # creating the output directory directly if it doesn't already exist. ensureParentDir(fd.name) init_dirname = os.path.dirname(fd.name) with FileAvoidWrite("StaticPrefsAll.h") as fd: fd.write(code["static_prefs_all_h"]) for group, text in sorted(code["static_pref_list_group_h"].items()): filename = "StaticPrefList_{}.h".format(group) with FileAvoidWrite(os.path.join(init_dirname, filename)) as fd: fd.write(text) for group, text in sorted(code["static_prefs_group_h"].items()): filename = "StaticPrefs_{}.h".format(group) with FileAvoidWrite(filename) as fd: fd.write(text) with FileAvoidWrite(os.path.join(init_dirname, "StaticPrefsCGetters.cpp")) as fd: fd.write(code["static_prefs_c_getters_cpp"]) with FileAvoidWrite("static_prefs.rs") as fd: fd.write(code["static_prefs_rs"])
def main(output, input_file): pp = Preprocessor() pp.context.update({ 'FFI_EXEC_TRAMPOLINE_TABLE': '0', 'HAVE_LONG_DOUBLE': '0', 'TARGET': buildconfig.substs['FFI_TARGET'], 'VERSION': '', }) pp.do_filter('substitution') pp.setMarker(None) pp.out = output pp.do_include(input_file)
def main(output, input_file, *defines): pp = Preprocessor() pp.context.update({ "FFI_EXEC_TRAMPOLINE_TABLE": "0", "HAVE_LONG_DOUBLE": "0", "TARGET": buildconfig.substs["FFI_TARGET"], "VERSION": "", }) for d in defines: pp.context.update({d: "1"}) pp.do_filter("substitution") pp.setMarker(None) pp.out = output pp.do_include(input_file)
def main(output, input_file): pp = Preprocessor() pp.context.update( { "FFI_EXEC_TRAMPOLINE_TABLE": "0", "HAVE_LONG_DOUBLE": "0", "TARGET": buildconfig.substs["FFI_TARGET"], "VERSION": "", } ) pp.do_filter("substitution") pp.setMarker(None) pp.out = output pp.do_include(input_file)
def rnp_preprocess(tmpl, dest, defines): """ Generic preprocessing :param BinaryIO tmpl: open filehandle (read) input :param BinaryIO dest: open filehandle (write) output :param dict defines: result of get_defines() :return boolean: """ pp = Preprocessor() pp.setMarker("%") pp.addDefines(defines) pp.do_filter("substitution") pp.out = dest pp.do_include(tmpl, True) return True
def process_package_overload(src, dst, version, app_buildid): ensureParentDir(dst) # First replace numeric version like '1.3' # Then replace with 'slashed' version like '1_4' # Finally set the full length addon version like 1.3.20131230 defines = { "NUM_VERSION": version, "SLASH_VERSION": version.replace(".", "_"), "FULL_VERSION": ("%s.%s" % (version, app_buildid)) } pp = Preprocessor(defines=defines) pp.do_filter("substitution") with open(dst, "w") as output: with open(src, "r") as input: pp.processFile(input=input, output=output)
def emit_code(fd, pref_list_filename): pp = Preprocessor() pp.context.update(buildconfig.defines['ALLDEFINES']) # A necessary hack until MOZ_DEBUG_FLAGS are part of buildconfig.defines. if buildconfig.substs.get('MOZ_DEBUG'): pp.context['DEBUG'] = '1' pp.out = BytesIO() pp.do_filter('substitution') pp.do_include(pref_list_filename) try: pref_list = yaml.safe_load(pp.out.getvalue()) code = generate_code(pref_list) except (IOError, ValueError) as e: print('{}: error:\n {}\n'.format(pref_list_filename, e)) sys.exit(1) # When generating multiple files from a script, the build system treats the # first named output file (StaticPrefListAll.h in this case) specially -- it # is created elsewhere, and written to via `fd`. fd.write(code['static_pref_list_all_h']) # We must create the remaining output files ourselves. This requires # creating the output directory directly if it doesn't already exist. ensureParentDir(fd.name) init_dirname = os.path.dirname(fd.name) with FileAvoidWrite('StaticPrefsAll.h') as fd: fd.write(code['static_prefs_all_h']) for group, text in sorted(code['static_pref_list_group_h'].items()): filename = 'StaticPrefList_{}.h'.format(group) with FileAvoidWrite(os.path.join(init_dirname, filename)) as fd: fd.write(text) for group, text in sorted(code['static_prefs_group_h'].items()): filename = 'StaticPrefs_{}.h'.format(group) with FileAvoidWrite(filename) as fd: fd.write(text) with FileAvoidWrite(os.path.join(init_dirname, 'StaticPrefsCGetters.cpp')) as fd: fd.write(code['static_prefs_c_getters_cpp']) with FileAvoidWrite('static_prefs.rs') as fd: fd.write(code['static_prefs_rs'])
def preprocess_file(src, dst, version, app_buildid, update_url): ensureParentDir(dst) defines = { "ADDON_ID": "fxos_" + version.replace(".", "_") + "*****@*****.**", # (reduce the app build id to only the build date # as addon manager doesn't handle big ints in addon versions) "ADDON_VERSION": ("%s.%s" % (version, app_buildid[:8])), "ADDON_NAME": "Firefox OS " + version + " Simulator", "ADDON_DESCRIPTION": "a Firefox OS " + version + " simulator", "ADDON_UPDATE_URL": update_url } pp = Preprocessor(defines=defines) pp.do_filter("substitution") with open(dst, "w") as output: with open(src, "r") as input: pp.processFile(input=input, output=output)
def _get_preprocessor(self, output, extra): '''Returns a preprocessor for use by create_config_file and create_makefile. ''' path = output.name pp = Preprocessor() pp.context.update(self.substs) pp.context.update(top_srcdir=self.get_top_srcdir(path)) pp.context.update(srcdir=self.get_file_srcdir(path)) pp.context.update(relativesrcdir=self.get_relative_srcdir(path)) pp.context.update(DEPTH=self.get_depth(path)) if extra: pp.context.update(extra) pp.do_filter('attemptSubstitution') pp.setMarker(None) pp.out = output return pp
def _get_preprocessor(self, output, extra): '''Returns a preprocessor for use by create_config_file and create_makefile. ''' path = output.name pp = Preprocessor() pp.context.update(self.substs) pp.context.update(top_srcdir = self.get_top_srcdir(path)) pp.context.update(srcdir = self.get_file_srcdir(path)) pp.context.update(relativesrcdir = self.get_relative_srcdir(path)) pp.context.update(DEPTH = self.get_depth(path)) if extra: pp.context.update(extra) pp.do_filter('attemptSubstitution') pp.setMarker(None) pp.out = output return pp
def main(): parser = argparse.ArgumentParser( description="Find duplicate files in directory.") parser.add_argument( "--warning", "-w", action="store_true", help="Only warn about duplicates, do not exit with an error", ) parser.add_argument( "--file", "-f", action="append", dest="dupes_files", default=[], help="Add exceptions to the duplicate list from this file", ) parser.add_argument("-D", action=DefinesAction) parser.add_argument("-U", action="append", default=[]) parser.add_argument("directory", help="The directory to check for duplicates in") args = parser.parse_args() allowed_dupes = [] for filename in args.dupes_files: pp = Preprocessor() pp.context.update(buildconfig.defines["ALLDEFINES"]) if args.D: pp.context.update(args.D) for undefine in args.U: if undefine in pp.context: del pp.context[undefine] pp.out = StringIO() pp.do_filter("substitution") pp.do_include(filename) allowed_dupes.extend([ line.partition("#")[0].rstrip() for line in pp.out.getvalue().splitlines() ]) find_dupes(args.directory, bail=not args.warning, allowed_dupes=allowed_dupes)
def load_yaml(yaml_path): # First invoke preprocessor.py so that we can use #ifdef JS_SIMULATOR in # the YAML file. pp = Preprocessor() pp.context.update(buildconfig.defines['ALLDEFINES']) pp.out = six.StringIO() pp.do_filter('substitution') pp.do_include(yaml_path) contents = pp.out.getvalue() # Load into an OrderedDict to ensure order is preserved. Note: Python 3.7+ # also preserves ordering for normal dictionaries. # Code based on https://stackoverflow.com/a/21912744. class OrderedLoader(yaml.Loader): pass def construct_mapping(loader, node): loader.flatten_mapping(node) return OrderedDict(loader.construct_pairs(node)) tag = yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG OrderedLoader.add_constructor(tag, construct_mapping) return yaml.load(contents, OrderedLoader)
def generate_symbols_file(output, *args): """ """ parser = argparse.ArgumentParser() parser.add_argument("input") parser.add_argument("-D", action=DefinesAction) parser.add_argument("-U", action="append", default=[]) args = parser.parse_args(args) input = os.path.abspath(args.input) pp = Preprocessor() pp.context.update(buildconfig.defines["ALLDEFINES"]) if args.D: pp.context.update(args.D) for undefine in args.U: if undefine in pp.context: del pp.context[undefine] # Hack until MOZ_DEBUG_FLAGS are simply part of buildconfig.defines if buildconfig.substs.get("MOZ_DEBUG"): pp.context["DEBUG"] = "1" # Ensure @DATA@ works as expected (see the Windows section further below) if buildconfig.substs["OS_TARGET"] == "WINNT": pp.context["DATA"] = "DATA" else: pp.context["DATA"] = "" pp.out = StringIO() pp.do_filter("substitution") pp.do_include(input) symbols = [s.strip() for s in pp.out.getvalue().splitlines() if s.strip()] libname, ext = os.path.splitext(os.path.basename(output.name)) if buildconfig.substs["OS_TARGET"] == "WINNT": # A def file is generated for MSVC link.exe that looks like the # following: # LIBRARY library.dll # EXPORTS # symbol1 # symbol2 # ... # # link.exe however requires special markers for data symbols, so in # that case the symbols look like: # data_symbol1 DATA # data_symbol2 DATA # ... # # In the input file, this is just annotated with the following syntax: # data_symbol1 @DATA@ # data_symbol2 @DATA@ # ... # The DATA variable is "simply" expanded by the preprocessor, to # nothing on non-Windows, such that we only get the symbol name on # those platforms, and to DATA on Windows, so that the "DATA" part # is, in fact, part of the symbol name as far as the symbols variable # is concerned. assert ext == ".def" output.write("LIBRARY %s\nEXPORTS\n %s\n" % (libname, "\n ".join(symbols))) elif ( buildconfig.substs.get("GCC_USE_GNU_LD") or buildconfig.substs["OS_TARGET"] == "SunOS" ): # A linker version script is generated for GNU LD that looks like the # following: # liblibrary.so { # global: # symbol1; # symbol2; # ... # local: # *; # }; output.write( "%s {\nglobal:\n %s;\nlocal:\n *;\n};" % (libname, ";\n ".join(symbols)) ) elif buildconfig.substs["OS_TARGET"] == "Darwin": # A list of symbols is generated for Apple ld that simply lists all # symbols, with an underscore prefix. output.write("".join("_%s\n" % s for s in symbols)) return set(pp.includes)
def generate_symbols_file(output, *args): ''' ''' parser = argparse.ArgumentParser() parser.add_argument('input') parser.add_argument('-D', action=DefinesAction) parser.add_argument('-U', action='append', default=[]) args = parser.parse_args(args) input = os.path.abspath(args.input) pp = Preprocessor() pp.context.update(buildconfig.defines) if args.D: pp.context.update(args.D) for undefine in args.U: if undefine in pp.context: del pp.context[undefine] # Hack until MOZ_DEBUG_FLAGS are simply part of buildconfig.defines if buildconfig.substs['MOZ_DEBUG']: pp.context['DEBUG'] = '1' # Ensure @DATA@ works as expected (see the Windows section further below) if buildconfig.substs['OS_TARGET'] == 'WINNT': pp.context['DATA'] = 'DATA' else: pp.context['DATA'] = '' pp.out = StringIO() pp.do_filter('substitution') pp.do_include(input) symbols = [s.strip() for s in pp.out.getvalue().splitlines() if s.strip()] if buildconfig.substs['OS_TARGET'] == 'WINNT': # A def file is generated for MSVC link.exe that looks like the # following: # LIBRARY library.dll # EXPORTS # symbol1 # symbol2 # ... # # link.exe however requires special markers for data symbols, so in # that case the symbols look like: # data_symbol1 DATA # data_symbol2 DATA # ... # # In the input file, this is just annotated with the following syntax: # data_symbol1 @DATA@ # data_symbol2 @DATA@ # ... # The DATA variable is "simply" expanded by the preprocessor, to # nothing on non-Windows, such that we only get the symbol name on # those platforms, and to DATA on Windows, so that the "DATA" part # is, in fact, part of the symbol name as far as the symbols variable # is concerned. libname, ext = os.path.splitext(os.path.basename(output.name)) assert ext == '.def' output.write('LIBRARY %s\nEXPORTS\n %s\n' % (libname, '\n '.join(symbols))) elif buildconfig.substs['GCC_USE_GNU_LD']: # A linker version script is generated for GNU LD that looks like the # following: # { # global: # symbol1; # symbol2; # ... # local: # *; # }; output.write('{\nglobal:\n %s;\nlocal:\n *;\n};' % ';\n '.join(symbols)) elif buildconfig.substs['OS_TARGET'] == 'Darwin': # A list of symbols is generated for Apple ld that simply lists all # symbols, with an underscore prefix. output.write(''.join('_%s\n' % s for s in symbols)) return set(pp.includes)
def generate_symbols_file(output, *args): ''' ''' parser = argparse.ArgumentParser() parser.add_argument('input') parser.add_argument('-D', action=DefinesAction) parser.add_argument('-U', action='append', default=[]) args = parser.parse_args(args) input = os.path.abspath(args.input) pp = Preprocessor() pp.context.update(buildconfig.defines['ALLDEFINES']) if args.D: pp.context.update(args.D) for undefine in args.U: if undefine in pp.context: del pp.context[undefine] # Hack until MOZ_DEBUG_FLAGS are simply part of buildconfig.defines if buildconfig.substs.get('MOZ_DEBUG'): pp.context['DEBUG'] = '1' # Ensure @DATA@ works as expected (see the Windows section further below) if buildconfig.substs['OS_TARGET'] == 'WINNT': pp.context['DATA'] = 'DATA' else: pp.context['DATA'] = '' pp.out = StringIO() pp.do_filter('substitution') pp.do_include(input) symbols = [s.strip() for s in pp.out.getvalue().splitlines() if s.strip()] if buildconfig.substs['OS_TARGET'] == 'WINNT': # A def file is generated for MSVC link.exe that looks like the # following: # LIBRARY library.dll # EXPORTS # symbol1 # symbol2 # ... # # link.exe however requires special markers for data symbols, so in # that case the symbols look like: # data_symbol1 DATA # data_symbol2 DATA # ... # # In the input file, this is just annotated with the following syntax: # data_symbol1 @DATA@ # data_symbol2 @DATA@ # ... # The DATA variable is "simply" expanded by the preprocessor, to # nothing on non-Windows, such that we only get the symbol name on # those platforms, and to DATA on Windows, so that the "DATA" part # is, in fact, part of the symbol name as far as the symbols variable # is concerned. libname, ext = os.path.splitext(os.path.basename(output.name)) assert ext == '.def' output.write('LIBRARY %s\nEXPORTS\n %s\n' % (libname, '\n '.join(symbols))) elif buildconfig.substs['GCC_USE_GNU_LD']: # A linker version script is generated for GNU LD that looks like the # following: # { # global: # symbol1; # symbol2; # ... # local: # *; # }; output.write('{\nglobal:\n %s;\nlocal:\n *;\n};' % ';\n '.join(symbols)) elif buildconfig.substs['OS_TARGET'] == 'Darwin': # A list of symbols is generated for Apple ld that simply lists all # symbols, with an underscore prefix. output.write(''.join('_%s\n' % s for s in symbols)) return set(pp.includes)