def main():
parser = argparse.ArgumentParser(description='Find duplicate files in directory.')
parser.add_argument('--warning', '-w', action='store_true',
help='Only warn about duplicates, do not exit with an error')
parser.add_argument('--file', '-f', action='append', dest='dupes_files', default=[],
help='Add exceptions to the duplicate list from this file')
parser.add_argument('-D', action=DefinesAction)
parser.add_argument('-U', action='append', default=[])
parser.add_argument('directory',
help='The directory to check for duplicates in')
args = parser.parse_args()
allowed_dupes = []
for filename in args.dupes_files:
pp = Preprocessor()
pp.context.update(buildconfig.defines['ALLDEFINES'])
if args.D:
pp.context.update(args.D)
for undefine in args.U:
if undefine in pp.context:
del pp.context[undefine]
pp.out = StringIO()
pp.do_filter('substitution')
pp.do_include(filename)
allowed_dupes.extend([line.partition('#')[0].rstrip()
for line in pp.out.getvalue().splitlines()])
find_dupes(args.directory, bail=not args.warning, allowed_dupes=allowed_dupes)
def preprocess(path, defines):
pp = Preprocessor(defines=defines, marker='%')
pp.context.update(defines)
pp.out = io.StringIO()
pp.do_filter('substitution')
pp.do_include(io.open(path, 'r', encoding='latin1'))
pp.out.seek(0)
return pp.out
def preprocess(path, defines):
pp = Preprocessor(defines=defines, marker="%")
pp.context.update(defines)
pp.out = io.StringIO()
pp.do_filter("substitution")
pp.do_include(io.open(path, "r", encoding="latin1"))
pp.out.seek(0)
return pp.out
def preprocess(input, parser, defines={}):
'''
Preprocess the file-like input with the given defines, and send the
preprocessed output line by line to the given parser.
'''
pp = Preprocessor()
pp.context.update(defines)
pp.do_filter('substitution')
pp.out = PreprocessorOutputWrapper(pp, parser)
pp.do_include(input)
def preprocess(input, parser, defines={}):
'''
Preprocess the file-like input with the given defines, and send the
preprocessed output line by line to the given parser.
'''
pp = Preprocessor()
pp.context.update(defines)
pp.do_filter('substitution')
pp.out = PreprocessorOutputWrapper(pp, parser)
pp.do_include(input)
def emit_code(fd, pref_list_filename):
pp = Preprocessor()
pp.context.update(buildconfig.defines["ALLDEFINES"])
# A necessary hack until MOZ_DEBUG_FLAGS are part of buildconfig.defines.
if buildconfig.substs.get("MOZ_DEBUG"):
pp.context["DEBUG"] = "1"
if buildconfig.substs.get("CPU_ARCH") == "aarch64":
pp.context["MOZ_AARCH64"] = True
pp.out = StringIO()
pp.do_filter("substitution")
pp.do_include(pref_list_filename)
try:
pref_list = yaml.safe_load(pp.out.getvalue())
input_file = os.path.relpath(
pref_list_filename,
os.environ.get("GECKO_PATH", os.environ.get("TOPSRCDIR")),
)
code = generate_code(pref_list, input_file)
except (IOError, ValueError) as e:
print("{}: error:\n {}\n".format(pref_list_filename, e))
sys.exit(1)
# When generating multiple files from a script, the build system treats the
# first named output file (StaticPrefListAll.h in this case) specially -- it
# is created elsewhere, and written to via `fd`.
fd.write(code["static_pref_list_all_h"])
# We must create the remaining output files ourselves. This requires
# creating the output directory directly if it doesn't already exist.
ensureParentDir(fd.name)
init_dirname = os.path.dirname(fd.name)
with FileAvoidWrite("StaticPrefsAll.h") as fd:
fd.write(code["static_prefs_all_h"])
for group, text in sorted(code["static_pref_list_group_h"].items()):
filename = "StaticPrefList_{}.h".format(group)
with FileAvoidWrite(os.path.join(init_dirname, filename)) as fd:
fd.write(text)
for group, text in sorted(code["static_prefs_group_h"].items()):
filename = "StaticPrefs_{}.h".format(group)
with FileAvoidWrite(filename) as fd:
fd.write(text)
with FileAvoidWrite(os.path.join(init_dirname,
"StaticPrefsCGetters.cpp")) as fd:
fd.write(code["static_prefs_c_getters_cpp"])
with FileAvoidWrite("static_prefs.rs") as fd:
fd.write(code["static_prefs_rs"])
def main(output, input_file):
pp = Preprocessor()
pp.context.update({
'FFI_EXEC_TRAMPOLINE_TABLE': '0',
'HAVE_LONG_DOUBLE': '0',
'TARGET': buildconfig.substs['FFI_TARGET'],
'VERSION': '',
})
pp.do_filter('substitution')
pp.setMarker(None)
pp.out = output
pp.do_include(input_file)
def main(output, input_file):
pp = Preprocessor()
pp.context.update({
'FFI_EXEC_TRAMPOLINE_TABLE': '0',
'HAVE_LONG_DOUBLE': '0',
'TARGET': buildconfig.substs['FFI_TARGET'],
'VERSION': '',
})
pp.do_filter('substitution')
pp.setMarker(None)
pp.out = output
pp.do_include(input_file)
def main(output, input_file, *defines):
pp = Preprocessor()
pp.context.update({
"FFI_EXEC_TRAMPOLINE_TABLE": "0",
"HAVE_LONG_DOUBLE": "0",
"TARGET": buildconfig.substs["FFI_TARGET"],
"VERSION": "",
})
for d in defines:
pp.context.update({d: "1"})
pp.do_filter("substitution")
pp.setMarker(None)
pp.out = output
pp.do_include(input_file)
def main(output, input_file):
pp = Preprocessor()
pp.context.update(
{
"FFI_EXEC_TRAMPOLINE_TABLE": "0",
"HAVE_LONG_DOUBLE": "0",
"TARGET": buildconfig.substs["FFI_TARGET"],
"VERSION": "",
}
)
pp.do_filter("substitution")
pp.setMarker(None)
pp.out = output
pp.do_include(input_file)
def rnp_preprocess(tmpl, dest, defines):
"""
Generic preprocessing
:param BinaryIO tmpl: open filehandle (read) input
:param BinaryIO dest: open filehandle (write) output
:param dict defines: result of get_defines()
:return boolean:
"""
pp = Preprocessor()
pp.setMarker("%")
pp.addDefines(defines)
pp.do_filter("substitution")
pp.out = dest
pp.do_include(tmpl, True)
return True
def process_package_overload(src, dst, version, app_buildid):
ensureParentDir(dst)
# First replace numeric version like '1.3'
# Then replace with 'slashed' version like '1_4'
# Finally set the full length addon version like 1.3.20131230
defines = {
"NUM_VERSION": version,
"SLASH_VERSION": version.replace(".", "_"),
"FULL_VERSION": ("%s.%s" % (version, app_buildid))
}
pp = Preprocessor(defines=defines)
pp.do_filter("substitution")
with open(dst, "w") as output:
with open(src, "r") as input:
pp.processFile(input=input, output=output)
def process_package_overload(src, dst, version, app_buildid):
ensureParentDir(dst)
# First replace numeric version like '1.3'
# Then replace with 'slashed' version like '1_4'
# Finally set the full length addon version like 1.3.20131230
defines = {
"NUM_VERSION": version,
"SLASH_VERSION": version.replace(".", "_"),
"FULL_VERSION": ("%s.%s" % (version, app_buildid))
}
pp = Preprocessor(defines=defines)
pp.do_filter("substitution")
with open(dst, "w") as output:
with open(src, "r") as input:
pp.processFile(input=input, output=output)
def emit_code(fd, pref_list_filename):
pp = Preprocessor()
pp.context.update(buildconfig.defines['ALLDEFINES'])
# A necessary hack until MOZ_DEBUG_FLAGS are part of buildconfig.defines.
if buildconfig.substs.get('MOZ_DEBUG'):
pp.context['DEBUG'] = '1'
pp.out = BytesIO()
pp.do_filter('substitution')
pp.do_include(pref_list_filename)
try:
pref_list = yaml.safe_load(pp.out.getvalue())
code = generate_code(pref_list)
except (IOError, ValueError) as e:
print('{}: error:\n {}\n'.format(pref_list_filename, e))
sys.exit(1)
# When generating multiple files from a script, the build system treats the
# first named output file (StaticPrefListAll.h in this case) specially -- it
# is created elsewhere, and written to via `fd`.
fd.write(code['static_pref_list_all_h'])
# We must create the remaining output files ourselves. This requires
# creating the output directory directly if it doesn't already exist.
ensureParentDir(fd.name)
init_dirname = os.path.dirname(fd.name)
with FileAvoidWrite('StaticPrefsAll.h') as fd:
fd.write(code['static_prefs_all_h'])
for group, text in sorted(code['static_pref_list_group_h'].items()):
filename = 'StaticPrefList_{}.h'.format(group)
with FileAvoidWrite(os.path.join(init_dirname, filename)) as fd:
fd.write(text)
for group, text in sorted(code['static_prefs_group_h'].items()):
filename = 'StaticPrefs_{}.h'.format(group)
with FileAvoidWrite(filename) as fd:
fd.write(text)
with FileAvoidWrite(os.path.join(init_dirname,
'StaticPrefsCGetters.cpp')) as fd:
fd.write(code['static_prefs_c_getters_cpp'])
with FileAvoidWrite('static_prefs.rs') as fd:
fd.write(code['static_prefs_rs'])
def preprocess_file(src, dst, version, app_buildid, update_url):
ensureParentDir(dst)
defines = {
"ADDON_ID": "fxos_" + version.replace(".", "_") + "*****@*****.**",
# (reduce the app build id to only the build date
# as addon manager doesn't handle big ints in addon versions)
"ADDON_VERSION": ("%s.%s" % (version, app_buildid[:8])),
"ADDON_NAME": "Firefox OS " + version + " Simulator",
"ADDON_DESCRIPTION": "a Firefox OS " + version + " simulator",
"ADDON_UPDATE_URL": update_url
}
pp = Preprocessor(defines=defines)
pp.do_filter("substitution")
with open(dst, "w") as output:
with open(src, "r") as input:
pp.processFile(input=input, output=output)
def preprocess_file(src, dst, version, app_buildid, update_url):
ensureParentDir(dst)
defines = {
"ADDON_ID": "fxos_" + version.replace(".", "_") + "*****@*****.**",
# (reduce the app build id to only the build date
# as addon manager doesn't handle big ints in addon versions)
"ADDON_VERSION": ("%s.%s" % (version, app_buildid[:8])),
"ADDON_NAME": "Firefox OS " + version + " Simulator",
"ADDON_DESCRIPTION": "a Firefox OS " + version + " simulator",
"ADDON_UPDATE_URL": update_url
}
pp = Preprocessor(defines=defines)
pp.do_filter("substitution")
with open(dst, "w") as output:
with open(src, "r") as input:
pp.processFile(input=input, output=output)
def _get_preprocessor(self, output, extra):
'''Returns a preprocessor for use by create_config_file and
create_makefile.
'''
path = output.name
pp = Preprocessor()
pp.context.update(self.substs)
pp.context.update(top_srcdir=self.get_top_srcdir(path))
pp.context.update(srcdir=self.get_file_srcdir(path))
pp.context.update(relativesrcdir=self.get_relative_srcdir(path))
pp.context.update(DEPTH=self.get_depth(path))
if extra:
pp.context.update(extra)
pp.do_filter('attemptSubstitution')
pp.setMarker(None)
pp.out = output
return pp
def _get_preprocessor(self, output, extra):
'''Returns a preprocessor for use by create_config_file and
create_makefile.
'''
path = output.name
pp = Preprocessor()
pp.context.update(self.substs)
pp.context.update(top_srcdir = self.get_top_srcdir(path))
pp.context.update(srcdir = self.get_file_srcdir(path))
pp.context.update(relativesrcdir = self.get_relative_srcdir(path))
pp.context.update(DEPTH = self.get_depth(path))
if extra:
pp.context.update(extra)
pp.do_filter('attemptSubstitution')
pp.setMarker(None)
pp.out = output
return pp
def main():
parser = argparse.ArgumentParser(
description="Find duplicate files in directory.")
parser.add_argument(
"--warning",
"-w",
action="store_true",
help="Only warn about duplicates, do not exit with an error",
)
parser.add_argument(
"--file",
"-f",
action="append",
dest="dupes_files",
default=[],
help="Add exceptions to the duplicate list from this file",
)
parser.add_argument("-D", action=DefinesAction)
parser.add_argument("-U", action="append", default=[])
parser.add_argument("directory",
help="The directory to check for duplicates in")
args = parser.parse_args()
allowed_dupes = []
for filename in args.dupes_files:
pp = Preprocessor()
pp.context.update(buildconfig.defines["ALLDEFINES"])
if args.D:
pp.context.update(args.D)
for undefine in args.U:
if undefine in pp.context:
del pp.context[undefine]
pp.out = StringIO()
pp.do_filter("substitution")
pp.do_include(filename)
allowed_dupes.extend([
line.partition("#")[0].rstrip()
for line in pp.out.getvalue().splitlines()
])
find_dupes(args.directory,
bail=not args.warning,
allowed_dupes=allowed_dupes)
def load_yaml(yaml_path):
# First invoke preprocessor.py so that we can use #ifdef JS_SIMULATOR in
# the YAML file.
pp = Preprocessor()
pp.context.update(buildconfig.defines['ALLDEFINES'])
pp.out = six.StringIO()
pp.do_filter('substitution')
pp.do_include(yaml_path)
contents = pp.out.getvalue()
# Load into an OrderedDict to ensure order is preserved. Note: Python 3.7+
# also preserves ordering for normal dictionaries.
# Code based on https://stackoverflow.com/a/21912744.
class OrderedLoader(yaml.Loader):
pass
def construct_mapping(loader, node):
loader.flatten_mapping(node)
return OrderedDict(loader.construct_pairs(node))
tag = yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG
OrderedLoader.add_constructor(tag, construct_mapping)
return yaml.load(contents, OrderedLoader)
def generate_symbols_file(output, *args):
""" """
parser = argparse.ArgumentParser()
parser.add_argument("input")
parser.add_argument("-D", action=DefinesAction)
parser.add_argument("-U", action="append", default=[])
args = parser.parse_args(args)
input = os.path.abspath(args.input)
pp = Preprocessor()
pp.context.update(buildconfig.defines["ALLDEFINES"])
if args.D:
pp.context.update(args.D)
for undefine in args.U:
if undefine in pp.context:
del pp.context[undefine]
# Hack until MOZ_DEBUG_FLAGS are simply part of buildconfig.defines
if buildconfig.substs.get("MOZ_DEBUG"):
pp.context["DEBUG"] = "1"
# Ensure @DATA@ works as expected (see the Windows section further below)
if buildconfig.substs["OS_TARGET"] == "WINNT":
pp.context["DATA"] = "DATA"
else:
pp.context["DATA"] = ""
pp.out = StringIO()
pp.do_filter("substitution")
pp.do_include(input)
symbols = [s.strip() for s in pp.out.getvalue().splitlines() if s.strip()]
libname, ext = os.path.splitext(os.path.basename(output.name))
if buildconfig.substs["OS_TARGET"] == "WINNT":
# A def file is generated for MSVC link.exe that looks like the
# following:
# LIBRARY library.dll
# EXPORTS
# symbol1
# symbol2
# ...
#
# link.exe however requires special markers for data symbols, so in
# that case the symbols look like:
# data_symbol1 DATA
# data_symbol2 DATA
# ...
#
# In the input file, this is just annotated with the following syntax:
# data_symbol1 @DATA@
# data_symbol2 @DATA@
# ...
# The DATA variable is "simply" expanded by the preprocessor, to
# nothing on non-Windows, such that we only get the symbol name on
# those platforms, and to DATA on Windows, so that the "DATA" part
# is, in fact, part of the symbol name as far as the symbols variable
# is concerned.
assert ext == ".def"
output.write("LIBRARY %s\nEXPORTS\n %s\n" % (libname, "\n ".join(symbols)))
elif (
buildconfig.substs.get("GCC_USE_GNU_LD")
or buildconfig.substs["OS_TARGET"] == "SunOS"
):
# A linker version script is generated for GNU LD that looks like the
# following:
# liblibrary.so {
# global:
# symbol1;
# symbol2;
# ...
# local:
# *;
# };
output.write(
"%s {\nglobal:\n %s;\nlocal:\n *;\n};" % (libname, ";\n ".join(symbols))
)
elif buildconfig.substs["OS_TARGET"] == "Darwin":
# A list of symbols is generated for Apple ld that simply lists all
# symbols, with an underscore prefix.
output.write("".join("_%s\n" % s for s in symbols))
return set(pp.includes)
def generate_symbols_file(output, *args):
''' '''
parser = argparse.ArgumentParser()
parser.add_argument('input')
parser.add_argument('-D', action=DefinesAction)
parser.add_argument('-U', action='append', default=[])
args = parser.parse_args(args)
input = os.path.abspath(args.input)
pp = Preprocessor()
pp.context.update(buildconfig.defines)
if args.D:
pp.context.update(args.D)
for undefine in args.U:
if undefine in pp.context:
del pp.context[undefine]
# Hack until MOZ_DEBUG_FLAGS are simply part of buildconfig.defines
if buildconfig.substs['MOZ_DEBUG']:
pp.context['DEBUG'] = '1'
# Ensure @DATA@ works as expected (see the Windows section further below)
if buildconfig.substs['OS_TARGET'] == 'WINNT':
pp.context['DATA'] = 'DATA'
else:
pp.context['DATA'] = ''
pp.out = StringIO()
pp.do_filter('substitution')
pp.do_include(input)
symbols = [s.strip() for s in pp.out.getvalue().splitlines() if s.strip()]
if buildconfig.substs['OS_TARGET'] == 'WINNT':
# A def file is generated for MSVC link.exe that looks like the
# following:
# LIBRARY library.dll
# EXPORTS
# symbol1
# symbol2
# ...
#
# link.exe however requires special markers for data symbols, so in
# that case the symbols look like:
# data_symbol1 DATA
# data_symbol2 DATA
# ...
#
# In the input file, this is just annotated with the following syntax:
# data_symbol1 @DATA@
# data_symbol2 @DATA@
# ...
# The DATA variable is "simply" expanded by the preprocessor, to
# nothing on non-Windows, such that we only get the symbol name on
# those platforms, and to DATA on Windows, so that the "DATA" part
# is, in fact, part of the symbol name as far as the symbols variable
# is concerned.
libname, ext = os.path.splitext(os.path.basename(output.name))
assert ext == '.def'
output.write('LIBRARY %s\nEXPORTS\n %s\n'
% (libname, '\n '.join(symbols)))
elif buildconfig.substs['GCC_USE_GNU_LD']:
# A linker version script is generated for GNU LD that looks like the
# following:
# {
# global:
# symbol1;
# symbol2;
# ...
# local:
# *;
# };
output.write('{\nglobal:\n %s;\nlocal:\n *;\n};'
% ';\n '.join(symbols))
elif buildconfig.substs['OS_TARGET'] == 'Darwin':
# A list of symbols is generated for Apple ld that simply lists all
# symbols, with an underscore prefix.
output.write(''.join('_%s\n' % s for s in symbols))
return set(pp.includes)
def generate_symbols_file(output, *args):
''' '''
parser = argparse.ArgumentParser()
parser.add_argument('input')
parser.add_argument('-D', action=DefinesAction)
parser.add_argument('-U', action='append', default=[])
args = parser.parse_args(args)
input = os.path.abspath(args.input)
pp = Preprocessor()
pp.context.update(buildconfig.defines['ALLDEFINES'])
if args.D:
pp.context.update(args.D)
for undefine in args.U:
if undefine in pp.context:
del pp.context[undefine]
# Hack until MOZ_DEBUG_FLAGS are simply part of buildconfig.defines
if buildconfig.substs.get('MOZ_DEBUG'):
pp.context['DEBUG'] = '1'
# Ensure @DATA@ works as expected (see the Windows section further below)
if buildconfig.substs['OS_TARGET'] == 'WINNT':
pp.context['DATA'] = 'DATA'
else:
pp.context['DATA'] = ''
pp.out = StringIO()
pp.do_filter('substitution')
pp.do_include(input)
symbols = [s.strip() for s in pp.out.getvalue().splitlines() if s.strip()]
if buildconfig.substs['OS_TARGET'] == 'WINNT':
# A def file is generated for MSVC link.exe that looks like the
# following:
# LIBRARY library.dll
# EXPORTS
# symbol1
# symbol2
# ...
#
# link.exe however requires special markers for data symbols, so in
# that case the symbols look like:
# data_symbol1 DATA
# data_symbol2 DATA
# ...
#
# In the input file, this is just annotated with the following syntax:
# data_symbol1 @DATA@
# data_symbol2 @DATA@
# ...
# The DATA variable is "simply" expanded by the preprocessor, to
# nothing on non-Windows, such that we only get the symbol name on
# those platforms, and to DATA on Windows, so that the "DATA" part
# is, in fact, part of the symbol name as far as the symbols variable
# is concerned.
libname, ext = os.path.splitext(os.path.basename(output.name))
assert ext == '.def'
output.write('LIBRARY %s\nEXPORTS\n %s\n' %
(libname, '\n '.join(symbols)))
elif buildconfig.substs['GCC_USE_GNU_LD']:
# A linker version script is generated for GNU LD that looks like the
# following:
# {
# global:
# symbol1;
# symbol2;
# ...
# local:
# *;
# };
output.write('{\nglobal:\n %s;\nlocal:\n *;\n};' %
';\n '.join(symbols))
elif buildconfig.substs['OS_TARGET'] == 'Darwin':
# A list of symbols is generated for Apple ld that simply lists all
# symbols, with an underscore prefix.
output.write(''.join('_%s\n' % s for s in symbols))
return set(pp.includes)
