def main(output, input_file):
pp = Preprocessor()
pp.context.update({
'VERSION': 'xul%s' % buildconfig.substs['MOZILLA_SYMBOLVERSION'],
})
pp.out = output
pp.do_include(input_file)
def main(output, input_file):
pp = Preprocessor()
pp.context.update(buildconfig.defines['ALLDEFINES'])
substs = buildconfig.substs
# Substs taken verbatim.
substs_vars = (
'BIN_SUFFIX',
)
for var in substs_vars:
pp.context[var] = '"%s"' % substs[var]
# Derived values.
for key, condition in (
('IS_MAC', substs['OS_ARCH'] == 'Darwin'),
('IS_LINUX', substs['OS_ARCH'] == 'Linux'),
('IS_TEST_BUILD', substs.get('ENABLE_TESTS') == '1'),
('IS_DEBUG_BUILD', substs.get('MOZ_DEBUG') == '1'),
('CRASHREPORTER', substs.get('MOZ_CRASHREPORTER')),
('IS_ASAN', substs.get('MOZ_ASAN'))):
if condition:
pp.context[key] = '1'
else:
pp.context[key] = '0'
pp.context.update({
'XPC_BIN_PATH': '"%s/dist/bin"' % buildconfig.topobjdir,
'CERTS_SRC_DIR': '"%s/build/pgo/certs"' % buildconfig.topsrcdir,
})
pp.out = output
pp.do_include(input_file)
def main():
parser = argparse.ArgumentParser(description='Find duplicate files in directory.')
parser.add_argument('--warning', '-w', action='store_true',
help='Only warn about duplicates, do not exit with an error')
parser.add_argument('--file', '-f', action='append', dest='dupes_files', default=[],
help='Add exceptions to the duplicate list from this file')
parser.add_argument('-D', action=DefinesAction)
parser.add_argument('-U', action='append', default=[])
parser.add_argument('directory',
help='The directory to check for duplicates in')
args = parser.parse_args()
allowed_dupes = []
for filename in args.dupes_files:
pp = Preprocessor()
pp.context.update(buildconfig.defines['ALLDEFINES'])
if args.D:
pp.context.update(args.D)
for undefine in args.U:
if undefine in pp.context:
del pp.context[undefine]
pp.out = StringIO()
pp.do_filter('substitution')
pp.do_include(filename)
allowed_dupes.extend([line.partition('#')[0].rstrip()
for line in pp.out.getvalue().splitlines()])
find_dupes(args.directory, bail=not args.warning, allowed_dupes=allowed_dupes)
def main(output, input_file, version):
pp = Preprocessor()
pp.context.update({
'VERSION': version,
})
pp.out = output
pp.do_include(input_file)
def preprocess(input, parser, defines={}):
'''
Preprocess the file-like input with the given defines, and send the
preprocessed output line by line to the given parser.
'''
pp = Preprocessor()
pp.context.update(defines)
pp.do_filter('substitution')
pp.out = PreprocessorOutputWrapper(pp, parser)
pp.do_include(input)
def main(output, input_file):
pp = Preprocessor()
pp.context.update({
'FFI_EXEC_TRAMPOLINE_TABLE': '0',
'HAVE_LONG_DOUBLE': '0',
'TARGET': buildconfig.substs['FFI_TARGET'],
'VERSION': '',
})
pp.do_filter('substitution')
pp.setMarker(None)
pp.out = output
pp.do_include(input_file)
def main(output, input_file):
pp = Preprocessor()
pp.context.update(
{
"FFI_EXEC_TRAMPOLINE_TABLE": "0",
"HAVE_LONG_DOUBLE": "0",
"TARGET": buildconfig.substs["FFI_TARGET"],
"VERSION": "",
}
)
pp.do_filter("substitution")
pp.setMarker(None)
pp.out = output
pp.do_include(input_file)
def _get_preprocessor(self, output, extra):
'''Returns a preprocessor for use by create_config_file and
create_makefile.
'''
path = output.name
pp = Preprocessor()
pp.context.update(self.substs)
pp.context.update(top_srcdir = self.get_top_srcdir(path))
pp.context.update(srcdir = self.get_file_srcdir(path))
pp.context.update(relativesrcdir = self.get_relative_srcdir(path))
pp.context.update(DEPTH = self.get_depth(path))
if extra:
pp.context.update(extra)
pp.do_filter('attemptSubstitution')
pp.setMarker(None)
pp.out = output
return pp
def _consume_jar_manifest(self, obj, defines):
# Ideally, this would all be handled somehow in the emitter, but
# this would require all the magic surrounding l10n and addons in
# the recursive make backend to die, which is not going to happen
# any time soon enough.
# Notably missing:
# - DEFINES from config/config.mk
# - L10n support
# - The equivalent of -e when USE_EXTENSION_MANIFEST is set in
# moz.build, but it doesn't matter in dist/bin.
pp = Preprocessor()
pp.context.update(defines)
pp.context.update(self.environment.defines)
pp.context.update(
AB_CD='en-US',
BUILD_FASTER=1,
)
pp.out = JarManifestParser()
pp.do_include(obj.path)
self.backend_input_files |= pp.includes
for jarinfo in pp.out:
install_target = obj.install_target
if jarinfo.base:
install_target = mozpath.normpath(
mozpath.join(install_target, jarinfo.base))
for e in jarinfo.entries:
if e.is_locale:
if jarinfo.relativesrcdir:
path = mozpath.join(self.environment.topsrcdir,
jarinfo.relativesrcdir)
else:
path = mozpath.dirname(obj.path)
src = mozpath.join( path, 'en-US', e.source)
elif e.source.startswith('/'):
src = mozpath.join(self.environment.topsrcdir,
e.source[1:])
else:
src = mozpath.join(mozpath.dirname(obj.path), e.source)
if '*' in e.source:
if e.preprocess:
raise Exception('%s: Wildcards are not supported with '
'preprocessing' % obj.path)
def _prefix(s):
for p in s.split('/'):
if '*' not in p:
yield p + '/'
prefix = ''.join(_prefix(src))
self._install_manifests[install_target] \
.add_pattern_symlink(
prefix,
src[len(prefix):],
mozpath.join(jarinfo.name, e.output))
continue
if not os.path.exists(src):
if e.is_locale:
raise Exception(
'%s: Cannot find %s' % (obj.path, e.source))
if e.source.startswith('/'):
src = mozpath.join(self.environment.topobjdir,
e.source[1:])
else:
# This actually gets awkward if the jar.mn is not
# in the same directory as the moz.build declaring
# it, but it's how it works in the recursive make,
# not that anything relies on that, but it's simpler.
src = mozpath.join(obj.objdir, e.source)
self._dependencies['install-%s' % install_target] \
.append(mozpath.relpath(
src, self.environment.topobjdir))
if e.preprocess:
kwargs = {}
if src.endswith('.css'):
kwargs['marker'] = '%'
self._add_preprocess(
obj,
src,
mozpath.join(jarinfo.name, mozpath.dirname(e.output)),
mozpath.basename(e.output),
defines=defines,
**kwargs)
else:
self._install_manifests[install_target].add_symlink(
src,
mozpath.join(jarinfo.name, e.output))
manifest = mozpath.normpath(mozpath.join(install_target,
jarinfo.name))
manifest += '.manifest'
for m in jarinfo.chrome_manifests:
self._manifest_entries[manifest].add(
m.replace('%', mozpath.basename(jarinfo.name) + '/'))
if jarinfo.name != 'chrome':
manifest = mozpath.normpath(mozpath.join(install_target,
'chrome.manifest'))
entry = 'manifest %s.manifest' % jarinfo.name
self._manifest_entries[manifest].add(entry)
def _consume_jar_manifest(self, obj):
# Ideally, this would all be handled somehow in the emitter, but
# this would require all the magic surrounding l10n and addons in
# the recursive make backend to die, which is not going to happen
# any time soon enough.
# Notably missing:
# - DEFINES from config/config.mk
# - L10n support
# - The equivalent of -e when USE_EXTENSION_MANIFEST is set in
# moz.build, but it doesn't matter in dist/bin.
pp = Preprocessor()
if obj.defines:
pp.context.update(obj.defines.defines)
pp.context.update(self.environment.defines)
pp.context.update(
AB_CD='en-US',
BUILD_FASTER=1,
)
pp.out = JarManifestParser()
try:
pp.do_include(obj.path.full_path)
except DeprecatedJarManifest as e:
raise DeprecatedJarManifest('Parsing error while processing %s: %s'
% (obj.path.full_path, e.message))
self.backend_input_files |= pp.includes
for jarinfo in pp.out:
jar_context = Context(
allowed_variables=VARIABLES, config=obj._context.config)
jar_context.push_source(obj._context.main_path)
jar_context.push_source(obj.path.full_path)
install_target = obj.install_target
if jarinfo.base:
install_target = mozpath.normpath(
mozpath.join(install_target, jarinfo.base))
jar_context['FINAL_TARGET'] = install_target
if obj.defines:
jar_context['DEFINES'] = obj.defines.defines
files = jar_context['FINAL_TARGET_FILES']
files_pp = jar_context['FINAL_TARGET_PP_FILES']
for e in jarinfo.entries:
if e.is_locale:
if jarinfo.relativesrcdir:
src = '/%s' % jarinfo.relativesrcdir
else:
src = ''
src = mozpath.join(src, 'en-US', e.source)
else:
src = e.source
src = Path(jar_context, src)
if '*' not in e.source and not os.path.exists(src.full_path):
if e.is_locale:
raise Exception(
'%s: Cannot find %s' % (obj.path, e.source))
if e.source.startswith('/'):
src = Path(jar_context, '!' + e.source)
else:
# This actually gets awkward if the jar.mn is not
# in the same directory as the moz.build declaring
# it, but it's how it works in the recursive make,
# not that anything relies on that, but it's simpler.
src = Path(obj._context, '!' + e.source)
output_basename = mozpath.basename(e.output)
if output_basename != src.target_basename:
src = RenamedSourcePath(jar_context,
(src, output_basename))
path = mozpath.dirname(mozpath.join(jarinfo.name, e.output))
if e.preprocess:
if '*' in e.source:
raise Exception('%s: Wildcards are not supported with '
'preprocessing' % obj.path)
files_pp[path] += [src]
else:
files[path] += [src]
if files:
self.consume_object(FinalTargetFiles(jar_context, files))
if files_pp:
self.consume_object(
FinalTargetPreprocessedFiles(jar_context, files_pp))
for m in jarinfo.chrome_manifests:
entry = parse_manifest_line(
mozpath.dirname(jarinfo.name),
m.replace('%', mozpath.basename(jarinfo.name) + '/'))
self.consume_object(ChromeManifestEntry(
jar_context, '%s.manifest' % jarinfo.name, entry))
def _consume_jar_manifest(self, obj):
# Ideally, this would all be handled somehow in the emitter, but
# this would require all the magic surrounding l10n and addons in
# the recursive make backend to die, which is not going to happen
# any time soon enough.
# Notably missing:
# - DEFINES from config/config.mk
# - L10n support
# - The equivalent of -e when USE_EXTENSION_MANIFEST is set in
# moz.build, but it doesn't matter in dist/bin.
pp = Preprocessor()
if obj.defines:
pp.context.update(obj.defines.defines)
pp.context.update(self.environment.defines)
pp.context.update(
AB_CD='en-US',
BUILD_FASTER=1,
)
pp.out = JarManifestParser()
try:
pp.do_include(obj.path.full_path)
except DeprecatedJarManifest as e:
raise DeprecatedJarManifest(
'Parsing error while processing %s: %s' %
(obj.path.full_path, e.message))
self.backend_input_files |= pp.includes
for jarinfo in pp.out:
jar_context = Context(allowed_variables=VARIABLES,
config=obj._context.config)
jar_context.push_source(obj._context.main_path)
jar_context.push_source(obj.path.full_path)
install_target = obj.install_target
if jarinfo.base:
install_target = mozpath.normpath(
mozpath.join(install_target, jarinfo.base))
jar_context['FINAL_TARGET'] = install_target
if obj.defines:
jar_context['DEFINES'] = obj.defines.defines
files = jar_context['FINAL_TARGET_FILES']
files_pp = jar_context['FINAL_TARGET_PP_FILES']
for e in jarinfo.entries:
if e.is_locale:
if jarinfo.relativesrcdir:
src = '/%s' % jarinfo.relativesrcdir
else:
src = ''
src = mozpath.join(src, 'en-US', e.source)
else:
src = e.source
src = Path(jar_context, src)
if '*' not in e.source and not os.path.exists(src.full_path):
if e.is_locale:
raise Exception('%s: Cannot find %s' %
(obj.path, e.source))
if e.source.startswith('/'):
src = Path(jar_context, '!' + e.source)
else:
# This actually gets awkward if the jar.mn is not
# in the same directory as the moz.build declaring
# it, but it's how it works in the recursive make,
# not that anything relies on that, but it's simpler.
src = Path(obj._context, '!' + e.source)
output_basename = mozpath.basename(e.output)
if output_basename != src.target_basename:
src = RenamedSourcePath(jar_context,
(src, output_basename))
path = mozpath.dirname(mozpath.join(jarinfo.name, e.output))
if e.preprocess:
if '*' in e.source:
raise Exception('%s: Wildcards are not supported with '
'preprocessing' % obj.path)
files_pp[path] += [src]
else:
files[path] += [src]
if files:
self.consume_object(FinalTargetFiles(jar_context, files))
if files_pp:
self.consume_object(
FinalTargetPreprocessedFiles(jar_context, files_pp))
for m in jarinfo.chrome_manifests:
entry = parse_manifest_line(
mozpath.dirname(jarinfo.name),
m.replace('%',
mozpath.basename(jarinfo.name) + '/'))
self.consume_object(
ChromeManifestEntry(jar_context,
'%s.manifest' % jarinfo.name, entry))
def _consume_jar_manifest(self, obj, defines):
# Ideally, this would all be handled somehow in the emitter, but
# this would require all the magic surrounding l10n and addons in
# the recursive make backend to die, which is not going to happen
# any time soon enough.
# Notably missing:
# - DEFINES from config/config.mk
# - L10n support
# - The equivalent of -e when USE_EXTENSION_MANIFEST is set in
# moz.build, but it doesn't matter in dist/bin.
pp = Preprocessor()
pp.context.update(defines)
pp.context.update(self.environment.defines)
pp.context.update(
AB_CD='en-US',
BUILD_FASTER=1,
)
pp.out = JarManifestParser()
pp.do_include(obj.path)
for jarinfo in pp.out:
install_target = obj.install_target
# Bug 1150417 added some gross hacks, which we don't try to
# support generically. Fortunately, the hacks don't define more
# than chrome manifest entries, so just assume we don't get
# any installation entries.
if jarinfo.name.startswith('../'):
assert not jarinfo.entries
base = mozpath.join('chrome', jarinfo.name)
for e in jarinfo.entries:
if e.is_locale:
src = mozpath.join(
jarinfo.relativesrcdir or mozpath.dirname(obj.path),
'en-US',
e.source)
elif e.source.startswith('/'):
src = mozpath.join(self.environment.topsrcdir,
e.source[1:])
else:
src = mozpath.join(mozpath.dirname(obj.path), e.source)
if '*' in e.source:
if e.preprocess:
raise Exception('%s: Wildcards are not supported with '
'preprocessing' % obj.path)
def _prefix(s):
for p in s.split('/'):
if '*' not in p:
yield p + '/'
prefix = ''.join(_prefix(src))
self._install_manifests[obj.install_target] \
.add_pattern_symlink(
prefix,
src[len(prefix):],
mozpath.join(base, e.output))
continue
if not os.path.exists(src):
if e.is_locale:
raise Exception(
'%s: Cannot find %s' % (obj.path, e.source))
if e.source.startswith('/'):
src = mozpath.join(self.environment.topobjdir,
e.source[1:])
else:
# This actually gets awkward if the jar.mn is not
# in the same directory as the moz.build declaring
# it, but it's how it works in the recursive make,
# not that anything relies on that, but it's simpler.
src = mozpath.join(obj.objdir, e.source)
self._dependencies['install-%s' % obj.install_target] \
.append(mozpath.relpath(
src, self.environment.topobjdir))
if e.preprocess:
kwargs = {}
if src.endswith('.css'):
kwargs['marker'] = '%'
self._add_preprocess(
obj,
src,
mozpath.join(base, mozpath.dirname(e.output)),
mozpath.basename(e.output),
defines=defines,
**kwargs)
else:
self._install_manifests[obj.install_target].add_symlink(
src,
mozpath.join(base, e.output))
manifest = mozpath.normpath(mozpath.join(obj.install_target, base))
manifest += '.manifest'
for m in jarinfo.chrome_manifests:
self._manifest_entries[manifest].append(
m.replace('%', jarinfo.name + '/'))
# ../ special cased for bug 1150417 again.
if not jarinfo.name.startswith('../'):
manifest = mozpath.normpath(mozpath.join(obj.install_target,
'chrome.manifest'))
entry = 'manifest %s.manifest' % base
if entry not in self._manifest_entries[manifest]:
self._manifest_entries[manifest].append(entry)
def generate(output, *args):
pp = Preprocessor()
pp.out = output
pp.handleCommandLine(list(args), True)
return set(pp.includes)
def generate_symbols_file(output, *args):
""" """
parser = argparse.ArgumentParser()
parser.add_argument("input")
parser.add_argument("-D", action=DefinesAction)
parser.add_argument("-U", action="append", default=[])
args = parser.parse_args(args)
input = os.path.abspath(args.input)
pp = Preprocessor()
pp.context.update(buildconfig.defines["ALLDEFINES"])
if args.D:
pp.context.update(args.D)
for undefine in args.U:
if undefine in pp.context:
del pp.context[undefine]
# Hack until MOZ_DEBUG_FLAGS are simply part of buildconfig.defines
if buildconfig.substs.get("MOZ_DEBUG"):
pp.context["DEBUG"] = "1"
# Ensure @DATA@ works as expected (see the Windows section further below)
if buildconfig.substs["OS_TARGET"] == "WINNT":
pp.context["DATA"] = "DATA"
else:
pp.context["DATA"] = ""
pp.out = StringIO()
pp.do_filter("substitution")
pp.do_include(input)
symbols = [s.strip() for s in pp.out.getvalue().splitlines() if s.strip()]
libname, ext = os.path.splitext(os.path.basename(output.name))
if buildconfig.substs["OS_TARGET"] == "WINNT":
# A def file is generated for MSVC link.exe that looks like the
# following:
# LIBRARY library.dll
# EXPORTS
# symbol1
# symbol2
# ...
#
# link.exe however requires special markers for data symbols, so in
# that case the symbols look like:
# data_symbol1 DATA
# data_symbol2 DATA
# ...
#
# In the input file, this is just annotated with the following syntax:
# data_symbol1 @DATA@
# data_symbol2 @DATA@
# ...
# The DATA variable is "simply" expanded by the preprocessor, to
# nothing on non-Windows, such that we only get the symbol name on
# those platforms, and to DATA on Windows, so that the "DATA" part
# is, in fact, part of the symbol name as far as the symbols variable
# is concerned.
assert ext == ".def"
output.write("LIBRARY %s\nEXPORTS\n %s\n" % (libname, "\n ".join(symbols)))
elif buildconfig.substs.get("GCC_USE_GNU_LD"):
# A linker version script is generated for GNU LD that looks like the
# following:
# liblibrary.so {
# global:
# symbol1;
# symbol2;
# ...
# local:
# *;
# };
output.write(
"%s {\nglobal:\n %s;\nlocal:\n *;\n};" % (libname, ";\n ".join(symbols))
)
elif buildconfig.substs["OS_TARGET"] == "Darwin":
# A list of symbols is generated for Apple ld that simply lists all
# symbols, with an underscore prefix.
output.write("".join("_%s\n" % s for s in symbols))
return set(pp.includes)
def generate_symbols_file(output, *args):
''' '''
parser = argparse.ArgumentParser()
parser.add_argument('input')
parser.add_argument('-D', action=DefinesAction)
parser.add_argument('-U', action='append', default=[])
args = parser.parse_args(args)
input = os.path.abspath(args.input)
pp = Preprocessor()
pp.context.update(buildconfig.defines)
if args.D:
pp.context.update(args.D)
for undefine in args.U:
if undefine in pp.context:
del pp.context[undefine]
# Hack until MOZ_DEBUG_FLAGS are simply part of buildconfig.defines
if buildconfig.substs['MOZ_DEBUG']:
pp.context['DEBUG'] = '1'
# Ensure @DATA@ works as expected (see the Windows section further below)
if buildconfig.substs['OS_TARGET'] == 'WINNT':
pp.context['DATA'] = 'DATA'
else:
pp.context['DATA'] = ''
pp.out = StringIO()
pp.do_filter('substitution')
pp.do_include(input)
symbols = [s.strip() for s in pp.out.getvalue().splitlines() if s.strip()]
if buildconfig.substs['OS_TARGET'] == 'WINNT':
# A def file is generated for MSVC link.exe that looks like the
# following:
# LIBRARY library.dll
# EXPORTS
# symbol1
# symbol2
# ...
#
# link.exe however requires special markers for data symbols, so in
# that case the symbols look like:
# data_symbol1 DATA
# data_symbol2 DATA
# ...
#
# In the input file, this is just annotated with the following syntax:
# data_symbol1 @DATA@
# data_symbol2 @DATA@
# ...
# The DATA variable is "simply" expanded by the preprocessor, to
# nothing on non-Windows, such that we only get the symbol name on
# those platforms, and to DATA on Windows, so that the "DATA" part
# is, in fact, part of the symbol name as far as the symbols variable
# is concerned.
libname, ext = os.path.splitext(os.path.basename(output.name))
assert ext == '.def'
output.write('LIBRARY %s\nEXPORTS\n %s\n'
% (libname, '\n '.join(symbols)))
elif buildconfig.substs['GCC_USE_GNU_LD']:
# A linker version script is generated for GNU LD that looks like the
# following:
# {
# global:
# symbol1;
# symbol2;
# ...
# local:
# *;
# };
output.write('{\nglobal:\n %s;\nlocal:\n *;\n};'
% ';\n '.join(symbols))
elif buildconfig.substs['OS_TARGET'] == 'Darwin':
# A list of symbols is generated for Apple ld that simply lists all
# symbols, with an underscore prefix.
output.write(''.join('_%s\n' % s for s in symbols))
return set(pp.includes)