Exemplo n.º 1
0
def main(output, input_file):
    pp = Preprocessor()
    pp.context.update({
        'VERSION': 'xul%s' % buildconfig.substs['MOZILLA_SYMBOLVERSION'],
    })
    pp.out = output
    pp.do_include(input_file)
Exemplo n.º 2
0
def main(output, input_file):
    pp = Preprocessor()
    pp.context.update(buildconfig.defines['ALLDEFINES'])

    substs = buildconfig.substs

    # Substs taken verbatim.
    substs_vars = (
        'BIN_SUFFIX',
    )
    for var in substs_vars:
        pp.context[var] = '"%s"' % substs[var]

    # Derived values.
    for key, condition in (
            ('IS_MAC', substs['OS_ARCH'] == 'Darwin'),
            ('IS_LINUX', substs['OS_ARCH'] == 'Linux'),
            ('IS_TEST_BUILD', substs.get('ENABLE_TESTS') == '1'),
            ('IS_DEBUG_BUILD', substs.get('MOZ_DEBUG') == '1'),
            ('CRASHREPORTER', substs.get('MOZ_CRASHREPORTER')),
            ('IS_ASAN', substs.get('MOZ_ASAN'))):
        if condition:
            pp.context[key] = '1'
        else:
            pp.context[key] = '0'

    pp.context.update({
        'XPC_BIN_PATH': '"%s/dist/bin"' % buildconfig.topobjdir,
        'CERTS_SRC_DIR': '"%s/build/pgo/certs"' % buildconfig.topsrcdir,
    })

    pp.out = output
    pp.do_include(input_file)
Exemplo n.º 3
0
def main():
    parser = argparse.ArgumentParser(description='Find duplicate files in directory.')
    parser.add_argument('--warning', '-w', action='store_true',
                        help='Only warn about duplicates, do not exit with an error')
    parser.add_argument('--file', '-f', action='append', dest='dupes_files', default=[],
                        help='Add exceptions to the duplicate list from this file')
    parser.add_argument('-D', action=DefinesAction)
    parser.add_argument('-U', action='append', default=[])
    parser.add_argument('directory',
                        help='The directory to check for duplicates in')

    args = parser.parse_args()

    allowed_dupes = []
    for filename in args.dupes_files:
        pp = Preprocessor()
        pp.context.update(buildconfig.defines['ALLDEFINES'])
        if args.D:
            pp.context.update(args.D)
        for undefine in args.U:
            if undefine in pp.context:
                del pp.context[undefine]
        pp.out = StringIO()
        pp.do_filter('substitution')
        pp.do_include(filename)
        allowed_dupes.extend([line.partition('#')[0].rstrip()
                              for line in pp.out.getvalue().splitlines()])

    find_dupes(args.directory, bail=not args.warning, allowed_dupes=allowed_dupes)
Exemplo n.º 4
0
def main(output, input_file, version):
    pp = Preprocessor()
    pp.context.update({
        'VERSION': version,
    })
    pp.out = output
    pp.do_include(input_file)
Exemplo n.º 5
0
def preprocess(input, parser, defines={}):
    '''
    Preprocess the file-like input with the given defines, and send the
    preprocessed output line by line to the given parser.
    '''
    pp = Preprocessor()
    pp.context.update(defines)
    pp.do_filter('substitution')
    pp.out = PreprocessorOutputWrapper(pp, parser)
    pp.do_include(input)
Exemplo n.º 6
0
def main(output, input_file):
    pp = Preprocessor()
    pp.context.update({
        'FFI_EXEC_TRAMPOLINE_TABLE': '0',
        'HAVE_LONG_DOUBLE': '0',
        'TARGET': buildconfig.substs['FFI_TARGET'],
        'VERSION': '',
    })
    pp.do_filter('substitution')
    pp.setMarker(None)
    pp.out = output
    pp.do_include(input_file)
Exemplo n.º 7
0
def main(output, input_file):
    pp = Preprocessor()
    pp.context.update(
        {
            "FFI_EXEC_TRAMPOLINE_TABLE": "0",
            "HAVE_LONG_DOUBLE": "0",
            "TARGET": buildconfig.substs["FFI_TARGET"],
            "VERSION": "",
        }
    )
    pp.do_filter("substitution")
    pp.setMarker(None)
    pp.out = output
    pp.do_include(input_file)
Exemplo n.º 8
0
    def _get_preprocessor(self, output, extra):
        '''Returns a preprocessor for use by create_config_file and
        create_makefile.
        '''
        path = output.name
        pp = Preprocessor()
        pp.context.update(self.substs)
        pp.context.update(top_srcdir = self.get_top_srcdir(path))
        pp.context.update(srcdir = self.get_file_srcdir(path))
        pp.context.update(relativesrcdir = self.get_relative_srcdir(path))
        pp.context.update(DEPTH = self.get_depth(path))
        if extra:
            pp.context.update(extra)
        pp.do_filter('attemptSubstitution')
        pp.setMarker(None)

        pp.out = output
        return pp
Exemplo n.º 9
0
    def _consume_jar_manifest(self, obj, defines):
        # Ideally, this would all be handled somehow in the emitter, but
        # this would require all the magic surrounding l10n and addons in
        # the recursive make backend to die, which is not going to happen
        # any time soon enough.
        # Notably missing:
        # - DEFINES from config/config.mk
        # - L10n support
        # - The equivalent of -e when USE_EXTENSION_MANIFEST is set in
        #   moz.build, but it doesn't matter in dist/bin.
        pp = Preprocessor()
        pp.context.update(defines)
        pp.context.update(self.environment.defines)
        pp.context.update(
            AB_CD='en-US',
            BUILD_FASTER=1,
        )
        pp.out = JarManifestParser()
        pp.do_include(obj.path)
        self.backend_input_files |= pp.includes

        for jarinfo in pp.out:
            install_target = obj.install_target
            if jarinfo.base:
                install_target = mozpath.normpath(
                    mozpath.join(install_target, jarinfo.base))
            for e in jarinfo.entries:
                if e.is_locale:
                    if jarinfo.relativesrcdir:
                        path = mozpath.join(self.environment.topsrcdir,
                                            jarinfo.relativesrcdir)
                    else:
                        path = mozpath.dirname(obj.path)
                    src = mozpath.join( path, 'en-US', e.source)
                elif e.source.startswith('/'):
                    src = mozpath.join(self.environment.topsrcdir,
                                       e.source[1:])
                else:
                    src = mozpath.join(mozpath.dirname(obj.path), e.source)

                if '*' in e.source:
                    if e.preprocess:
                        raise Exception('%s: Wildcards are not supported with '
                                        'preprocessing' % obj.path)
                    def _prefix(s):
                        for p in s.split('/'):
                            if '*' not in p:
                                yield p + '/'
                    prefix = ''.join(_prefix(src))

                    self._install_manifests[install_target] \
                        .add_pattern_symlink(
                        prefix,
                        src[len(prefix):],
                        mozpath.join(jarinfo.name, e.output))
                    continue

                if not os.path.exists(src):
                    if e.is_locale:
                        raise Exception(
                            '%s: Cannot find %s' % (obj.path, e.source))
                    if e.source.startswith('/'):
                        src = mozpath.join(self.environment.topobjdir,
                                           e.source[1:])
                    else:
                        # This actually gets awkward if the jar.mn is not
                        # in the same directory as the moz.build declaring
                        # it, but it's how it works in the recursive make,
                        # not that anything relies on that, but it's simpler.
                        src = mozpath.join(obj.objdir, e.source)
                    self._dependencies['install-%s' % install_target] \
                        .append(mozpath.relpath(
                        src, self.environment.topobjdir))

                if e.preprocess:
                    kwargs = {}
                    if src.endswith('.css'):
                        kwargs['marker'] = '%'
                    self._add_preprocess(
                        obj,
                        src,
                        mozpath.join(jarinfo.name, mozpath.dirname(e.output)),
                        mozpath.basename(e.output),
                        defines=defines,
                        **kwargs)
                else:
                    self._install_manifests[install_target].add_symlink(
                        src,
                        mozpath.join(jarinfo.name, e.output))

            manifest = mozpath.normpath(mozpath.join(install_target,
                                                     jarinfo.name))
            manifest += '.manifest'
            for m in jarinfo.chrome_manifests:
                self._manifest_entries[manifest].add(
                    m.replace('%', mozpath.basename(jarinfo.name) + '/'))

            if jarinfo.name != 'chrome':
                manifest = mozpath.normpath(mozpath.join(install_target,
                                                         'chrome.manifest'))
                entry = 'manifest %s.manifest' % jarinfo.name
                self._manifest_entries[manifest].add(entry)
Exemplo n.º 10
0
    def _consume_jar_manifest(self, obj):
        # Ideally, this would all be handled somehow in the emitter, but
        # this would require all the magic surrounding l10n and addons in
        # the recursive make backend to die, which is not going to happen
        # any time soon enough.
        # Notably missing:
        # - DEFINES from config/config.mk
        # - L10n support
        # - The equivalent of -e when USE_EXTENSION_MANIFEST is set in
        #   moz.build, but it doesn't matter in dist/bin.
        pp = Preprocessor()
        if obj.defines:
            pp.context.update(obj.defines.defines)
        pp.context.update(self.environment.defines)
        pp.context.update(
            AB_CD='en-US',
            BUILD_FASTER=1,
        )
        pp.out = JarManifestParser()
        try:
            pp.do_include(obj.path.full_path)
        except DeprecatedJarManifest as e:
            raise DeprecatedJarManifest('Parsing error while processing %s: %s'
                                        % (obj.path.full_path, e.message))
        self.backend_input_files |= pp.includes

        for jarinfo in pp.out:
            jar_context = Context(
                allowed_variables=VARIABLES, config=obj._context.config)
            jar_context.push_source(obj._context.main_path)
            jar_context.push_source(obj.path.full_path)

            install_target = obj.install_target
            if jarinfo.base:
                install_target = mozpath.normpath(
                    mozpath.join(install_target, jarinfo.base))
            jar_context['FINAL_TARGET'] = install_target
            if obj.defines:
                jar_context['DEFINES'] = obj.defines.defines
            files = jar_context['FINAL_TARGET_FILES']
            files_pp = jar_context['FINAL_TARGET_PP_FILES']

            for e in jarinfo.entries:
                if e.is_locale:
                    if jarinfo.relativesrcdir:
                        src = '/%s' % jarinfo.relativesrcdir
                    else:
                        src = ''
                    src = mozpath.join(src, 'en-US', e.source)
                else:
                    src = e.source

                src = Path(jar_context, src)

                if '*' not in e.source and not os.path.exists(src.full_path):
                    if e.is_locale:
                        raise Exception(
                            '%s: Cannot find %s' % (obj.path, e.source))
                    if e.source.startswith('/'):
                        src = Path(jar_context, '!' + e.source)
                    else:
                        # This actually gets awkward if the jar.mn is not
                        # in the same directory as the moz.build declaring
                        # it, but it's how it works in the recursive make,
                        # not that anything relies on that, but it's simpler.
                        src = Path(obj._context, '!' + e.source)

                output_basename = mozpath.basename(e.output)
                if output_basename != src.target_basename:
                    src = RenamedSourcePath(jar_context,
                                            (src, output_basename))
                path = mozpath.dirname(mozpath.join(jarinfo.name, e.output))

                if e.preprocess:
                    if '*' in e.source:
                        raise Exception('%s: Wildcards are not supported with '
                                        'preprocessing' % obj.path)
                    files_pp[path] += [src]
                else:
                    files[path] += [src]

            if files:
                self.consume_object(FinalTargetFiles(jar_context, files))
            if files_pp:
                self.consume_object(
                    FinalTargetPreprocessedFiles(jar_context, files_pp))

            for m in jarinfo.chrome_manifests:
                entry = parse_manifest_line(
                    mozpath.dirname(jarinfo.name),
                    m.replace('%', mozpath.basename(jarinfo.name) + '/'))
                self.consume_object(ChromeManifestEntry(
                    jar_context, '%s.manifest' % jarinfo.name, entry))
Exemplo n.º 11
0
    def _consume_jar_manifest(self, obj):
        # Ideally, this would all be handled somehow in the emitter, but
        # this would require all the magic surrounding l10n and addons in
        # the recursive make backend to die, which is not going to happen
        # any time soon enough.
        # Notably missing:
        # - DEFINES from config/config.mk
        # - L10n support
        # - The equivalent of -e when USE_EXTENSION_MANIFEST is set in
        #   moz.build, but it doesn't matter in dist/bin.
        pp = Preprocessor()
        if obj.defines:
            pp.context.update(obj.defines.defines)
        pp.context.update(self.environment.defines)
        pp.context.update(
            AB_CD='en-US',
            BUILD_FASTER=1,
        )
        pp.out = JarManifestParser()
        try:
            pp.do_include(obj.path.full_path)
        except DeprecatedJarManifest as e:
            raise DeprecatedJarManifest(
                'Parsing error while processing %s: %s' %
                (obj.path.full_path, e.message))
        self.backend_input_files |= pp.includes

        for jarinfo in pp.out:
            jar_context = Context(allowed_variables=VARIABLES,
                                  config=obj._context.config)
            jar_context.push_source(obj._context.main_path)
            jar_context.push_source(obj.path.full_path)

            install_target = obj.install_target
            if jarinfo.base:
                install_target = mozpath.normpath(
                    mozpath.join(install_target, jarinfo.base))
            jar_context['FINAL_TARGET'] = install_target
            if obj.defines:
                jar_context['DEFINES'] = obj.defines.defines
            files = jar_context['FINAL_TARGET_FILES']
            files_pp = jar_context['FINAL_TARGET_PP_FILES']

            for e in jarinfo.entries:
                if e.is_locale:
                    if jarinfo.relativesrcdir:
                        src = '/%s' % jarinfo.relativesrcdir
                    else:
                        src = ''
                    src = mozpath.join(src, 'en-US', e.source)
                else:
                    src = e.source

                src = Path(jar_context, src)

                if '*' not in e.source and not os.path.exists(src.full_path):
                    if e.is_locale:
                        raise Exception('%s: Cannot find %s' %
                                        (obj.path, e.source))
                    if e.source.startswith('/'):
                        src = Path(jar_context, '!' + e.source)
                    else:
                        # This actually gets awkward if the jar.mn is not
                        # in the same directory as the moz.build declaring
                        # it, but it's how it works in the recursive make,
                        # not that anything relies on that, but it's simpler.
                        src = Path(obj._context, '!' + e.source)

                output_basename = mozpath.basename(e.output)
                if output_basename != src.target_basename:
                    src = RenamedSourcePath(jar_context,
                                            (src, output_basename))
                path = mozpath.dirname(mozpath.join(jarinfo.name, e.output))

                if e.preprocess:
                    if '*' in e.source:
                        raise Exception('%s: Wildcards are not supported with '
                                        'preprocessing' % obj.path)
                    files_pp[path] += [src]
                else:
                    files[path] += [src]

            if files:
                self.consume_object(FinalTargetFiles(jar_context, files))
            if files_pp:
                self.consume_object(
                    FinalTargetPreprocessedFiles(jar_context, files_pp))

            for m in jarinfo.chrome_manifests:
                entry = parse_manifest_line(
                    mozpath.dirname(jarinfo.name),
                    m.replace('%',
                              mozpath.basename(jarinfo.name) + '/'))
                self.consume_object(
                    ChromeManifestEntry(jar_context,
                                        '%s.manifest' % jarinfo.name, entry))
Exemplo n.º 12
0
    def _consume_jar_manifest(self, obj, defines):
        # Ideally, this would all be handled somehow in the emitter, but
        # this would require all the magic surrounding l10n and addons in
        # the recursive make backend to die, which is not going to happen
        # any time soon enough.
        # Notably missing:
        # - DEFINES from config/config.mk
        # - L10n support
        # - The equivalent of -e when USE_EXTENSION_MANIFEST is set in
        #   moz.build, but it doesn't matter in dist/bin.
        pp = Preprocessor()
        pp.context.update(defines)
        pp.context.update(self.environment.defines)
        pp.context.update(
            AB_CD='en-US',
            BUILD_FASTER=1,
        )
        pp.out = JarManifestParser()
        pp.do_include(obj.path)

        for jarinfo in pp.out:
            install_target = obj.install_target
            # Bug 1150417 added some gross hacks, which we don't try to
            # support generically. Fortunately, the hacks don't define more
            # than chrome manifest entries, so just assume we don't get
            # any installation entries.
            if jarinfo.name.startswith('../'):
                assert not jarinfo.entries

            base = mozpath.join('chrome', jarinfo.name)

            for e in jarinfo.entries:
                if e.is_locale:
                    src = mozpath.join(
                        jarinfo.relativesrcdir or mozpath.dirname(obj.path),
                        'en-US',
                        e.source)
                elif e.source.startswith('/'):
                    src = mozpath.join(self.environment.topsrcdir,
                                       e.source[1:])
                else:
                    src = mozpath.join(mozpath.dirname(obj.path), e.source)

                if '*' in e.source:
                    if e.preprocess:
                        raise Exception('%s: Wildcards are not supported with '
                                        'preprocessing' % obj.path)
                    def _prefix(s):
                        for p in s.split('/'):
                            if '*' not in p:
                                yield p + '/'
                    prefix = ''.join(_prefix(src))

                    self._install_manifests[obj.install_target] \
                        .add_pattern_symlink(
                        prefix,
                        src[len(prefix):],
                        mozpath.join(base, e.output))
                    continue

                if not os.path.exists(src):
                    if e.is_locale:
                        raise Exception(
                            '%s: Cannot find %s' % (obj.path, e.source))
                    if e.source.startswith('/'):
                        src = mozpath.join(self.environment.topobjdir,
                                           e.source[1:])
                    else:
                        # This actually gets awkward if the jar.mn is not
                        # in the same directory as the moz.build declaring
                        # it, but it's how it works in the recursive make,
                        # not that anything relies on that, but it's simpler.
                        src = mozpath.join(obj.objdir, e.source)
                    self._dependencies['install-%s' % obj.install_target] \
                        .append(mozpath.relpath(
                        src, self.environment.topobjdir))

                if e.preprocess:
                    kwargs = {}
                    if src.endswith('.css'):
                        kwargs['marker'] = '%'
                    self._add_preprocess(
                        obj,
                        src,
                        mozpath.join(base, mozpath.dirname(e.output)),
                        mozpath.basename(e.output),
                        defines=defines,
                        **kwargs)
                else:
                    self._install_manifests[obj.install_target].add_symlink(
                        src,
                        mozpath.join(base, e.output))

            manifest = mozpath.normpath(mozpath.join(obj.install_target, base))
            manifest += '.manifest'
            for m in jarinfo.chrome_manifests:
                self._manifest_entries[manifest].append(
                    m.replace('%', jarinfo.name + '/'))

            # ../ special cased for bug 1150417 again.
            if not jarinfo.name.startswith('../'):
                manifest = mozpath.normpath(mozpath.join(obj.install_target,
                                                         'chrome.manifest'))
                entry = 'manifest %s.manifest' % base
                if entry not in self._manifest_entries[manifest]:
                    self._manifest_entries[manifest].append(entry)
Exemplo n.º 13
0
def generate(output, *args):
    pp = Preprocessor()
    pp.out = output
    pp.handleCommandLine(list(args), True)
    return set(pp.includes)
Exemplo n.º 14
0
def generate_symbols_file(output, *args):
    """ """
    parser = argparse.ArgumentParser()
    parser.add_argument("input")
    parser.add_argument("-D", action=DefinesAction)
    parser.add_argument("-U", action="append", default=[])
    args = parser.parse_args(args)
    input = os.path.abspath(args.input)

    pp = Preprocessor()
    pp.context.update(buildconfig.defines["ALLDEFINES"])
    if args.D:
        pp.context.update(args.D)
    for undefine in args.U:
        if undefine in pp.context:
            del pp.context[undefine]
    # Hack until MOZ_DEBUG_FLAGS are simply part of buildconfig.defines
    if buildconfig.substs.get("MOZ_DEBUG"):
        pp.context["DEBUG"] = "1"
    # Ensure @DATA@ works as expected (see the Windows section further below)
    if buildconfig.substs["OS_TARGET"] == "WINNT":
        pp.context["DATA"] = "DATA"
    else:
        pp.context["DATA"] = ""
    pp.out = StringIO()
    pp.do_filter("substitution")
    pp.do_include(input)

    symbols = [s.strip() for s in pp.out.getvalue().splitlines() if s.strip()]

    libname, ext = os.path.splitext(os.path.basename(output.name))

    if buildconfig.substs["OS_TARGET"] == "WINNT":
        # A def file is generated for MSVC link.exe that looks like the
        # following:
        # LIBRARY library.dll
        # EXPORTS
        #   symbol1
        #   symbol2
        #   ...
        #
        # link.exe however requires special markers for data symbols, so in
        # that case the symbols look like:
        #   data_symbol1 DATA
        #   data_symbol2 DATA
        #   ...
        #
        # In the input file, this is just annotated with the following syntax:
        #   data_symbol1 @DATA@
        #   data_symbol2 @DATA@
        #   ...
        # The DATA variable is "simply" expanded by the preprocessor, to
        # nothing on non-Windows, such that we only get the symbol name on
        # those platforms, and to DATA on Windows, so that the "DATA" part
        # is, in fact, part of the symbol name as far as the symbols variable
        # is concerned.
        assert ext == ".def"
        output.write("LIBRARY %s\nEXPORTS\n  %s\n" % (libname, "\n  ".join(symbols)))
    elif buildconfig.substs.get("GCC_USE_GNU_LD"):
        # A linker version script is generated for GNU LD that looks like the
        # following:
        # liblibrary.so {
        # global:
        #   symbol1;
        #   symbol2;
        #   ...
        # local:
        #   *;
        # };
        output.write(
            "%s {\nglobal:\n  %s;\nlocal:\n  *;\n};" % (libname, ";\n  ".join(symbols))
        )
    elif buildconfig.substs["OS_TARGET"] == "Darwin":
        # A list of symbols is generated for Apple ld that simply lists all
        # symbols, with an underscore prefix.
        output.write("".join("_%s\n" % s for s in symbols))

    return set(pp.includes)
Exemplo n.º 15
0
def generate_symbols_file(output, *args):
    ''' '''
    parser = argparse.ArgumentParser()
    parser.add_argument('input')
    parser.add_argument('-D', action=DefinesAction)
    parser.add_argument('-U', action='append', default=[])
    args = parser.parse_args(args)
    input = os.path.abspath(args.input)

    pp = Preprocessor()
    pp.context.update(buildconfig.defines)
    if args.D:
        pp.context.update(args.D)
    for undefine in args.U:
        if undefine in pp.context:
            del pp.context[undefine]
    # Hack until MOZ_DEBUG_FLAGS are simply part of buildconfig.defines
    if buildconfig.substs['MOZ_DEBUG']:
        pp.context['DEBUG'] = '1'
    # Ensure @DATA@ works as expected (see the Windows section further below)
    if buildconfig.substs['OS_TARGET'] == 'WINNT':
        pp.context['DATA'] = 'DATA'
    else:
        pp.context['DATA'] = ''
    pp.out = StringIO()
    pp.do_filter('substitution')
    pp.do_include(input)

    symbols = [s.strip() for s in pp.out.getvalue().splitlines() if s.strip()]

    if buildconfig.substs['OS_TARGET'] == 'WINNT':
        # A def file is generated for MSVC link.exe that looks like the
        # following:
        # LIBRARY library.dll
        # EXPORTS
        #   symbol1
        #   symbol2
        #   ...
        #
        # link.exe however requires special markers for data symbols, so in
        # that case the symbols look like:
        #   data_symbol1 DATA
        #   data_symbol2 DATA
        #   ...
        #
        # In the input file, this is just annotated with the following syntax:
        #   data_symbol1 @DATA@
        #   data_symbol2 @DATA@
        #   ...
        # The DATA variable is "simply" expanded by the preprocessor, to
        # nothing on non-Windows, such that we only get the symbol name on
        # those platforms, and to DATA on Windows, so that the "DATA" part
        # is, in fact, part of the symbol name as far as the symbols variable
        # is concerned.
        libname, ext = os.path.splitext(os.path.basename(output.name))
        assert ext == '.def'
        output.write('LIBRARY %s\nEXPORTS\n  %s\n'
                     % (libname, '\n  '.join(symbols)))
    elif buildconfig.substs['GCC_USE_GNU_LD']:
        # A linker version script is generated for GNU LD that looks like the
        # following:
        # {
        # global:
        #   symbol1;
        #   symbol2;
        #   ...
        # local:
        #   *;
        # };
        output.write('{\nglobal:\n  %s;\nlocal:\n  *;\n};'
                     % ';\n  '.join(symbols))
    elif buildconfig.substs['OS_TARGET'] == 'Darwin':
        # A list of symbols is generated for Apple ld that simply lists all
        # symbols, with an underscore prefix.
        output.write(''.join('_%s\n' % s for s in symbols))

    return set(pp.includes)