def generate_build_files(self):
"""Generate files required for the build.
This function is in charge of generating all the .h/.cpp files derived
from input .webidl files. Please note that there are build actions
required to produce .webidl files and these build actions are
explicitly not captured here: this function assumes all .webidl files
are present and up to date.
This routine is called as part of the build to ensure files that need
to exist are present and up to date. This routine may not be called if
the build dependencies (generated as a result of calling this the first
time) say everything is up to date.
Because reprocessing outputs for every .webidl on every invocation
is expensive, we only regenerate the minimal set of files on every
invocation. The rules for deciding what needs done are roughly as
follows:
1. If any .webidl changes, reparse all .webidl files and regenerate
the global derived files. Only regenerate output files (.h/.cpp)
impacted by the modified .webidl files.
2. If an non-.webidl dependency (Python files, config file) changes,
assume everything is out of date and regenerate the world. This
is because changes in those could globally impact every output
file.
3. If an output file is missing, ensure it is present by performing
necessary regeneration.
"""
# Despite #1 above, we assume the build system is smart enough to not
# invoke us if nothing has changed. Therefore, any invocation means
# something has changed. And, if anything has changed, we need to
# parse the WebIDL.
self._parse_webidl()
result = BuildResult()
# If we parse, we always update globals - they are cheap and it is
# easier that way.
created, updated, unchanged = self._write_global_derived()
result.created |= created
result.updated |= updated
result.unchanged |= unchanged
# If any of the extra dependencies changed, regenerate the world.
global_changed, global_hashes = self._global_dependencies_changed()
if global_changed:
# Make a copy because we may modify.
changed_inputs = set(self._input_paths)
else:
changed_inputs = self._compute_changed_inputs()
self._state['global_depends'] = global_hashes
# Generate bindings from .webidl files.
for filename in sorted(changed_inputs):
basename = mozpath.basename(filename)
result.inputs.add(filename)
written, deps = self._generate_build_files_for_webidl(filename)
result.created |= written[0]
result.updated |= written[1]
result.unchanged |= written[2]
self._state['webidls'][basename] = dict(
filename=filename,
outputs=written[0] | written[1] | written[2],
inputs=set(deps),
sha1=self._input_hashes[filename],
)
# Process some special interfaces required for testing.
for interface in self._example_interfaces:
written = self.generate_example_files(interface)
result.created |= written[0]
result.updated |= written[1]
result.unchanged |= written[2]
# Generate a make dependency file.
if self._make_deps_path:
mk = Makefile()
codegen_rule = mk.create_rule([self._make_deps_target])
codegen_rule.add_dependencies(global_hashes.keys())
codegen_rule.add_dependencies(self._input_paths)
with FileAvoidWrite(self._make_deps_path) as fh:
mk.dump(fh)
self._save_state()
return result
def config_status(config):
# Sanitize config data to feed config.status
# Ideally, all the backend and frontend code would handle the booleans, but
# there are so many things involved, that it's easier to keep config.status
# untouched for now.
def sanitized_bools(v):
if v is True:
return '1'
if v is False:
return ''
return v
sanitized_config = {}
sanitized_config['substs'] = {
k: sanitized_bools(v) for k, v in config.iteritems()
if k not in ('DEFINES', 'non_global_defines', 'TOPSRCDIR', 'TOPOBJDIR',
'ALL_CONFIGURE_PATHS')
}
sanitized_config['defines'] = {
k: sanitized_bools(v) for k, v in config['DEFINES'].iteritems()
}
sanitized_config['non_global_defines'] = config['non_global_defines']
sanitized_config['topsrcdir'] = config['TOPSRCDIR']
sanitized_config['topobjdir'] = config['TOPOBJDIR']
sanitized_config['mozconfig'] = config.get('MOZCONFIG')
# Create config.status. Eventually, we'll want to just do the work it does
# here, when we're able to skip configure tests/use cached results/not rely
# on autoconf.
print("Creating config.status", file=sys.stderr)
encoding = 'mbcs' if sys.platform == 'win32' else 'utf-8'
with codecs.open('config.status', 'w', encoding) as fh:
fh.write(textwrap.dedent('''\
#!%(python)s
# coding=%(encoding)s
from __future__ import unicode_literals
from mozbuild.util import encode
encoding = '%(encoding)s'
''') % {'python': config['PYTHON'], 'encoding': encoding})
# A lot of the build backend code is currently expecting byte
# strings and breaks in subtle ways with unicode strings. (bug 1296508)
for k, v in sanitized_config.iteritems():
fh.write('%s = encode(%s, encoding)\n' % (k, indented_repr(v)))
fh.write("__all__ = ['topobjdir', 'topsrcdir', 'defines', "
"'non_global_defines', 'substs', 'mozconfig']")
if config.get('MOZ_BUILD_APP') != 'js' or config.get('JS_STANDALONE'):
fh.write(textwrap.dedent('''
if __name__ == '__main__':
from mozbuild.util import patch_main
patch_main()
from mozbuild.config_status import config_status
args = dict([(name, globals()[name]) for name in __all__])
config_status(**args)
'''))
# Write out a depfile so Make knows to re-run configure when relevant Python
# changes.
mk = Makefile()
rule = mk.create_rule()
rule.add_targets(["$(OBJDIR)/config.status"])
rule.add_dependencies(itertools.chain(config['ALL_CONFIGURE_PATHS'],
iter_modules_in_path(config['TOPOBJDIR'],
config['TOPSRCDIR'])))
with open('configure.d', 'w') as fh:
mk.dump(fh)
# Other things than us are going to run this file, so we need to give it
# executable permissions.
os.chmod('config.status', 0o755)
if config.get('MOZ_BUILD_APP') != 'js' or config.get('JS_STANDALONE'):
os.environ[b'WRITE_MOZINFO'] = b'1'
from mozbuild.config_status import config_status
# Some values in sanitized_config also have more complex types, such as
# EnumString, which using when calling config_status would currently
# break the build, as well as making it inconsistent with re-running
# config.status. Fortunately, EnumString derives from unicode, so it's
# covered by converting unicode strings.
# A lot of the build backend code is currently expecting byte strings
# and breaks in subtle ways with unicode strings.
return config_status(args=[], **encode(sanitized_config, encoding))
return 0
def InvokeClWithDependencyGeneration(cmdline):
target = ""
# Figure out what the target is
for arg in cmdline:
if arg.startswith("-Fo"):
target = arg[3:]
break
if target is None:
print >> sys.stderr, "No target set"
return 1
# Assume the source file is the last argument
source = cmdline[-1]
assert not source.startswith('-')
# The deps target lives here
depstarget = os.path.basename(target) + ".pp"
cmdline += ['-showIncludes']
mk = Makefile()
rule = mk.create_rule([target])
rule.add_dependencies([normcase(source)])
def on_line(line):
# cl -showIncludes prefixes every header with "Note: including file:"
# and an indentation corresponding to the depth (which we don't need)
if line.startswith(CL_INCLUDES_PREFIX):
dep = line[len(CL_INCLUDES_PREFIX):].strip()
# We can't handle pathes with spaces properly in mddepend.pl, but
# we can assume that anything in a path with spaces is a system
# header and throw it away.
dep = normcase(dep)
if ' ' not in dep:
rule.add_dependencies([dep])
else:
# Make sure we preserve the relevant output from cl. mozprocess
# swallows the newline delimiter, so we need to re-add it.
sys.stdout.write(line)
sys.stdout.write('\n')
# We need to ignore children because MSVC can fire up a background process
# during compilation. This process is cleaned up on its own. If we kill it,
# we can run into weird compilation issues.
p = ProcessHandlerMixin(cmdline,
processOutputLine=[on_line],
ignore_children=True)
p.run()
p.processOutput()
ret = p.wait()
if ret != 0 or target == "":
# p.wait() returns a long. Somehow sys.exit(long(0)) is like
# sys.exit(1). Don't ask why.
return int(ret)
depsdir = os.path.normpath(os.path.join(os.curdir, ".deps"))
depstarget = os.path.join(depsdir, depstarget)
if not os.path.isdir(depsdir):
try:
os.makedirs(depsdir)
except OSError:
pass # This suppresses the error we get when the dir exists, at the
# cost of masking failure to create the directory. We'll just
# die on the next line though, so it's not that much of a loss.
with open(depstarget, "w") as f:
mk.dump(f)
return 0
def main(argv):
parser = argparse.ArgumentParser('Generate a file from a Python script',
add_help=False)
parser.add_argument('python_script', metavar='python-script', type=str,
help='The Python script to run')
parser.add_argument('method_name', metavar='method-name', type=str,
help='The method of the script to invoke')
parser.add_argument('output_file', metavar='output-file', type=str,
help='The file to generate')
parser.add_argument('dep_file', metavar='dep-file', type=str,
help='File to write any additional make dependencies to')
parser.add_argument('additional_arguments', metavar='arg',
nargs=argparse.REMAINDER,
help="Additional arguments to the script's main() method")
args = parser.parse_args(argv)
script = args.python_script
# Permit the script to import modules from the same directory in which it
# resides. The justification for doing this is that if we were invoking
# the script as:
#
# python script arg1...
#
# then importing modules from the script's directory would come for free.
# Since we're invoking the script in a roundabout way, we provide this
# bit of convenience.
sys.path.append(os.path.dirname(script))
with open(script, 'r') as fh:
module = imp.load_module('script', fh, script,
('.py', 'r', imp.PY_SOURCE))
method = args.method_name
if not hasattr(module, method):
print('Error: script "{0}" is missing a {1} method'.format(script, method),
file=sys.stderr)
return 1
ret = 1
try:
with FileAvoidWrite(args.output_file) as output:
ret = module.__dict__[method](output, *args.additional_arguments)
# The following values indicate a statement of success:
# - a set() (see below)
# - 0
# - False
# - None
#
# Everything else is an error (so scripts can conveniently |return
# 1| or similar). If a set is returned, the elements of the set
# indicate additional dependencies that will be listed in the deps
# file. Python module imports are automatically included as
# dependencies.
if isinstance(ret, set):
deps = ret
# The script succeeded, so reset |ret| to indicate that.
ret = None
else:
deps = set()
# Only write out the dependencies if the script was successful
if not ret:
# Add dependencies on any python modules that were imported by
# the script.
deps |= set(iter_modules_in_path(buildconfig.topsrcdir,
buildconfig.topobjdir))
# Add dependencies on any buildconfig items that were accessed
# by the script.
deps |= set(buildconfig.get_dependencies())
mk = Makefile()
mk.create_rule([args.output_file]).add_dependencies(deps)
with FileAvoidWrite(args.dep_file) as dep_file:
mk.dump(dep_file)
# Even when our file's contents haven't changed, we want to update
# the file's mtime so make knows this target isn't still older than
# whatever prerequisite caused it to be built this time around.
try:
os.utime(args.output_file, None)
except:
print('Error processing file "{0}"'.format(args.output_file),
file=sys.stderr)
traceback.print_exc()
except IOError as e:
print('Error opening file "{0}"'.format(e.filename), file=sys.stderr)
traceback.print_exc()
return 1
return ret
def process(
input_dirs,
inc_paths,
bindings_conf,
header_dir,
xpcrs_dir,
xpt_dir,
deps_dir,
module,
idl_files,
):
p = IDLParser()
xpts = []
mk = Makefile()
rule = mk.create_rule()
glbl = {}
exec(open(bindings_conf, encoding="utf-8").read(), glbl)
webidlconfig = glbl["DOMInterfaces"]
# Write out dependencies for Python modules we import. If this list isn't
# up to date, we will not re-process XPIDL files if the processor changes.
rule.add_dependencies(
six.ensure_text(s) for s in iter_modules_in_path(topsrcdir))
for path in idl_files:
basename = os.path.basename(path)
stem, _ = os.path.splitext(basename)
idl_data = open(path, encoding="utf-8").read()
idl = p.parse(idl_data, filename=path)
idl.resolve(inc_paths, p, webidlconfig)
header_path = os.path.join(header_dir, "%s.h" % stem)
rs_rt_path = os.path.join(xpcrs_dir, "rt", "%s.rs" % stem)
rs_bt_path = os.path.join(xpcrs_dir, "bt", "%s.rs" % stem)
xpts.append(jsonxpt.build_typelib(idl))
rule.add_dependencies(six.ensure_text(s) for s in idl.deps)
# The print_* functions don't actually do anything with the
# passed-in path other than writing it into the file to let people
# know where the original source was. This script receives
# absolute paths, which are not so great to embed in header files
# (they mess with deterministic generation of files on different
# machines, Searchfox logic, shared compilation caches, etc.), so
# we pass in fake paths that are the same across compilations, but
# should still enable people to figure out where to go.
relpath = mozpath.relpath(path, topsrcdir)
with FileAvoidWrite(header_path) as fh:
print_header(idl, fh, path, relpath)
with FileAvoidWrite(rs_rt_path) as fh:
print_rust_bindings(idl, fh, relpath)
with FileAvoidWrite(rs_bt_path) as fh:
print_rust_macros_bindings(idl, fh, relpath)
# NOTE: We don't use FileAvoidWrite here as we may re-run this code due to a
# number of different changes in the code, which may not cause the .xpt
# files to be changed in any way. This means that make will re-run us every
# time a build is run whether or not anything changed. To fix this we
# unconditionally write out the file.
xpt_path = os.path.join(xpt_dir, "%s.xpt" % module)
with open(xpt_path, "w", encoding="utf-8", newline="\n") as fh:
jsonxpt.write(jsonxpt.link(xpts), fh)
rule.add_targets([six.ensure_text(xpt_path)])
if deps_dir:
deps_path = os.path.join(deps_dir, "%s.pp" % module)
with FileAvoidWrite(deps_path) as fh:
mk.dump(fh)
def consume_finished(self):
mk = Makefile()
# Add the default rule at the very beginning.
mk.create_rule(['default'])
mk.add_statement('TOPSRCDIR = %s' % self.environment.topsrcdir)
mk.add_statement('TOPOBJDIR = %s' % self.environment.topobjdir)
if not self._has_xpidl:
mk.add_statement('NO_XPIDL = 1')
# Add a few necessary variables inherited from configure
for var in (
'PYTHON',
'ACDEFINES',
'MOZ_BUILD_APP',
'MOZ_WIDGET_TOOLKIT',
):
value = self.environment.substs.get(var)
if value is not None:
mk.add_statement('%s = %s' % (var, value))
install_manifests_bases = self._install_manifests.keys()
# Add information for chrome manifest generation
manifest_targets = []
for target, entries in self._manifest_entries.iteritems():
manifest_targets.append(target)
install_target = mozpath.basedir(target, install_manifests_bases)
self._install_manifests[install_target].add_content(
''.join('%s\n' % e for e in sorted(entries)),
mozpath.relpath(target, install_target))
# Add information for install manifests.
mk.add_statement('INSTALL_MANIFESTS = %s' %
' '.join(self._install_manifests.keys()))
# Add dependencies we inferred:
for target, deps in self._dependencies.iteritems():
mk.create_rule([target]).add_dependencies('$(TOPOBJDIR)/%s' % d
for d in deps)
# This is not great, but it's better to have some dependencies on these Python files.
python_deps = [
'$(TOPSRCDIR)/python/mozbuild/mozbuild/action/l10n_merge.py',
'$(TOPSRCDIR)/third_party/python/compare-locales/compare_locales/compare.py',
'$(TOPSRCDIR)/third_party/python/compare-locales/compare_locales/paths.py',
]
# Add l10n dependencies we inferred:
for target, deps in self._l10n_dependencies.iteritems():
mk.create_rule([target]).add_dependencies('%s' % d[0]
for d in deps)
for (merge, ref_file, l10n_file) in deps:
rule = mk.create_rule([
merge
]).add_dependencies([ref_file, l10n_file] + python_deps)
rule.add_commands([
'$(PYTHON) -m mozbuild.action.l10n_merge --output {} --ref-file {} --l10n-file {}'
.format(merge, ref_file, l10n_file)
])
# Add a dummy rule for the l10n file since it might not exist.
mk.create_rule([l10n_file])
mk.add_statement('include $(TOPSRCDIR)/config/faster/rules.mk')
for base, install_manifest in self._install_manifests.iteritems():
with self._write_file(
mozpath.join(self.environment.topobjdir, 'faster',
'install_%s' % base.replace('/', '_'))) as fh:
install_manifest.write(fileobj=fh)
# For artifact builds only, write a single unified manifest for consumption by |mach watch|.
if self.environment.is_artifact_build:
unified_manifest = InstallManifest()
for base, install_manifest in self._install_manifests.iteritems():
# Expect 'dist/bin/**', which includes 'dist/bin' with no trailing slash.
assert base.startswith('dist/bin')
base = base[len('dist/bin'):]
if base and base[0] == '/':
base = base[1:]
unified_manifest.add_entries_from(install_manifest, base=base)
with self._write_file(
mozpath.join(self.environment.topobjdir, 'faster',
'unified_install_dist_bin')) as fh:
unified_manifest.write(fileobj=fh)
with self._write_file(
mozpath.join(self.environment.topobjdir, 'faster',
'Makefile')) as fh:
mk.dump(fh, removal_guard=False)
