def _global_dependencies_changed(self):
"""Determine whether the global dependencies have changed."""
current_files = set(iter_modules_in_path(mozpath.dirname(__file__)))
# We need to catch other .py files from /dom/bindings. We assume these
# are in the same directory as the config file.
current_files |= set(iter_modules_in_path(mozpath.dirname(self._config_path)))
current_files.add(self._config_path)
current_hashes = {}
for f in current_files:
# This will fail if the file doesn't exist. If a current global
# dependency doesn't exist, something else is wrong.
with open(f, 'rb') as fh:
current_hashes[f] = hashlib.sha1(fh.read()).hexdigest()
# The set of files has changed.
if current_files ^ set(self._state['global_depends'].keys()):
return True, current_hashes
# Compare hashes.
for f, sha1 in current_hashes.items():
if sha1 != self._state['global_depends'][f]:
return True, current_hashes
return False, current_hashes
def _global_dependencies_changed(self):
"""Determine whether the global dependencies have changed."""
current_files = set(iter_modules_in_path(mozpath.dirname(__file__)))
# We need to catch other .py files from /dom/bindings. We assume these
# are in the same directory as the config file.
current_files |= set(
iter_modules_in_path(mozpath.dirname(self._config_path)))
current_files.add(self._config_path)
current_hashes = {}
for f in current_files:
# This will fail if the file doesn't exist. If a current global
# dependency doesn't exist, something else is wrong.
with open(f, 'rb') as fh:
current_hashes[f] = hashlib.sha1(fh.read()).hexdigest()
# The set of files has changed.
if current_files ^ set(self._state['global_depends'].keys()):
return True, current_hashes
# Compare hashes.
for f, sha1 in current_hashes.items():
if sha1 != self._state['global_depends'][f]:
return True, current_hashes
return False, current_hashes
def generate_binding_files(config, outputprefix, srcprefix, webidlfile,
generatedEventsWebIDLFiles):
"""
|config| Is the configuration object.
|outputprefix| is a prefix to use for the header guards and filename.
"""
depsname = ".deps/" + outputprefix + ".pp"
root = CGBindingRoot(config, outputprefix, webidlfile)
replaceFileIfChanged(outputprefix + ".h", root.declare())
replaceFileIfChanged(outputprefix + ".cpp", root.define())
if webidlfile in generatedEventsWebIDLFiles:
eventName = webidlfile[:-len(".webidl")]
generatedEvent = CGEventRoot(config, eventName)
replaceFileIfChanged(eventName + ".h", generatedEvent.declare())
replaceFileIfChanged(eventName + ".cpp", generatedEvent.define())
mk = Makefile()
# NOTE: it's VERY important that we output dependencies for the FooBinding
# file here, not for the header or generated cpp file. These dependencies
# are used later to properly determine changedDeps and prevent rebuilding
# too much. See the comment explaining $(binding_dependency_trackers) in
# Makefile.in.
rule = mk.create_rule([outputprefix])
rule.add_dependencies(
os.path.join(srcprefix, x) for x in sorted(root.deps()))
rule.add_dependencies(iter_modules_in_path(topsrcdir))
with open(depsname, 'w') as f:
mk.dump(f)
def generate_binding_files(config, outputprefix, srcprefix, webidlfile,
generatedEventsWebIDLFiles):
"""
|config| Is the configuration object.
|outputprefix| is a prefix to use for the header guards and filename.
"""
depsname = ".deps/" + outputprefix + ".pp"
root = CGBindingRoot(config, outputprefix, webidlfile)
replaceFileIfChanged(outputprefix + ".h", root.declare())
replaceFileIfChanged(outputprefix + ".cpp", root.define())
if webidlfile in generatedEventsWebIDLFiles:
eventName = webidlfile[:-len(".webidl")]
generatedEvent = CGEventRoot(config, eventName)
replaceFileIfChanged(eventName + ".h", generatedEvent.declare())
replaceFileIfChanged(eventName + ".cpp", generatedEvent.define())
mk = Makefile()
# NOTE: it's VERY important that we output dependencies for the FooBinding
# file here, not for the header or generated cpp file. These dependencies
# are used later to properly determine changedDeps and prevent rebuilding
# too much. See the comment explaining $(binding_dependency_trackers) in
# Makefile.in.
rule = mk.create_rule([outputprefix])
rule.add_dependencies(os.path.join(srcprefix, x) for x in root.deps())
rule.add_dependencies(iter_modules_in_path(topsrcdir))
with open(depsname, 'w') as f:
mk.dump(f)
def test_iter_modules_in_path():
tests_path = os.path.normcase(os.path.dirname(__file__))
paths = list(iter_modules_in_path(tests_path))
assert set(paths) == set([
os.path.join(os.path.abspath(tests_path), "__init__.py"),
os.path.join(os.path.abspath(tests_path), "test_pythonutil.py"),
])
def test_iter_modules_in_path(self):
mozbuild_path = os.path.dirname(os.path.dirname(__file__))
paths = list(iter_modules_in_path(mozbuild_path))
self.assertEquals(sorted(paths), [
os.path.join(os.path.abspath(mozbuild_path), '__init__.py'),
os.path.join(os.path.abspath(mozbuild_path), 'pythonutil.py'),
os.path.join(os.path.abspath(mozbuild_path), 'test', 'test_pythonutil.py'),
])
def process(input_dirs, inc_paths, bindings_conf, cache_dir, header_dir,
xpcrs_dir, xpt_dir, deps_dir, module, idl_files):
p = IDLParser(outputdir=cache_dir)
xpts = []
mk = Makefile()
rule = mk.create_rule()
glbl = {}
execfile(bindings_conf, glbl)
webidlconfig = glbl['DOMInterfaces']
# Write out dependencies for Python modules we import. If this list isn't
# up to date, we will not re-process XPIDL files if the processor changes.
rule.add_dependencies(six.ensure_text(s) for s in
iter_modules_in_path(topsrcdir))
for path in idl_files:
basename = os.path.basename(path)
stem, _ = os.path.splitext(basename)
idl_data = open(path).read()
idl = p.parse(idl_data, filename=path)
idl.resolve(inc_paths, p, webidlconfig)
header_path = os.path.join(header_dir, '%s.h' % stem)
rs_rt_path = os.path.join(xpcrs_dir, 'rt', '%s.rs' % stem)
rs_bt_path = os.path.join(xpcrs_dir, 'bt', '%s.rs' % stem)
xpts.append(jsonxpt.build_typelib(idl))
rule.add_dependencies(six.ensure_text(s) for s in idl.deps)
with FileAvoidWrite(header_path) as fh:
print_header(idl, fh, path)
with FileAvoidWrite(rs_rt_path) as fh:
print_rust_bindings(idl, fh, path)
with FileAvoidWrite(rs_bt_path) as fh:
print_rust_macros_bindings(idl, fh, path)
# NOTE: We don't use FileAvoidWrite here as we may re-run this code due to a
# number of different changes in the code, which may not cause the .xpt
# files to be changed in any way. This means that make will re-run us every
# time a build is run whether or not anything changed. To fix this we
# unconditionally write out the file.
xpt_path = os.path.join(xpt_dir, '%s.xpt' % module)
with open(xpt_path, 'w') as fh:
jsonxpt.write(jsonxpt.link(xpts), fh)
rule.add_targets([six.ensure_text(xpt_path)])
if deps_dir:
deps_path = os.path.join(deps_dir, '%s.pp' % module)
with FileAvoidWrite(deps_path) as fh:
mk.dump(fh)
def test_iter_modules_in_path(self):
mozbuild_path = os.path.normcase(
os.path.dirname(os.path.dirname(__file__)))
paths = list(iter_modules_in_path(mozbuild_path))
self.assertEquals(sorted(paths), [
os.path.join(os.path.abspath(mozbuild_path), '__init__.py'),
os.path.join(os.path.abspath(mozbuild_path), 'pythonutil.py'),
os.path.join(os.path.abspath(mozbuild_path), 'test',
'test_pythonutil.py'),
])
def process(input_dir, inc_paths, bindings_conf, cache_dir, header_dir,
xpcrs_dir, xpt_dir, deps_dir, module, stems):
p = IDLParser(outputdir=cache_dir)
xpts = []
mk = Makefile()
rule = mk.create_rule()
glbl = {}
execfile(bindings_conf, glbl)
webidlconfig = glbl['DOMInterfaces']
# Write out dependencies for Python modules we import. If this list isn't
# up to date, we will not re-process XPIDL files if the processor changes.
rule.add_dependencies(iter_modules_in_path(topsrcdir))
for stem in stems:
path = os.path.join(input_dir, '%s.idl' % stem)
idl_data = open(path).read()
idl = p.parse(idl_data, filename=path)
idl.resolve([input_dir] + inc_paths, p, webidlconfig)
header_path = os.path.join(header_dir, '%s.h' % stem)
rs_rt_path = os.path.join(xpcrs_dir, 'rt', '%s.rs' % stem)
rs_bt_path = os.path.join(xpcrs_dir, 'bt', '%s.rs' % stem)
xpts.append(jsonxpt.build_typelib(idl))
rule.add_dependencies(idl.deps)
with FileAvoidWrite(header_path) as fh:
print_header(idl, fh, path)
with FileAvoidWrite(rs_rt_path) as fh:
print_rust_bindings(idl, fh, path)
with FileAvoidWrite(rs_bt_path) as fh:
print_rust_macros_bindings(idl, fh, path)
xpt_path = os.path.join(xpt_dir, '%s.xpt' % module)
with FileAvoidWrite(xpt_path) as fh:
jsonxpt.write(jsonxpt.link(xpts), fh)
rule.add_targets([xpt_path])
if deps_dir:
deps_path = os.path.join(deps_dir, '%s.pp' % module)
with FileAvoidWrite(deps_path) as fh:
mk.dump(fh)
def process(input_dir, inc_paths, cache_dir, header_dir, xpcrs_dir,
xpt_dir, deps_dir, module, stems):
p = IDLParser(outputdir=cache_dir)
xpts = {}
mk = Makefile()
rule = mk.create_rule()
# Write out dependencies for Python modules we import. If this list isn't
# up to date, we will not re-process XPIDL files if the processor changes.
rule.add_dependencies(iter_modules_in_path(topsrcdir))
for stem in stems:
path = os.path.join(input_dir, '%s.idl' % stem)
idl_data = open(path).read()
idl = p.parse(idl_data, filename=path)
idl.resolve([input_dir] + inc_paths, p)
header_path = os.path.join(header_dir, '%s.h' % stem)
rs_rt_path = os.path.join(xpcrs_dir, 'rt', '%s.rs' % stem)
rs_bt_path = os.path.join(xpcrs_dir, 'bt', '%s.rs' % stem)
xpt = BytesIO()
write_typelib(idl, xpt, path)
xpt.seek(0)
xpts[stem] = xpt
rule.add_dependencies(idl.deps)
with FileAvoidWrite(header_path) as fh:
print_header(idl, fh, path)
with FileAvoidWrite(rs_rt_path) as fh:
print_rust_bindings(idl, fh, path)
with FileAvoidWrite(rs_bt_path) as fh:
print_rust_macros_bindings(idl, fh, path)
# TODO use FileAvoidWrite once it supports binary mode.
xpt_path = os.path.join(xpt_dir, '%s.xpt' % module)
xpt_link(xpts.values()).write(xpt_path)
rule.add_targets([xpt_path])
if deps_dir:
deps_path = os.path.join(deps_dir, '%s.pp' % module)
with FileAvoidWrite(deps_path) as fh:
mk.dump(fh)
def generate_binding_files(config, outputprefix, srcprefix, webidlfile):
"""
|config| Is the configuration object.
|outputprefix| is a prefix to use for the header guards and filename.
"""
depsname = ".deps/" + outputprefix + ".pp"
root = CGBindingRoot(config, outputprefix, webidlfile)
replaceFileIfChanged(outputprefix + ".h", root.declare())
replaceFileIfChanged(outputprefix + ".cpp", root.define())
mk = Makefile()
rule = mk.create_rule([outputprefix + '.h', outputprefix + '.cpp'])
rule.add_dependencies(os.path.join(srcprefix, x) for x in root.deps())
rule.add_dependencies(iter_modules_in_path(topsrcdir))
with open(depsname, 'w') as f:
mk.dump(f)
def generate_binding_files(config, outputprefix, srcprefix, webidlfile):
"""
|config| Is the configuration object.
|outputprefix| is a prefix to use for the header guards and filename.
"""
depsname = ".deps/" + outputprefix + ".pp"
root = CGBindingRoot(config, outputprefix, webidlfile)
replaceFileIfChanged(outputprefix + ".h", root.declare())
replaceFileIfChanged(outputprefix + ".cpp", root.define())
mk = Makefile()
rule = mk.create_rule([outputprefix + ".h", outputprefix + ".cpp"])
rule.add_dependencies(os.path.join(srcprefix, x) for x in root.deps())
rule.add_dependencies(iter_modules_in_path(topsrcdir))
with open(depsname, "w") as f:
mk.dump(f)
def config_status(config, execute=True):
# Sanitize config data to feed config.status
# Ideally, all the backend and frontend code would handle the booleans, but
# there are so many things involved, that it's easier to keep config.status
# untouched for now.
def sanitize_config(v):
if v is True:
return "1"
if v is False:
return ""
# Serialize types that look like lists and tuples as lists.
if not isinstance(v, (bytes, six.text_type, dict)) and isinstance(
v, Iterable):
return list(v)
return v
sanitized_config = {}
sanitized_config["substs"] = {
k: sanitize_config(v)
for k, v in six.iteritems(config) if k not in (
"DEFINES",
"TOPSRCDIR",
"TOPOBJDIR",
"CONFIG_STATUS_DEPS",
"OLD_CONFIGURE_SUBSTS",
"OLD_CONFIGURE_DEFINES",
)
}
for k, v in config["OLD_CONFIGURE_SUBSTS"]:
sanitized_config["substs"][k] = sanitize_config(v)
sanitized_config["defines"] = {
k: sanitize_config(v)
for k, v in six.iteritems(config["DEFINES"])
}
for k, v in config["OLD_CONFIGURE_DEFINES"]:
sanitized_config["defines"][k] = sanitize_config(v)
sanitized_config["topsrcdir"] = config["TOPSRCDIR"]
sanitized_config["topobjdir"] = config["TOPOBJDIR"]
sanitized_config["mozconfig"] = config.get("MOZCONFIG")
if not check_unicode(sanitized_config):
print("Configuration should be all unicode.", file=sys.stderr)
print("Please file a bug for the above.", file=sys.stderr)
sys.exit(1)
# Some values in sanitized_config also have more complex types, such as
# EnumString, which using when calling config_status would currently
# break the build, as well as making it inconsistent with re-running
# config.status, for which they are normalized to plain strings via
# indented_repr. Likewise for non-dict non-string iterables being
# converted to lists.
def normalize(obj):
if isinstance(obj, dict):
return {k: normalize(v) for k, v in six.iteritems(obj)}
if isinstance(obj, six.text_type):
return six.text_type(obj)
if isinstance(obj, Iterable):
return [normalize(o) for o in obj]
return obj
sanitized_config = normalize(sanitized_config)
# Create config.status. Eventually, we'll want to just do the work it does
# here, when we're able to skip configure tests/use cached results/not rely
# on autoconf.
with codecs.open("config.status", "w", "utf-8") as fh:
fh.write(
textwrap.dedent("""\
#!%(python)s
# coding=utf-8
from __future__ import unicode_literals
""") % {"python": config["PYTHON3"]})
for k, v in sorted(six.iteritems(sanitized_config)):
fh.write("%s = " % k)
write_indented_repr(fh, v)
fh.write("__all__ = ['topobjdir', 'topsrcdir', 'defines', "
"'substs', 'mozconfig']")
if execute:
fh.write(
textwrap.dedent("""
if __name__ == '__main__':
from mozbuild.util import patch_main
patch_main()
from mozbuild.config_status import config_status
args = dict([(name, globals()[name]) for name in __all__])
config_status(**args)
"""))
partial_config = PartialConfigEnvironment(config["TOPOBJDIR"])
partial_config.write_vars(sanitized_config)
# Write out a file so the build backend knows to re-run configure when
# relevant Python changes.
with io.open("config_status_deps.in", "w", encoding="utf-8",
newline="\n") as fh:
for f in sorted(
itertools.chain(
config["CONFIG_STATUS_DEPS"],
iter_modules_in_path(config["TOPOBJDIR"],
config["TOPSRCDIR"]),
)):
fh.write("%s\n" % mozpath.normpath(f))
# Other things than us are going to run this file, so we need to give it
# executable permissions.
os.chmod("config.status", 0o755)
if execute:
from mozbuild.config_status import config_status
return config_status(args=[], **sanitized_config)
return 0
def config_status(config):
# Sanitize config data to feed config.status
# Ideally, all the backend and frontend code would handle the booleans, but
# there are so many things involved, that it's easier to keep config.status
# untouched for now.
def sanitized_bools(v):
if v is True:
return '1'
if v is False:
return ''
return v
sanitized_config = {}
sanitized_config['substs'] = {
k: sanitized_bools(v) for k, v in config.iteritems()
if k not in ('DEFINES', 'non_global_defines', 'TOPSRCDIR', 'TOPOBJDIR',
'CONFIG_STATUS_DEPS')
}
sanitized_config['defines'] = {
k: sanitized_bools(v) for k, v in config['DEFINES'].iteritems()
}
sanitized_config['non_global_defines'] = config['non_global_defines']
sanitized_config['topsrcdir'] = config['TOPSRCDIR']
sanitized_config['topobjdir'] = config['TOPOBJDIR']
sanitized_config['mozconfig'] = config.get('MOZCONFIG')
# Create config.status. Eventually, we'll want to just do the work it does
# here, when we're able to skip configure tests/use cached results/not rely
# on autoconf.
logging.getLogger('moz.configure').info('Creating config.status')
encoding = 'mbcs' if sys.platform == 'win32' else 'utf-8'
with codecs.open('config.status', 'w', encoding) as fh:
fh.write(textwrap.dedent('''\
#!%(python)s
# coding=%(encoding)s
from __future__ import unicode_literals
from mozbuild.util import encode
encoding = '%(encoding)s'
''') % {'python': config['PYTHON'], 'encoding': encoding})
# A lot of the build backend code is currently expecting byte
# strings and breaks in subtle ways with unicode strings. (bug 1296508)
for k, v in sanitized_config.iteritems():
fh.write('%s = encode(%s, encoding)\n' % (k, indented_repr(v)))
fh.write("__all__ = ['topobjdir', 'topsrcdir', 'defines', "
"'non_global_defines', 'substs', 'mozconfig']")
if config.get('MOZ_BUILD_APP') != 'js' or config.get('JS_STANDALONE'):
fh.write(textwrap.dedent('''
if __name__ == '__main__':
from mozbuild.util import patch_main
patch_main()
from mozbuild.config_status import config_status
args = dict([(name, globals()[name]) for name in __all__])
config_status(**args)
'''))
partial_config = PartialConfigEnvironment(config['TOPOBJDIR'])
partial_config.write_vars(sanitized_config)
# Write out a file so the build backend knows to re-run configure when
# relevant Python changes.
with open('config_status_deps.in', 'w') as fh:
for f in itertools.chain(config['CONFIG_STATUS_DEPS'],
iter_modules_in_path(config['TOPOBJDIR'],
config['TOPSRCDIR'])):
fh.write('%s\n' % mozpath.normpath(f))
# Other things than us are going to run this file, so we need to give it
# executable permissions.
os.chmod('config.status', 0o755)
if config.get('MOZ_BUILD_APP') != 'js' or config.get('JS_STANDALONE'):
from mozbuild.config_status import config_status
# Some values in sanitized_config also have more complex types, such as
# EnumString, which using when calling config_status would currently
# break the build, as well as making it inconsistent with re-running
# config.status. Fortunately, EnumString derives from unicode, so it's
# covered by converting unicode strings.
# A lot of the build backend code is currently expecting byte strings
# and breaks in subtle ways with unicode strings.
return config_status(args=[], **encode(sanitized_config, encoding))
return 0
def main(argv):
parser = argparse.ArgumentParser('Generate a file from a Python script',
add_help=False)
parser.add_argument('python_script', metavar='python-script', type=str,
help='The Python script to run')
parser.add_argument('method_name', metavar='method-name', type=str,
help='The method of the script to invoke')
parser.add_argument('output_file', metavar='output-file', type=str,
help='The file to generate')
parser.add_argument('dep_file', metavar='dep-file', type=str,
help='File to write any additional make dependencies to')
parser.add_argument('additional_arguments', metavar='arg',
nargs=argparse.REMAINDER,
help="Additional arguments to the script's main() method")
args = parser.parse_args(argv)
script = args.python_script
# Permit the script to import modules from the same directory in which it
# resides. The justification for doing this is that if we were invoking
# the script as:
#
# python script arg1...
#
# then importing modules from the script's directory would come for free.
# Since we're invoking the script in a roundabout way, we provide this
# bit of convenience.
sys.path.append(os.path.dirname(script))
with open(script, 'r') as fh:
module = imp.load_module('script', fh, script,
('.py', 'r', imp.PY_SOURCE))
method = args.method_name
if not hasattr(module, method):
print('Error: script "{0}" is missing a {1} method'.format(script, method),
file=sys.stderr)
return 1
ret = 1
try:
with FileAvoidWrite(args.output_file) as output:
ret = module.__dict__[method](output, *args.additional_arguments)
# The following values indicate a statement of success:
# - a set() (see below)
# - 0
# - False
# - None
#
# Everything else is an error (so scripts can conveniently |return
# 1| or similar). If a set is returned, the elements of the set
# indicate additional dependencies that will be listed in the deps
# file. Python module imports are automatically included as
# dependencies.
if isinstance(ret, set):
deps = ret
# The script succeeded, so reset |ret| to indicate that.
ret = None
else:
deps = set()
# Only write out the dependencies if the script was successful
if not ret:
# Add dependencies on any python modules that were imported by
# the script.
deps |= set(iter_modules_in_path(buildconfig.topsrcdir,
buildconfig.topobjdir))
mk = Makefile()
mk.create_rule([args.output_file]).add_dependencies(deps)
with FileAvoidWrite(args.dep_file) as dep_file:
mk.dump(dep_file)
# Even when our file's contents haven't changed, we want to update
# the file's mtime so make knows this target isn't still older than
# whatever prerequisite caused it to be built this time around.
try:
os.utime(args.output_file, None)
except:
print('Error processing file "{0}"'.format(args.output_file),
file=sys.stderr)
traceback.print_exc()
except IOError as e:
print('Error opening file "{0}"'.format(e.filename), file=sys.stderr)
traceback.print_exc()
return 1
return ret
syncMsgList = options.syncMsgList
headersdir = options.headersdir
cppdir = options.cppdir
includedirs = [ os.path.abspath(incdir) for incdir in options.includedirs ]
if not len(files):
op.error("No IPDL files specified")
ipcmessagestartpath = os.path.join(headersdir, 'IPCMessageStart.h')
ipc_msgtype_name_path = os.path.join(cppdir, 'IPCMessageTypeName.cpp')
# Compiling the IPDL files can take a long time, even on a fast machine.
# Check to see whether we need to do any work.
latestipdlmod = max(os.stat(f).st_mtime
for f in itertools.chain(files,
iter_modules_in_path(mozpath.dirname(__file__))))
def outputModTime(f):
# A non-existant file is newer than everything.
if not os.path.exists(f):
return 0
return os.stat(f).st_mtime
# Because the IPDL headers are placed into directories reflecting their
# namespace, collect a list here so we can easily map output names without
# parsing the actual IPDL files themselves.
headersmap = {}
for (path, dirs, headers) in os.walk(headersdir):
for h in headers:
base = os.path.basename(h)
if base in headersmap:
def main(argv):
parser = argparse.ArgumentParser("Generate a file from a Python script", add_help=False)
parser.add_argument("python_script", metavar="python-script", type=str, help="The Python script to run")
parser.add_argument("method_name", metavar="method-name", type=str, help="The method of the script to invoke")
parser.add_argument("output_file", metavar="output-file", type=str, help="The file to generate")
parser.add_argument(
"dep_file", metavar="dep-file", type=str, help="File to write any additional make dependencies to"
)
parser.add_argument(
"additional_arguments",
metavar="arg",
nargs=argparse.REMAINDER,
help="Additional arguments to the script's main() method",
)
args = parser.parse_args(argv)
script = args.python_script
# Permit the script to import modules from the same directory in which it
# resides. The justification for doing this is that if we were invoking
# the script as:
#
# python script arg1...
#
# then importing modules from the script's directory would come for free.
# Since we're invoking the script in a roundabout way, we provide this
# bit of convenience.
sys.path.append(os.path.dirname(script))
with open(script, "r") as fh:
module = imp.load_module("script", fh, script, (".py", "r", imp.PY_SOURCE))
method = args.method_name
if not hasattr(module, method):
print('Error: script "{0}" is missing a {1} method'.format(script, method), file=sys.stderr)
return 1
ret = 1
try:
with FileAvoidWrite(args.output_file) as output:
ret = module.__dict__[method](output, *args.additional_arguments)
# We treat sets as a statement of success. Everything else
# is an error (so scripts can conveniently |return 1| or
# similar).
if isinstance(ret, set) and ret:
ret |= set(iter_modules_in_path(buildconfig.topsrcdir, buildconfig.topobjdir))
mk = Makefile()
mk.create_rule([args.output_file]).add_dependencies(ret)
with FileAvoidWrite(args.dep_file) as dep_file:
mk.dump(dep_file)
# The script succeeded, so reset |ret| to indicate that.
ret = None
# Even when our file's contents haven't changed, we want to update
# the file's mtime so make knows this target isn't still older than
# whatever prerequisite caused it to be built this time around.
try:
os.utime(args.output_file, None)
except:
print('Error processing file "{0}"'.format(args.output_file), file=sys.stderr)
traceback.print_exc()
except IOError as e:
print('Error opening file "{0}"'.format(e.filename), file=sys.stderr)
traceback.print_exc()
return 1
return ret
def config_status(config):
# Sanitize config data to feed config.status
# Ideally, all the backend and frontend code would handle the booleans, but
# there are so many things involved, that it's easier to keep config.status
# untouched for now.
def sanitized_bools(v):
if v is True:
return '1'
if v is False:
return ''
return v
sanitized_config = {}
sanitized_config['substs'] = {
k: sanitized_bools(v) for k, v in six.iteritems(config)
if k not in ('DEFINES', 'non_global_defines', 'TOPSRCDIR', 'TOPOBJDIR',
'CONFIG_STATUS_DEPS')
}
sanitized_config['defines'] = {
k: sanitized_bools(v) for k, v in six.iteritems(config['DEFINES'])
}
sanitized_config['non_global_defines'] = config['non_global_defines']
sanitized_config['topsrcdir'] = config['TOPSRCDIR']
sanitized_config['topobjdir'] = config['TOPOBJDIR']
sanitized_config['mozconfig'] = config.get('MOZCONFIG')
if not check_unicode(sanitized_config):
print("Configuration should be all unicode.", file=sys.stderr)
print("Please file a bug for the above.", file=sys.stderr)
sys.exit(1)
# Create config.status. Eventually, we'll want to just do the work it does
# here, when we're able to skip configure tests/use cached results/not rely
# on autoconf.
logging.getLogger('moz.configure').info('Creating config.status')
with codecs.open('config.status', 'w', 'utf-8') as fh:
fh.write(textwrap.dedent('''\
#!%(python)s
# coding=utf-8
from __future__ import unicode_literals
''') % {'python': config['PYTHON']})
for k, v in six.iteritems(sanitized_config):
fh.write('%s = %s\n' % (k, indented_repr(v)))
fh.write("__all__ = ['topobjdir', 'topsrcdir', 'defines', "
"'non_global_defines', 'substs', 'mozconfig']")
if config.get('MOZ_BUILD_APP') != 'js' or config.get('JS_STANDALONE'):
fh.write(textwrap.dedent('''
if __name__ == '__main__':
from mozbuild.util import patch_main
patch_main()
from mozbuild.config_status import config_status
args = dict([(name, globals()[name]) for name in __all__])
config_status(**args)
'''))
partial_config = PartialConfigEnvironment(config['TOPOBJDIR'])
partial_config.write_vars(sanitized_config)
# Write out a file so the build backend knows to re-run configure when
# relevant Python changes.
with open('config_status_deps.in', 'w') as fh:
for f in itertools.chain(config['CONFIG_STATUS_DEPS'],
iter_modules_in_path(config['TOPOBJDIR'],
config['TOPSRCDIR'])):
fh.write('%s\n' % mozpath.normpath(f))
# Other things than us are going to run this file, so we need to give it
# executable permissions.
os.chmod('config.status', 0o755)
if config.get('MOZ_BUILD_APP') != 'js' or config.get('JS_STANDALONE'):
from mozbuild.config_status import config_status
# Some values in sanitized_config also have more complex types, such as
# EnumString, which using when calling config_status would currently
# break the build, as well as making it inconsistent with re-running
# config.status, for which they are normalized to plain strings via
# indented_repr. Likewise for non-dict non-string iterables being
# converted to lists.
def normalize(obj):
if isinstance(obj, dict):
return {
k: normalize(v)
for k, v in six.iteritems(obj)
}
if isinstance(obj, six.text_type):
return six.text_type(obj)
if isinstance(obj, Iterable):
return [normalize(o) for o in obj]
return obj
return config_status(args=[], **normalize(sanitized_config))
return 0
def main(argv):
parser = argparse.ArgumentParser('Generate a file from a Python script',
add_help=False)
parser.add_argument('python_script',
metavar='python-script',
type=str,
help='The Python script to run')
parser.add_argument('method_name',
metavar='method-name',
type=str,
help='The method of the script to invoke')
parser.add_argument('output_file',
metavar='output-file',
type=str,
help='The file to generate')
parser.add_argument(
'dep_file',
metavar='dep-file',
type=str,
help='File to write any additional make dependencies to')
parser.add_argument(
'additional_arguments',
metavar='arg',
nargs='*',
help="Additional arguments to the script's main() method")
args = parser.parse_args(argv)
script = args.python_script
# Permit the script to import modules from the same directory in which it
# resides. The justification for doing this is that if we were invoking
# the script as:
#
# python script arg1...
#
# then importing modules from the script's directory would come for free.
# Since we're invoking the script in a roundabout way, we provide this
# bit of convenience.
sys.path.append(os.path.dirname(script))
with open(script, 'r') as fh:
module = imp.load_module('script', fh, script,
('.py', 'r', imp.PY_SOURCE))
method = args.method_name
if not hasattr(module, method):
print('Error: script "{0}" is missing a {1} method'.format(
script, method),
file=sys.stderr)
return 1
ret = 1
try:
with FileAvoidWrite(args.output_file) as output:
ret = module.__dict__[method](output, *args.additional_arguments)
# We treat sets as a statement of success. Everything else
# is an error (so scripts can conveniently |return 1| or
# similar).
if isinstance(ret, set) and ret:
ret |= set(
iter_modules_in_path(buildconfig.topsrcdir,
buildconfig.topobjdir))
mk = Makefile()
mk.create_rule([args.output_file]).add_dependencies(ret)
with FileAvoidWrite(args.dep_file) as dep_file:
mk.dump(dep_file)
# The script succeeded, so reset |ret| to indicate that.
ret = None
# Even when our file's contents haven't changed, we want to update
# the file's mtime so make knows this target isn't still older than
# whatever prerequisite caused it to be built this time around.
try:
os.utime(args.output_file, None)
except:
print('Error processing file "{0}"'.format(args.output_file),
file=sys.stderr)
traceback.print_exc()
except IOError as e:
print('Error opening file "{0}"'.format(e.filename), file=sys.stderr)
traceback.print_exc()
return 1
return ret
def process(input_dirs, inc_paths, bindings_conf, header_dir, xpcrs_dir,
xpt_dir, deps_dir, module, idl_files):
p = IDLParser()
xpts = []
mk = Makefile()
rule = mk.create_rule()
glbl = {}
exec(open(bindings_conf, encoding='utf-8').read(), glbl)
webidlconfig = glbl['DOMInterfaces']
# Write out dependencies for Python modules we import. If this list isn't
# up to date, we will not re-process XPIDL files if the processor changes.
rule.add_dependencies(
six.ensure_text(s) for s in iter_modules_in_path(topsrcdir))
for path in idl_files:
basename = os.path.basename(path)
stem, _ = os.path.splitext(basename)
idl_data = open(path, encoding='utf-8').read()
idl = p.parse(idl_data, filename=path)
idl.resolve(inc_paths, p, webidlconfig)
header_path = os.path.join(header_dir, '%s.h' % stem)
rs_rt_path = os.path.join(xpcrs_dir, 'rt', '%s.rs' % stem)
rs_bt_path = os.path.join(xpcrs_dir, 'bt', '%s.rs' % stem)
xpts.append(jsonxpt.build_typelib(idl))
rule.add_dependencies(six.ensure_text(s) for s in idl.deps)
# The print_* functions don't actually do anything with the
# passed-in path other than writing it into the file to let people
# know where the original source was. This script receives
# absolute paths, which are not so great to embed in header files
# (they mess with deterministic generation of files on different
# machines, Searchfox logic, shared compilation caches, etc.), so
# we pass in fake paths that are the same across compilations, but
# should still enable people to figure out where to go.
relpath = mozpath.relpath(path, topsrcdir)
with FileAvoidWrite(header_path) as fh:
print_header(idl, fh, path, relpath)
with FileAvoidWrite(rs_rt_path) as fh:
print_rust_bindings(idl, fh, relpath)
with FileAvoidWrite(rs_bt_path) as fh:
print_rust_macros_bindings(idl, fh, relpath)
# NOTE: We don't use FileAvoidWrite here as we may re-run this code due to a
# number of different changes in the code, which may not cause the .xpt
# files to be changed in any way. This means that make will re-run us every
# time a build is run whether or not anything changed. To fix this we
# unconditionally write out the file.
xpt_path = os.path.join(xpt_dir, '%s.xpt' % module)
with open(xpt_path, 'w', encoding='utf-8', newline='\n') as fh:
jsonxpt.write(jsonxpt.link(xpts), fh)
rule.add_targets([six.ensure_text(xpt_path)])
if deps_dir:
deps_path = os.path.join(deps_dir, '%s.pp' % module)
with FileAvoidWrite(deps_path) as fh:
mk.dump(fh)
def main(argv):
parser = argparse.ArgumentParser('Generate a file from a Python script',
add_help=False)
parser.add_argument('--locale', metavar='locale', type=str,
help='The locale in use.')
parser.add_argument('python_script', metavar='python-script', type=str,
help='The Python script to run')
parser.add_argument('method_name', metavar='method-name', type=str,
help='The method of the script to invoke')
parser.add_argument('output_file', metavar='output-file', type=str,
help='The file to generate')
parser.add_argument('dep_file', metavar='dep-file', type=str,
help='File to write any additional make dependencies to')
parser.add_argument('dep_target', metavar='dep-target', type=str,
help='Make target to use in the dependencies file')
parser.add_argument('additional_arguments', metavar='arg',
nargs=argparse.REMAINDER,
help="Additional arguments to the script's main() method")
args = parser.parse_args(argv)
kwargs = {}
if args.locale:
kwargs['locale'] = args.locale
script = args.python_script
# Permit the script to import modules from the same directory in which it
# resides. The justification for doing this is that if we were invoking
# the script as:
#
# python script arg1...
#
# then importing modules from the script's directory would come for free.
# Since we're invoking the script in a roundabout way, we provide this
# bit of convenience.
sys.path.append(os.path.dirname(script))
with open(script, 'r') as fh:
module = imp.load_module('script', fh, script,
('.py', 'r', imp.PY_SOURCE))
method = args.method_name
if not hasattr(module, method):
print('Error: script "{0}" is missing a {1} method'.format(script, method),
file=sys.stderr)
return 1
ret = 1
try:
with FileAvoidWrite(args.output_file, mode='rb') as output:
try:
ret = module.__dict__[method](output, *args.additional_arguments, **kwargs)
except Exception:
# Ensure that we don't overwrite the file if the script failed.
output.avoid_writing_to_file()
raise
# The following values indicate a statement of success:
# - a set() (see below)
# - 0
# - False
# - None
#
# Everything else is an error (so scripts can conveniently |return
# 1| or similar). If a set is returned, the elements of the set
# indicate additional dependencies that will be listed in the deps
# file. Python module imports are automatically included as
# dependencies.
if isinstance(ret, set):
deps = ret
# The script succeeded, so reset |ret| to indicate that.
ret = None
else:
deps = set()
# Only write out the dependencies if the script was successful
if not ret:
# Add dependencies on any python modules that were imported by
# the script.
deps |= set(iter_modules_in_path(buildconfig.topsrcdir,
buildconfig.topobjdir))
# Add dependencies on any buildconfig items that were accessed
# by the script.
deps |= set(buildconfig.get_dependencies())
mk = Makefile()
mk.create_rule([args.dep_target]).add_dependencies(deps)
with FileAvoidWrite(args.dep_file) as dep_file:
mk.dump(dep_file)
else:
# Ensure that we don't overwrite the file if the script failed.
output.avoid_writing_to_file()
except IOError as e:
print('Error opening file "{0}"'.format(e.filename), file=sys.stderr)
traceback.print_exc()
return 1
return ret
def config_status(config):
# Sanitize config data to feed config.status
# Ideally, all the backend and frontend code would handle the booleans, but
# there are so many things involved, that it's easier to keep config.status
# untouched for now.
def sanitized_bools(v):
if v is True:
return '1'
if v is False:
return ''
return v
sanitized_config = {}
sanitized_config['substs'] = {
k: sanitized_bools(v)
for k, v in config.iteritems()
if k not in ('DEFINES', 'non_global_defines', 'TOPSRCDIR', 'TOPOBJDIR',
'ALL_CONFIGURE_PATHS')
}
sanitized_config['defines'] = {
k: sanitized_bools(v)
for k, v in config['DEFINES'].iteritems()
}
sanitized_config['non_global_defines'] = config['non_global_defines']
sanitized_config['topsrcdir'] = config['TOPSRCDIR']
sanitized_config['topobjdir'] = config['TOPOBJDIR']
sanitized_config['mozconfig'] = config.get('MOZCONFIG')
# Create config.status. Eventually, we'll want to just do the work it does
# here, when we're able to skip configure tests/use cached results/not rely
# on autoconf.
print("Creating config.status", file=sys.stderr)
encoding = 'mbcs' if sys.platform == 'win32' else 'utf-8'
with codecs.open('config.status', 'w', encoding) as fh:
fh.write(
textwrap.dedent('''\
#!%(python)s
# coding=%(encoding)s
from __future__ import unicode_literals
from mozbuild.util import encode
encoding = '%(encoding)s'
''') % {
'python': config['PYTHON'],
'encoding': encoding
})
# A lot of the build backend code is currently expecting byte
# strings and breaks in subtle ways with unicode strings. (bug 1296508)
for k, v in sanitized_config.iteritems():
fh.write('%s = encode(%s, encoding)\n' % (k, indented_repr(v)))
fh.write("__all__ = ['topobjdir', 'topsrcdir', 'defines', "
"'non_global_defines', 'substs', 'mozconfig']")
if config.get('MOZ_BUILD_APP') != 'js' or config.get('JS_STANDALONE'):
fh.write(
textwrap.dedent('''
if __name__ == '__main__':
from mozbuild.util import patch_main
patch_main()
from mozbuild.config_status import config_status
args = dict([(name, globals()[name]) for name in __all__])
config_status(**args)
'''))
partial_config = PartialConfigEnvironment(config['TOPOBJDIR'])
partial_config.write_vars(sanitized_config)
# Write out a depfile so Make knows to re-run configure when relevant Python
# changes.
mk = Makefile()
rule = mk.create_rule()
rule.add_targets(["%s/config.status" % config['TOPOBJDIR']])
rule.add_dependencies(
itertools.chain(
config['ALL_CONFIGURE_PATHS'],
iter_modules_in_path(config['TOPOBJDIR'], config['TOPSRCDIR'])))
with open('configure.d', 'w') as fh:
mk.dump(fh)
# Other things than us are going to run this file, so we need to give it
# executable permissions.
os.chmod('config.status', 0o755)
if config.get('MOZ_BUILD_APP') != 'js' or config.get('JS_STANDALONE'):
from mozbuild.config_status import config_status
# Some values in sanitized_config also have more complex types, such as
# EnumString, which using when calling config_status would currently
# break the build, as well as making it inconsistent with re-running
# config.status. Fortunately, EnumString derives from unicode, so it's
# covered by converting unicode strings.
# A lot of the build backend code is currently expecting byte strings
# and breaks in subtle ways with unicode strings.
return config_status(args=[], **encode(sanitized_config, encoding))
return 0
msgMetadata = options.msgMetadata
headersdir = options.headersdir
cppdir = options.cppdir
includedirs = [os.path.abspath(incdir) for incdir in options.includedirs]
if not len(files):
op.error("No IPDL files specified")
ipcmessagestartpath = os.path.join(headersdir, 'IPCMessageStart.h')
ipc_msgtype_name_path = os.path.join(cppdir, 'IPCMessageTypeName.cpp')
# Compiling the IPDL files can take a long time, even on a fast machine.
# Check to see whether we need to do any work.
latestipdlmod = max(
os.stat(f).st_mtime for f in itertools.chain(
files, iter_modules_in_path(mozpath.dirname(__file__))))
def outputModTime(f):
# A non-existant file is newer than everything.
if not os.path.exists(f):
return 0
return os.stat(f).st_mtime
# Because the IPDL headers are placed into directories reflecting their
# namespace, collect a list here so we can easily map output names without
# parsing the actual IPDL files themselves.
headersmap = {}
for (path, dirs, headers) in os.walk(headersdir):
for h in headers:
