def _get_manager_args(self):
tmp = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, tmp)
cache_dir = mozpath.join(tmp, 'cache')
os.mkdir(cache_dir)
ip = self._static_input_paths
inputs = (
ip,
{mozpath.splitext(mozpath.basename(p))[0] for p in ip},
set(),
set(),
)
return dict(
config_path=self._config_path,
inputs=inputs,
exported_header_dir=mozpath.join(tmp, 'exports'),
codegen_dir=mozpath.join(tmp, 'codegen'),
state_path=mozpath.join(tmp, 'state.json'),
make_deps_path=mozpath.join(tmp, 'codegen.pp'),
make_deps_target='codegen.pp',
cache_dir=cache_dir,
)
def register_idl(self, idl, allow_existing=False):
"""Registers an IDL file with this instance.
The IDL file will be built, installed, etc.
"""
basename = mozpath.basename(idl.source_path)
root = mozpath.splitext(basename)[0]
xpt = "%s.xpt" % idl.module
manifest = mozpath.join(idl.install_target, "components", "interfaces.manifest")
chrome_manifest = mozpath.join(idl.install_target, "chrome.manifest")
entry = {
"source": idl.source_path,
"module": idl.module,
"basename": basename,
"root": root,
"manifest": manifest,
}
if not allow_existing and entry["basename"] in self.idls:
raise Exception("IDL already registered: %s" % entry["basename"])
self.idls[entry["basename"]] = entry
t = self.modules.setdefault(entry["module"], (idl.install_target, set()))
t[1].add(entry["root"])
if idl.add_to_manifest:
self.interface_manifests.setdefault(manifest, set()).add(xpt)
self.chrome_manifests.add(chrome_manifest)
def _handle_manifest_entry(self, entry, jars):
jarpath = None
if isinstance(entry, ManifestEntryWithRelPath) and \
urlparse(entry.relpath).scheme == 'jar':
jarpath, entry = self._unjarize(entry, entry.relpath)
elif isinstance(entry, ManifestResource) and \
urlparse(entry.target).scheme == 'jar':
jarpath, entry = self._unjarize(entry, entry.target)
if jarpath:
# Don't defer unpacking the jar file. If we already saw
# it, take (and remove) it from the registry. If we
# haven't, try to find it now.
if self.files.contains(jarpath):
jar = self.files[jarpath]
self.files.remove(jarpath)
else:
jar = [f for p, f in FileFinder.find(self, jarpath)]
assert len(jar) == 1
jar = jar[0]
if not jarpath in jars:
base = mozpath.splitext(jarpath)[0]
for j in self._open_jar(jarpath, jar):
self.files.add(mozpath.join(base,
j.filename),
DeflatedFile(j))
jars.add(jarpath)
self.kind = 'jar'
return entry
def _write_global_derived(self):
from Codegen import GlobalGenRoots
things = [('declare', f) for f in self.GLOBAL_DECLARE_FILES]
things.extend(('define', f) for f in self.GLOBAL_DEFINE_FILES)
result = (set(), set(), set())
for what, filename in things:
stem = mozpath.splitext(filename)[0]
root = getattr(GlobalGenRoots, stem)(self._config)
if what == 'declare':
code = root.declare()
output_root = self._exported_header_dir
elif what == 'define':
code = root.define()
output_root = self._codegen_dir
else:
raise Exception('Unknown global gen type: %s' % what)
output_path = mozpath.join(output_root, filename)
self._maybe_write_file(output_path, code, result)
return result
def _binding_info(self, p):
"""Compute binding metadata for an input path.
Returns a tuple of:
(stem, binding_stem, is_event, output_files)
output_files is itself a tuple. The first two items are the binding
header and C++ paths, respectively. The 2nd pair are the event header
and C++ paths or None if this isn't an event binding.
"""
basename = mozpath.basename(p)
stem = mozpath.splitext(basename)[0]
binding_stem = '%sBinding' % stem
if stem in self._exported_stems:
header_dir = self._exported_header_dir
else:
header_dir = self._codegen_dir
is_event = stem in self._generated_events_stems
files = (
mozpath.join(header_dir, '%s.h' % binding_stem),
mozpath.join(self._codegen_dir, '%s.cpp' % binding_stem),
mozpath.join(header_dir, '%s.h' % stem) if is_event else None,
mozpath.join(self._codegen_dir, '%s.cpp' % stem) if is_event else None,
)
return stem, binding_stem, is_event, header_dir, files
def register_idl(self, idl, allow_existing=False):
"""Registers an IDL file with this instance.
The IDL file will be built, installed, etc.
"""
basename = mozpath.basename(idl.source_path)
root = mozpath.splitext(basename)[0]
xpt = '%s.xpt' % idl.module
manifest = mozpath.join(idl.install_target, 'components', 'interfaces.manifest')
chrome_manifest = mozpath.join(idl.install_target, 'chrome.manifest')
entry = {
'source': idl.source_path,
'module': idl.module,
'basename': basename,
'root': root,
'manifest': manifest,
}
if not allow_existing and entry['basename'] in self.idls:
raise Exception('IDL already registered: %s' % entry['basename'])
self.idls[entry['basename']] = entry
t = self.modules.setdefault(entry['module'], (idl.install_target, set()))
t[1].add(entry['root'])
if idl.add_to_manifest:
self.interface_manifests.setdefault(manifest, set()).add(xpt)
self.chrome_manifests.add(chrome_manifest)
def files_from(ipdl):
base = mozpath.basename(ipdl)
root, ext = mozpath.splitext(base)
# Both .ipdl and .ipdlh become .cpp files
files = ["%s.cpp" % root]
if ext == ".ipdl":
# .ipdl also becomes Child/Parent.cpp files
files.extend(["%sChild.cpp" % root, "%sParent.cpp" % root])
return files
def _unjarize(self, entry, relpath):
'''
Transform a manifest entry pointing to chrome data in a jar in one
pointing to the corresponding unpacked path. Return the jar path and
the new entry.
'''
base = entry.base
jar, relpath = urlparse(relpath).path.split('!', 1)
entry = entry.rebase(mozpath.join(base, 'jar:%s!' % jar)) \
.move(mozpath.join(base, mozpath.splitext(jar)[0])) \
.rebase(base)
return mozpath.join(base, jar), entry
def _preprocess(self, backend_file, input_file, destdir=None):
cmd = self._py_action('preprocessor')
cmd.extend(backend_file.defines)
cmd.extend(['$(ACDEFINES)', '%f', '-o', '%o'])
base_input = mozpath.basename(input_file)
if base_input.endswith('.in'):
base_input = mozpath.splitext(base_input)[0]
output = mozpath.join(destdir, base_input) if destdir else base_input
backend_file.rule(
inputs=[input_file],
display='Preprocess %o',
cmd=cmd,
outputs=[output],
)
def register_idl(self, source, module, allow_existing=False):
"""Registers an IDL file with this instance.
The IDL file will be built, installed, etc.
"""
basename = mozpath.basename(source)
root = mozpath.splitext(basename)[0]
entry = {
'source': source,
'module': module,
'basename': basename,
'root': root,
}
if not allow_existing and entry['basename'] in self.idls:
raise Exception('IDL already registered: %' % entry['basename'])
self.idls[entry['basename']] = entry
self.modules.setdefault(entry['module'], set()).add(entry['root'])
def register_idl(self, source, module, allow_existing=False):
"""Registers an IDL file with this instance.
The IDL file will be built, installed, etc.
"""
basename = mozpath.basename(source)
root = mozpath.splitext(basename)[0]
entry = {
'source': source,
'module': module,
'basename': basename,
'root': root,
}
if not allow_existing and entry['basename'] in self.idls:
raise Exception('IDL already registered: %' % entry['basename'])
self.idls[entry['basename']] = entry
self.modules.setdefault(entry['module'], set()).add(entry['root'])
def _preprocess(self, backend_file, input_file, destdir=None):
# .css files use '%' as the preprocessor marker, which must be scaped as
# '%%' in the Tupfile.
marker = '%%' if input_file.endswith('.css') else '#'
cmd = self._py_action('preprocessor')
cmd.extend([shell_quote(d) for d in backend_file.defines])
cmd.extend(['$(ACDEFINES)', '%f', '-o', '%o', '--marker=%s' % marker])
base_input = mozpath.basename(input_file)
if base_input.endswith('.in'):
base_input = mozpath.splitext(base_input)[0]
output = mozpath.join(destdir, base_input) if destdir else base_input
backend_file.rule(
inputs=[input_file],
display='Preprocess %o',
cmd=cmd,
outputs=[output],
)
def register_idl(self, idl, allow_existing=False):
"""Registers an IDL file with this instance.
The IDL file will be built, installed, etc.
"""
basename = mozpath.basename(idl.source_path)
root = mozpath.splitext(basename)[0]
xpt = '%s.xpt' % idl.module
entry = {
'source': idl.source_path,
'module': idl.module,
'basename': basename,
'root': root,
}
if not allow_existing and entry['basename'] in self.idls:
raise Exception('IDL already registered: %s' % entry['basename'])
self.idls[entry['basename']] = entry
t = self.modules.setdefault(entry['module'], (idl.install_target, set()))
t[1].add(entry['root'])
def _write_global_derived(self):
things = [('declare', f) for f in self.GLOBAL_DECLARE_FILES]
things.extend(('define', f) for f in self.GLOBAL_DEFINE_FILES)
result = (set(), set(), set())
for what, filename in things:
stem = mozpath.splitext(filename)[0]
root = getattr(GlobalGenRoots, stem)(self._config)
if what == 'declare':
code = root.declare()
output_root = self._exported_header_dir
elif what == 'define':
code = root.define()
output_root = self._codegen_dir
else:
raise Exception('Unknown global gen type: %s' % what)
output_path = mozpath.join(output_root, filename)
self._maybe_write_file(output_path, code, result)
return result
def _preprocess(self, backend_file, input_file, destdir=None, target=None):
if target is None:
target = mozpath.basename(input_file)
# .css files use '%' as the preprocessor marker, which must be scaped as
# '%%' in the Tupfile.
marker = '%%' if target.endswith('.css') else '#'
cmd = self._py_action('preprocessor')
cmd.extend([shell_quote(d) for d in backend_file.defines])
cmd.extend(['$(ACDEFINES)', '%f', '-o', '%o', '--marker=%s' % marker])
base_input = mozpath.basename(target)
if base_input.endswith('.in'):
base_input = mozpath.splitext(base_input)[0]
output = mozpath.join(destdir, base_input) if destdir else base_input
backend_file.rule(
inputs=[input_file],
extra_inputs=[self._early_generated_files],
display='Preprocess %o',
cmd=cmd,
outputs=[output],
)
def read_from_gyp(config, path, output, vars, non_unified_sources = set()):
"""Read a gyp configuration and emits GypSandboxes for the backend to
process.
config is a ConfigEnvironment, path is the path to a root gyp configuration
file, output is the base path under which the objdir for the various gyp
dependencies will be, and vars a dict of variables to pass to the gyp
processor.
"""
time_start = time.time()
all_sources = set()
# gyp expects plain str instead of unicode. The frontend code gives us
# unicode strings, so convert them.
path = encode(path)
str_vars = dict((name, encode(value)) for name, value in vars.items())
params = {
b'parallel': False,
b'generator_flags': {},
b'build_files': [path],
}
# Files that gyp_chromium always includes
includes = [encode(mozpath.join(script_dir, 'common.gypi'))]
finder = FileFinder(chrome_src, find_executables=False)
includes.extend(encode(mozpath.join(chrome_src, name))
for name, _ in finder.find('*/supplement.gypi'))
# Read the given gyp file and its dependencies.
generator, flat_list, targets, data = \
gyp.Load([path], format=b'mozbuild',
default_variables=str_vars,
includes=includes,
depth=encode(mozpath.dirname(path)),
params=params)
# Process all targets from the given gyp files and its dependencies.
# The path given to AllTargets needs to use os.sep, while the frontend code
# gives us paths normalized with forward slash separator.
for target in gyp.common.AllTargets(flat_list, targets, path.replace(b'/', os.sep)):
build_file, target_name, toolset = gyp.common.ParseQualifiedTarget(target)
# The list of included files returned by gyp are relative to build_file
included_files = [mozpath.abspath(mozpath.join(mozpath.dirname(build_file), f))
for f in data[build_file]['included_files']]
# Emit a sandbox for each target.
sandbox = GypSandbox(mozpath.abspath(build_file), included_files)
sandbox.config = config
with sandbox.allow_all_writes() as d:
topsrcdir = d['TOPSRCDIR'] = config.topsrcdir
d['TOPOBJDIR'] = config.topobjdir
relsrcdir = d['RELATIVEDIR'] = mozpath.relpath(mozpath.dirname(build_file), config.topsrcdir)
d['SRCDIR'] = mozpath.join(topsrcdir, relsrcdir)
# Each target is given its own objdir. The base of that objdir
# is derived from the relative path from the root gyp file path
# to the current build_file, placed under the given output
# directory. Since several targets can be in a given build_file,
# separate them in subdirectories using the build_file basename
# and the target_name.
reldir = mozpath.relpath(mozpath.dirname(build_file),
mozpath.dirname(path))
subdir = '%s_%s' % (
mozpath.splitext(mozpath.basename(build_file))[0],
target_name,
)
d['OBJDIR'] = mozpath.join(output, reldir, subdir)
d['IS_GYP_DIR'] = True
spec = targets[target]
# Derive which gyp configuration to use based on MOZ_DEBUG.
c = 'Debug' if config.substs['MOZ_DEBUG'] else 'Release'
if c not in spec['configurations']:
raise RuntimeError('Missing %s gyp configuration for target %s '
'in %s' % (c, target_name, build_file))
target_conf = spec['configurations'][c]
if spec['type'] == 'none':
continue
elif spec['type'] == 'static_library':
sandbox['FORCE_STATIC_LIB'] = True
# Remove leading 'lib' from the target_name if any, and use as
# library name.
name = spec['target_name']
if name.startswith('lib'):
name = name[3:]
# The sandbox expects an unicode string.
sandbox['LIBRARY_NAME'] = name.decode('utf-8')
# gyp files contain headers and asm sources in sources lists.
sources = set(mozpath.normpath(mozpath.join(sandbox['SRCDIR'], f))
for f in spec.get('sources', [])
if mozpath.splitext(f)[-1] != '.h')
asm_sources = set(f for f in sources if f.endswith('.S'))
unified_sources = sources - non_unified_sources - asm_sources
sources -= unified_sources
all_sources |= sources
# The sandbox expects alphabetical order when adding sources
sandbox['SOURCES'] = alphabetical_sorted(sources)
sandbox['UNIFIED_SOURCES'] = alphabetical_sorted(unified_sources)
for define in target_conf.get('defines', []):
if '=' in define:
name, value = define.split('=', 1)
sandbox['DEFINES'][name] = value
else:
sandbox['DEFINES'][define] = True
for include in target_conf.get('include_dirs', []):
sandbox['LOCAL_INCLUDES'] += [include]
with sandbox.allow_all_writes() as d:
d['EXTRA_ASSEMBLER_FLAGS'] = target_conf.get('asflags_mozilla', [])
d['EXTRA_COMPILE_FLAGS'] = target_conf.get('cflags_mozilla', [])
else:
# Ignore other types than static_library because we don't have
# anything using them, and we're not testing them. They can be
# added when that becomes necessary.
raise NotImplementedError('Unsupported gyp target type: %s' % spec['type'])
sandbox.execution_time = time.time() - time_start
yield sandbox
time_start = time.time()
def _handle_idl_manager(self, manager):
if self.environment.is_artifact_build:
return
backend_file = self._get_backend_file('xpcom/xpidl')
backend_file.export_shell()
all_idl_directories = set()
all_idl_directories.update(
*map(lambda x: x[1], manager.modules.itervalues()))
all_xpts = []
for module, (idls, _) in sorted(manager.modules.iteritems()):
cmd = [
'$(PYTHON_PATH)',
'$(PLY_INCLUDE)',
'-I$(IDL_PARSER_DIR)',
'-I$(IDL_PARSER_CACHE_DIR)',
'$(topsrcdir)/python/mozbuild/mozbuild/action/xpidl-process.py',
'--cache-dir',
'$(IDL_PARSER_CACHE_DIR)',
'--bindings-conf',
'$(topsrcdir)/dom/bindings/Bindings.conf',
]
for d in all_idl_directories:
cmd.extend(['-I', d])
cmd.extend([
'$(DIST)/include',
'$(DIST)/xpcrs',
'.',
module,
])
cmd.extend(sorted(idls))
all_xpts.append('$(MOZ_OBJ_ROOT)/%s/%s.xpt' %
(backend_file.relobjdir, module))
outputs = ['%s.xpt' % module]
stems = sorted(
mozpath.splitext(mozpath.basename(idl))[0] for idl in idls)
outputs.extend(
['$(MOZ_OBJ_ROOT)/dist/include/%s.h' % f for f in stems])
outputs.extend(
['$(MOZ_OBJ_ROOT)/dist/xpcrs/rt/%s.rs' % f for f in stems])
outputs.extend(
['$(MOZ_OBJ_ROOT)/dist/xpcrs/bt/%s.rs' % f for f in stems])
backend_file.rule(
inputs=[
'$(MOZ_OBJ_ROOT)/xpcom/idl-parser/xpidl/xpidllex.py',
'$(MOZ_OBJ_ROOT)/xpcom/idl-parser/xpidl/xpidlyacc.py',
self._installed_idls,
],
display='XPIDL %s' % module,
cmd=cmd,
outputs=outputs,
output_group=self._installed_files,
check_unchanged=True,
)
cpp_backend_file = self._get_backend_file('xpcom/reflect/xptinfo')
cpp_backend_file.export_shell()
cpp_backend_file.rule(
inputs=all_xpts,
display='XPIDL xptcodegen.py %o',
cmd=[
'$(PYTHON_PATH)',
'$(PLY_INCLUDE)',
'$(topsrcdir)/xpcom/reflect/xptinfo/xptcodegen.py',
'%o',
'%f',
],
outputs=['xptdata.cpp'],
check_unchanged=True,
)
def process_gyp_result(gyp_result, gyp_dir_attrs, path, config, output,
non_unified_sources, action_overrides):
flat_list, targets, data = gyp_result
no_chromium = gyp_dir_attrs.no_chromium
no_unified = gyp_dir_attrs.no_unified
# Process all targets from the given gyp files and its dependencies.
# The path given to AllTargets needs to use os.sep, while the frontend code
# gives us paths normalized with forward slash separator.
for target in gyp.common.AllTargets(flat_list, targets, path.replace(b'/', os.sep)):
build_file, target_name, toolset = gyp.common.ParseQualifiedTarget(target)
# Each target is given its own objdir. The base of that objdir
# is derived from the relative path from the root gyp file path
# to the current build_file, placed under the given output
# directory. Since several targets can be in a given build_file,
# separate them in subdirectories using the build_file basename
# and the target_name.
reldir = mozpath.relpath(mozpath.dirname(build_file),
mozpath.dirname(path))
subdir = '%s_%s' % (
mozpath.splitext(mozpath.basename(build_file))[0],
target_name,
)
# Emit a context for each target.
context = GypContext(config, mozpath.relpath(
mozpath.join(output, reldir, subdir), config.topobjdir))
context.add_source(mozpath.abspath(build_file))
# The list of included files returned by gyp are relative to build_file
for f in data[build_file]['included_files']:
context.add_source(mozpath.abspath(mozpath.join(
mozpath.dirname(build_file), f)))
spec = targets[target]
# Derive which gyp configuration to use based on MOZ_DEBUG.
c = 'Debug' if config.substs['MOZ_DEBUG'] else 'Release'
if c not in spec['configurations']:
raise RuntimeError('Missing %s gyp configuration for target %s '
'in %s' % (c, target_name, build_file))
target_conf = spec['configurations'][c]
if 'actions' in spec:
handle_actions(spec['actions'], context, action_overrides)
if 'copies' in spec:
handle_copies(spec['copies'], context)
use_libs = []
libs = []
def add_deps(s):
for t in s.get('dependencies', []) + s.get('dependencies_original', []):
ty = targets[t]['type']
if ty in ('static_library', 'shared_library'):
use_libs.append(targets[t]['target_name'])
# Manually expand out transitive dependencies--
# gyp won't do this for static libs or none targets.
if ty in ('static_library', 'none'):
add_deps(targets[t])
libs.extend(spec.get('libraries', []))
#XXX: this sucks, but webrtc breaks with this right now because
# it builds a library called 'gtest' and we just get lucky
# that it isn't in USE_LIBS by that name anywhere.
if no_chromium:
add_deps(spec)
os_libs = []
for l in libs:
if l.startswith('-'):
os_libs.append(l)
elif l.endswith('.lib'):
os_libs.append(l[:-4])
elif l:
# For library names passed in from moz.build.
use_libs.append(os.path.basename(l))
if spec['type'] == 'none':
if not ('actions' in spec or 'copies' in spec):
continue
elif spec['type'] in ('static_library', 'shared_library', 'executable'):
# Remove leading 'lib' from the target_name if any, and use as
# library name.
name = spec['target_name']
if spec['type'] in ('static_library', 'shared_library'):
if name.startswith('lib'):
name = name[3:]
# The context expects an unicode string.
context['LIBRARY_NAME'] = name.decode('utf-8')
else:
context['PROGRAM'] = name.decode('utf-8')
if spec['type'] == 'shared_library':
context['FORCE_SHARED_LIB'] = True
elif spec['type'] == 'static_library' and spec.get('variables', {}).get('no_expand_libs', '0') == '1':
# PSM links a NSS static library, but our folded libnss
# doesn't actually export everything that all of the
# objects within would need, so that one library
# should be built as a real static library.
context['NO_EXPAND_LIBS'] = True
if use_libs:
context['USE_LIBS'] = sorted(use_libs, key=lambda s: s.lower())
if os_libs:
context['OS_LIBS'] = os_libs
# gyp files contain headers and asm sources in sources lists.
sources = []
unified_sources = []
extensions = set()
use_defines_in_asflags = False
for f in spec.get('sources', []):
ext = mozpath.splitext(f)[-1]
extensions.add(ext)
if f.startswith('$INTERMEDIATE_DIR/'):
s = ObjDirPath(context, f.replace('$INTERMEDIATE_DIR/', '!'))
else:
s = SourcePath(context, f)
if ext == '.h':
continue
if ext == '.def':
context['SYMBOLS_FILE'] = s
elif ext != '.S' and not no_unified and s not in non_unified_sources:
unified_sources.append(s)
else:
sources.append(s)
# The Mozilla build system doesn't use DEFINES for building
# ASFILES.
if ext == '.s':
use_defines_in_asflags = True
# The context expects alphabetical order when adding sources
context['SOURCES'] = alphabetical_sorted(sources)
context['UNIFIED_SOURCES'] = alphabetical_sorted(unified_sources)
defines = target_conf.get('defines', [])
if bool(config.substs['_MSC_VER']) and no_chromium:
msvs_settings = gyp.msvs_emulation.MsvsSettings(spec, {})
defines.extend(msvs_settings.GetComputedDefines(c))
for define in defines:
if '=' in define:
name, value = define.split('=', 1)
context['DEFINES'][name] = value
else:
context['DEFINES'][define] = True
product_dir_dist = '$PRODUCT_DIR/dist/'
for include in target_conf.get('include_dirs', []):
if include.startswith(product_dir_dist):
# special-case includes of <(PRODUCT_DIR)/dist/ to match
# handle_copies above. This is used for NSS' exports.
include = '!/dist/include/' + include[len(product_dir_dist):]
elif include.startswith(config.topobjdir):
# NSPR_INCLUDE_DIR gets passed into the NSS build this way.
include = '!/' + mozpath.relpath(include, config.topobjdir)
else:
# moz.build expects all LOCAL_INCLUDES to exist, so ensure they do.
#
# NB: gyp files sometimes have actual absolute paths (e.g.
# /usr/include32) and sometimes paths that moz.build considers
# absolute, i.e. starting from topsrcdir. There's no good way
# to tell them apart here, and the actual absolute paths are
# likely bogus. In any event, actual absolute paths will be
# filtered out by trying to find them in topsrcdir.
if include.startswith('/'):
resolved = mozpath.abspath(mozpath.join(config.topsrcdir, include[1:]))
else:
resolved = mozpath.abspath(mozpath.join(mozpath.dirname(build_file), include))
if not os.path.exists(resolved):
continue
context['LOCAL_INCLUDES'] += [include]
context['ASFLAGS'] = target_conf.get('asflags_mozilla', [])
if use_defines_in_asflags and defines:
context['ASFLAGS'] += ['-D' + d for d in defines]
flags = target_conf.get('cflags_mozilla', [])
if flags:
suffix_map = {
'.c': 'CFLAGS',
'.cpp': 'CXXFLAGS',
'.cc': 'CXXFLAGS',
'.m': 'CMFLAGS',
'.mm': 'CMMFLAGS',
}
variables = (
suffix_map[e]
for e in extensions if e in suffix_map
)
for var in variables:
for f in flags:
# We may be getting make variable references out of the
# gyp data, and we don't want those in emitted data, so
# substitute them with their actual value.
f = expand_variables(f, config.substs).split()
if not f:
continue
# the result may be a string or a list.
if isinstance(f, types.StringTypes):
context[var].append(f)
else:
context[var].extend(f)
else:
# Ignore other types because we don't have
# anything using them, and we're not testing them. They can be
# added when that becomes necessary.
raise NotImplementedError('Unsupported gyp target type: %s' % spec['type'])
if not no_chromium:
# Add some features to all contexts. Put here in case LOCAL_INCLUDES
# order matters.
context['LOCAL_INCLUDES'] += [
'!/ipc/ipdl/_ipdlheaders',
'/ipc/chromium/src',
'/ipc/glue',
]
# These get set via VC project file settings for normal GYP builds.
if config.substs['OS_TARGET'] == 'WINNT':
context['DEFINES']['UNICODE'] = True
context['DEFINES']['_UNICODE'] = True
context['DISABLE_STL_WRAPPING'] = True
for key, value in gyp_dir_attrs.sandbox_vars.items():
if context.get(key) and isinstance(context[key], list):
# If we have a key from sanbox_vars that's also been
# populated here we use the value from sandbox_vars as our
# basis rather than overriding outright.
context[key] = value + context[key]
else:
context[key] = value
yield context
def canonical_suffix_for_file(f):
return canonicalized_suffix_map[mozpath.splitext(f)[1]]
def test_splitext(self):
self.assertEqual(splitext(self.SEP.join(('foo', 'bar', 'baz.qux'))),
('foo/bar/baz', '.qux'))
def emit_from_sandbox(self, sandbox):
"""Convert a MozbuildSandbox to tree metadata objects.
This is a generator of mozbuild.frontend.data.SandboxDerived instances.
"""
# We always emit a directory traversal descriptor. This is needed by
# the recursive make backend.
for o in self._emit_directory_traversal_from_sandbox(sandbox):
yield o
for path in sandbox['CONFIGURE_SUBST_FILES']:
yield self._create_substitution(ConfigFileSubstitution, sandbox,
path)
for path in sandbox['CONFIGURE_DEFINE_FILES']:
yield self._create_substitution(HeaderFileSubstitution, sandbox,
path)
# XPIDL source files get processed and turned into .h and .xpt files.
# If there are multiple XPIDL files in a directory, they get linked
# together into a final .xpt, which has the name defined by
# XPIDL_MODULE.
xpidl_module = sandbox['XPIDL_MODULE']
if sandbox['XPIDL_SOURCES'] and not xpidl_module:
raise SandboxValidationError('XPIDL_MODULE must be defined if '
'XPIDL_SOURCES is defined.')
if xpidl_module and not sandbox['XPIDL_SOURCES']:
raise SandboxValidationError('XPIDL_MODULE cannot be defined '
'unless there are XPIDL_SOURCES: %s' %
sandbox['RELATIVEDIR'])
if sandbox['XPIDL_SOURCES'] and sandbox['NO_DIST_INSTALL']:
self.log(
logging.WARN, 'mozbuild_warning', dict(path=sandbox.main_path),
'{path}: NO_DIST_INSTALL has no effect on XPIDL_SOURCES.')
for idl in sandbox['XPIDL_SOURCES']:
yield XPIDLFile(sandbox, mozpath.join(sandbox['SRCDIR'], idl),
xpidl_module)
for symbol in ('SOURCES', 'HOST_SOURCES', 'UNIFIED_SOURCES'):
for src in (sandbox[symbol] or []):
if not os.path.exists(mozpath.join(sandbox['SRCDIR'], src)):
raise SandboxValidationError(
'Reference to a file that '
'doesn\'t exist in %s (%s) in %s' %
(symbol, src, sandbox['RELATIVEDIR']))
if sandbox.get('LIBXUL_LIBRARY') and sandbox.get('FORCE_STATIC_LIB'):
raise SandboxValidationError(
'LIBXUL_LIBRARY implies FORCE_STATIC_LIB')
# Proxy some variables as-is until we have richer classes to represent
# them. We should aim to keep this set small because it violates the
# desired abstraction of the build definition away from makefiles.
passthru = VariablePassthru(sandbox)
varlist = [
'ANDROID_GENERATED_RESFILES',
'ANDROID_RES_DIRS',
'CPP_UNIT_TESTS',
'EXPORT_LIBRARY',
'EXTRA_ASSEMBLER_FLAGS',
'EXTRA_COMPILE_FLAGS',
'EXTRA_COMPONENTS',
'EXTRA_JS_MODULES',
'EXTRA_PP_COMPONENTS',
'EXTRA_PP_JS_MODULES',
'FAIL_ON_WARNINGS',
'FILES_PER_UNIFIED_FILE',
'FORCE_SHARED_LIB',
'FORCE_STATIC_LIB',
'GENERATED_FILES',
'HOST_LIBRARY_NAME',
'IS_COMPONENT',
'IS_GYP_DIR',
'JS_MODULES_PATH',
'LIBS',
'LIBXUL_LIBRARY',
'MSVC_ENABLE_PGO',
'NO_DIST_INSTALL',
'OS_LIBS',
'SDK_LIBRARY',
]
for v in varlist:
if v in sandbox and sandbox[v]:
passthru.variables[v] = sandbox[v]
# NO_VISIBILITY_FLAGS is slightly different
if sandbox['NO_VISIBILITY_FLAGS']:
passthru.variables['VISIBILITY_FLAGS'] = ''
varmap = dict(SOURCES={
'.s': 'ASFILES',
'.asm': 'ASFILES',
'.c': 'CSRCS',
'.m': 'CMSRCS',
'.mm': 'CMMSRCS',
'.cc': 'CPPSRCS',
'.cpp': 'CPPSRCS',
'.S': 'SSRCS',
},
HOST_SOURCES={
'.c': 'HOST_CSRCS',
'.mm': 'HOST_CMMSRCS',
'.cc': 'HOST_CPPSRCS',
'.cpp': 'HOST_CPPSRCS',
},
UNIFIED_SOURCES={
'.c': 'UNIFIED_CSRCS',
'.mm': 'UNIFIED_CMMSRCS',
'.cc': 'UNIFIED_CPPSRCS',
'.cpp': 'UNIFIED_CPPSRCS',
})
varmap.update(
dict(('GENERATED_%s' % k, v) for k, v in varmap.items()
if k in ('SOURCES', 'UNIFIED_SOURCES')))
for variable, mapping in varmap.items():
for f in sandbox[variable]:
ext = mozpath.splitext(f)[1]
if ext not in mapping:
raise SandboxValidationError(
'%s has an unknown file type in %s' %
(f, sandbox['RELATIVEDIR']))
l = passthru.variables.setdefault(mapping[ext], [])
l.append(f)
if variable.startswith('GENERATED_'):
l = passthru.variables.setdefault('GARBAGE', [])
l.append(f)
no_pgo = sandbox.get('NO_PGO')
sources = sandbox.get('SOURCES', [])
no_pgo_sources = [f for f in sources if sources[f].no_pgo]
if no_pgo:
if no_pgo_sources:
raise SandboxValidationError(
'NO_PGO and SOURCES[...].no_pgo cannot be set at the same time'
)
passthru.variables['NO_PROFILE_GUIDED_OPTIMIZE'] = no_pgo
if no_pgo_sources:
passthru.variables['NO_PROFILE_GUIDED_OPTIMIZE'] = no_pgo_sources
exports = sandbox.get('EXPORTS')
if exports:
yield Exports(
sandbox,
exports,
dist_install=not sandbox.get('NO_DIST_INSTALL', False))
defines = sandbox.get('DEFINES')
if defines:
yield Defines(sandbox, defines)
program = sandbox.get('PROGRAM')
if program:
yield Program(sandbox, program, sandbox['CONFIG']['BIN_SUFFIX'])
program = sandbox.get('HOST_PROGRAM')
if program:
yield HostProgram(sandbox, program,
sandbox['CONFIG']['HOST_BIN_SUFFIX'])
for program in sandbox['SIMPLE_PROGRAMS']:
yield SimpleProgram(sandbox, program,
sandbox['CONFIG']['BIN_SUFFIX'])
for program in sandbox['HOST_SIMPLE_PROGRAMS']:
yield HostSimpleProgram(sandbox, program,
sandbox['CONFIG']['HOST_BIN_SUFFIX'])
simple_lists = [
('GENERATED_EVENTS_WEBIDL_FILES', GeneratedEventWebIDLFile),
('GENERATED_WEBIDL_FILES', GeneratedWebIDLFile),
('IPDL_SOURCES', IPDLFile),
('LOCAL_INCLUDES', LocalInclude),
('GENERATED_INCLUDES', GeneratedInclude),
('PREPROCESSED_TEST_WEBIDL_FILES', PreprocessedTestWebIDLFile),
('PREPROCESSED_WEBIDL_FILES', PreprocessedWebIDLFile),
('TEST_WEBIDL_FILES', TestWebIDLFile),
('WEBIDL_FILES', WebIDLFile),
('WEBIDL_EXAMPLE_INTERFACES', ExampleWebIDLInterface),
]
for sandbox_var, klass in simple_lists:
for name in sandbox.get(sandbox_var, []):
yield klass(sandbox, name)
if sandbox.get('FINAL_TARGET') or sandbox.get('XPI_NAME') or \
sandbox.get('DIST_SUBDIR'):
yield InstallationTarget(sandbox)
libname = sandbox.get('LIBRARY_NAME')
final_lib = sandbox.get('FINAL_LIBRARY')
if not libname and final_lib:
# If no LIBRARY_NAME is given, create one.
libname = sandbox['RELATIVEDIR'].replace('/', '_')
if libname:
self._libs.setdefault(libname, {})[sandbox['OBJDIR']] = \
LibraryDefinition(sandbox, libname)
if final_lib:
if isinstance(sandbox,
MozbuildSandbox) and sandbox.get('FORCE_STATIC_LIB'):
raise SandboxValidationError(
'FINAL_LIBRARY implies FORCE_STATIC_LIB')
self._final_libs.append((sandbox['OBJDIR'], libname, final_lib))
passthru.variables['FORCE_STATIC_LIB'] = True
# While there are multiple test manifests, the behavior is very similar
# across them. We enforce this by having common handling of all
# manifests and outputting a single class type with the differences
# described inside the instance.
#
# Keys are variable prefixes and values are tuples describing how these
# manifests should be handled:
#
# (flavor, install_prefix, active)
#
# flavor identifies the flavor of this test.
# install_prefix is the path prefix of where to install the files in
# the tests directory.
# active indicates whether to filter out inactive tests from the
# manifest.
#
# We ideally don't filter out inactive tests. However, not every test
# harness can yet deal with test filtering. Once all harnesses can do
# this, this feature can be dropped.
test_manifests = dict(
A11Y=('a11y', 'testing/mochitest/a11y', True),
BROWSER_CHROME=('browser-chrome', 'testing/mochitest/browser',
True),
METRO_CHROME=('metro-chrome', 'testing/mochitest/metro', True),
MOCHITEST=('mochitest', 'testing/mochitest/tests', True),
MOCHITEST_CHROME=('chrome', 'testing/mochitest/chrome', True),
MOCHITEST_WEBAPPRT_CHROME=('webapprt-chrome',
'testing/mochitest/webapprtChrome',
True),
WEBRTC_SIGNALLING_TEST=('steeplechase', 'steeplechase', True),
XPCSHELL_TESTS=('xpcshell', 'xpcshell', False),
)
for prefix, info in test_manifests.items():
for path in sandbox.get('%s_MANIFESTS' % prefix, []):
for obj in self._process_test_manifest(sandbox, info, path):
yield obj
jar_manifests = sandbox.get('JAR_MANIFESTS', [])
if len(jar_manifests) > 1:
raise SandboxValidationError(
'While JAR_MANIFESTS is a list, '
'it is currently limited to one value.')
for path in jar_manifests:
yield JARManifest(sandbox, mozpath.join(sandbox['SRCDIR'], path))
# Temporary test to look for jar.mn files that creep in without using
# the new declaration. Before, we didn't require jar.mn files to
# declared anywhere (they were discovered). This will detect people
# relying on the old behavior.
if os.path.exists(os.path.join(sandbox['SRCDIR'], 'jar.mn')):
if 'jar.mn' not in jar_manifests:
raise SandboxValidationError(
'A jar.mn exists in %s but it '
'is not referenced in the corresponding moz.build file. '
'Please define JAR_MANIFESTS in the moz.build file.' %
sandbox['SRCDIR'])
for name, jar in sandbox.get('JAVA_JAR_TARGETS', {}).items():
yield SandboxWrapped(sandbox, jar)
if passthru.variables:
yield passthru
def generated_events_stems(self):
return [
mozpath.splitext(b)[0] for b in self.generated_events_basenames()
]
def emit_from_context(self, context):
"""Convert a Context to tree metadata objects.
This is a generator of mozbuild.frontend.data.ContextDerived instances.
"""
# We only want to emit an InstallationTarget if one of the consulted
# variables is defined. Later on, we look up FINAL_TARGET, which has
# the side-effect of populating it. So, we need to do this lookup
# early.
if any(k in context
for k in ('FINAL_TARGET', 'XPI_NAME', 'DIST_SUBDIR')):
yield InstallationTarget(context)
# We always emit a directory traversal descriptor. This is needed by
# the recursive make backend.
for o in self._emit_directory_traversal_from_context(context):
yield o
for path in context['CONFIGURE_SUBST_FILES']:
yield self._create_substitution(ConfigFileSubstitution, context,
path)
for path in context['CONFIGURE_DEFINE_FILES']:
yield self._create_substitution(HeaderFileSubstitution, context,
path)
# XPIDL source files get processed and turned into .h and .xpt files.
# If there are multiple XPIDL files in a directory, they get linked
# together into a final .xpt, which has the name defined by
# XPIDL_MODULE.
xpidl_module = context['XPIDL_MODULE']
if context['XPIDL_SOURCES'] and not xpidl_module:
raise SandboxValidationError(
'XPIDL_MODULE must be defined if '
'XPIDL_SOURCES is defined.', context)
if xpidl_module and not context['XPIDL_SOURCES']:
raise SandboxValidationError(
'XPIDL_MODULE cannot be defined '
'unless there are XPIDL_SOURCES', context)
if context['XPIDL_SOURCES'] and context['NO_DIST_INSTALL']:
self.log(
logging.WARN, 'mozbuild_warning', dict(path=context.main_path),
'{path}: NO_DIST_INSTALL has no effect on XPIDL_SOURCES.')
for idl in context['XPIDL_SOURCES']:
yield XPIDLFile(context, mozpath.join(context.srcdir, idl),
xpidl_module)
for symbol in ('SOURCES', 'HOST_SOURCES', 'UNIFIED_SOURCES'):
for src in (context[symbol] or []):
if not os.path.exists(mozpath.join(context.srcdir, src)):
raise SandboxValidationError(
'File listed in %s does not '
'exist: \'%s\'' % (symbol, src), context)
# Proxy some variables as-is until we have richer classes to represent
# them. We should aim to keep this set small because it violates the
# desired abstraction of the build definition away from makefiles.
passthru = VariablePassthru(context)
varlist = [
'ANDROID_GENERATED_RESFILES',
'ANDROID_RES_DIRS',
'DISABLE_STL_WRAPPING',
'EXTRA_ASSEMBLER_FLAGS',
'EXTRA_COMPILE_FLAGS',
'EXTRA_COMPONENTS',
'EXTRA_DSO_LDOPTS',
'EXTRA_PP_COMPONENTS',
'FAIL_ON_WARNINGS',
'USE_STATIC_LIBS',
'IS_GYP_DIR',
'MSVC_ENABLE_PGO',
'NO_DIST_INSTALL',
'PYTHON_UNIT_TESTS',
'RCFILE',
'RESFILE',
'RCINCLUDE',
'DEFFILE',
'WIN32_EXE_LDFLAGS',
'LD_VERSION_SCRIPT',
]
for v in varlist:
if v in context and context[v]:
passthru.variables[v] = context[v]
if context.config.substs.get('OS_TARGET') == 'WINNT' and \
context['DELAYLOAD_DLLS']:
context['LDFLAGS'].extend([('-DELAYLOAD:%s' % dll)
for dll in context['DELAYLOAD_DLLS']])
context['OS_LIBS'].append('delayimp')
for v in ['CFLAGS', 'CXXFLAGS', 'CMFLAGS', 'CMMFLAGS', 'LDFLAGS']:
if v in context and context[v]:
passthru.variables['MOZBUILD_' + v] = context[v]
# NO_VISIBILITY_FLAGS is slightly different
if context['NO_VISIBILITY_FLAGS']:
passthru.variables['VISIBILITY_FLAGS'] = ''
no_pgo = context.get('NO_PGO')
sources = context.get('SOURCES', [])
no_pgo_sources = [f for f in sources if sources[f].no_pgo]
if no_pgo:
if no_pgo_sources:
raise SandboxValidationError(
'NO_PGO and SOURCES[...].no_pgo '
'cannot be set at the same time', context)
passthru.variables['NO_PROFILE_GUIDED_OPTIMIZE'] = no_pgo
if no_pgo_sources:
passthru.variables['NO_PROFILE_GUIDED_OPTIMIZE'] = no_pgo_sources
# A map from "canonical suffixes" for a particular source file
# language to the range of suffixes associated with that language.
#
# We deliberately don't list the canonical suffix in the suffix list
# in the definition; we'll add it in programmatically after defining
# things.
suffix_map = {
'.s': set(['.asm']),
'.c': set(),
'.m': set(),
'.mm': set(),
'.cpp': set(['.cc', '.cxx']),
'.S': set(),
}
# The inverse of the above, mapping suffixes to their canonical suffix.
canonicalized_suffix_map = {}
for suffix, alternatives in suffix_map.iteritems():
alternatives.add(suffix)
for a in alternatives:
canonicalized_suffix_map[a] = suffix
def canonical_suffix_for_file(f):
return canonicalized_suffix_map[mozpath.splitext(f)[1]]
# A map from moz.build variables to the canonical suffixes of file
# kinds that can be listed therein.
all_suffixes = list(suffix_map.keys())
varmap = dict(
SOURCES=(Sources, all_suffixes),
HOST_SOURCES=(HostSources, ['.c', '.mm', '.cpp']),
UNIFIED_SOURCES=(UnifiedSources, ['.c', '.mm', '.cpp']),
GENERATED_SOURCES=(GeneratedSources, all_suffixes),
)
for variable, (klass, suffixes) in varmap.items():
allowed_suffixes = set().union(*[suffix_map[s] for s in suffixes])
# First ensure that we haven't been given filetypes that we don't
# recognize.
for f in context[variable]:
ext = mozpath.splitext(f)[1]
if ext not in allowed_suffixes:
raise SandboxValidationError(
'%s has an unknown file type.' % f, context)
if variable.startswith('GENERATED_'):
l = passthru.variables.setdefault('GARBAGE', [])
l.append(f)
# Now sort the files to let groupby work.
sorted_files = sorted(context[variable],
key=canonical_suffix_for_file)
for canonical_suffix, files in itertools.groupby(
sorted_files, canonical_suffix_for_file):
arglist = [context, list(files), canonical_suffix]
if variable.startswith(
'UNIFIED_') and 'FILES_PER_UNIFIED_FILE' in context:
arglist.append(context['FILES_PER_UNIFIED_FILE'])
yield klass(*arglist)
sources_with_flags = [f for f in sources if sources[f].flags]
for f in sources_with_flags:
ext = mozpath.splitext(f)[1]
yield PerSourceFlag(context, f, sources[f].flags)
exports = context.get('EXPORTS')
if exports:
yield Exports(
context,
exports,
dist_install=not context.get('NO_DIST_INSTALL', False))
generated_files = context.get('GENERATED_FILES')
if generated_files:
for f in generated_files:
flags = generated_files[f]
output = f
if flags.script:
script = mozpath.join(context.srcdir, flags.script)
inputs = [
mozpath.join(context.srcdir, i) for i in flags.inputs
]
if not os.path.exists(script):
raise SandboxValidationError(
'Script for generating %s does not exist: %s' %
(f, script), context)
if os.path.splitext(script)[1] != '.py':
raise SandboxValidationError(
'Script for generating %s does not end in .py: %s'
% (f, script), context)
for i in inputs:
if not os.path.exists(i):
raise SandboxValidationError(
'Input for generating %s does not exist: %s' %
(f, i), context)
else:
script = None
inputs = []
yield GeneratedFile(context, script, output, inputs)
test_harness_files = context.get('TEST_HARNESS_FILES')
if test_harness_files:
srcdir_files = defaultdict(list)
srcdir_pattern_files = defaultdict(list)
objdir_files = defaultdict(list)
for path, strings in test_harness_files.walk():
if not path and strings:
raise SandboxValidationError(
'Cannot install files to the root of TEST_HARNESS_FILES',
context)
for s in strings:
if context.is_objdir_path(s):
if s.startswith('!/'):
objdir_files[path].append('$(DEPTH)/%s' % s[2:])
else:
objdir_files[path].append(s[1:])
else:
resolved = context.resolve_path(s)
if '*' in s:
srcdir_pattern_files[path].append(s)
elif not os.path.exists(resolved):
raise SandboxValidationError(
'File listed in TEST_HARNESS_FILES does not exist: %s'
% s, context)
else:
srcdir_files[path].append(resolved)
yield TestHarnessFiles(context, srcdir_files, srcdir_pattern_files,
objdir_files)
defines = context.get('DEFINES')
if defines:
yield Defines(context, defines)
resources = context.get('RESOURCE_FILES')
if resources:
yield Resources(context, resources, defines)
for pref in sorted(context['JS_PREFERENCE_FILES']):
yield JsPreferenceFile(context, pref)
for kind, cls in [('PROGRAM', Program), ('HOST_PROGRAM', HostProgram)]:
program = context.get(kind)
if program:
if program in self._binaries:
raise SandboxValidationError(
'Cannot use "%s" as %s name, '
'because it is already used in %s' %
(program, kind, self._binaries[program].relativedir),
context)
self._binaries[program] = cls(context, program)
self._linkage.append((context, self._binaries[program],
kind.replace('PROGRAM', 'USE_LIBS')))
for kind, cls in [('SIMPLE_PROGRAMS', SimpleProgram),
('CPP_UNIT_TESTS', SimpleProgram),
('HOST_SIMPLE_PROGRAMS', HostSimpleProgram)]:
for program in context[kind]:
if program in self._binaries:
raise SandboxValidationError(
'Cannot use "%s" in %s, '
'because it is already used in %s' %
(program, kind, self._binaries[program].relativedir),
context)
self._binaries[program] = cls(
context, program, is_unit_test=kind == 'CPP_UNIT_TESTS')
self._linkage.append(
(context, self._binaries[program], 'HOST_USE_LIBS'
if kind == 'HOST_SIMPLE_PROGRAMS' else 'USE_LIBS'))
extra_js_modules = context.get('EXTRA_JS_MODULES')
if extra_js_modules:
yield JavaScriptModules(context, extra_js_modules, 'extra')
extra_pp_js_modules = context.get('EXTRA_PP_JS_MODULES')
if extra_pp_js_modules:
yield JavaScriptModules(context, extra_pp_js_modules, 'extra_pp')
test_js_modules = context.get('TESTING_JS_MODULES')
if test_js_modules:
yield JavaScriptModules(context, test_js_modules, 'testing')
simple_lists = [
('GENERATED_EVENTS_WEBIDL_FILES', GeneratedEventWebIDLFile),
('GENERATED_WEBIDL_FILES', GeneratedWebIDLFile),
('IPDL_SOURCES', IPDLFile),
('GENERATED_INCLUDES', GeneratedInclude),
('PREPROCESSED_TEST_WEBIDL_FILES', PreprocessedTestWebIDLFile),
('PREPROCESSED_WEBIDL_FILES', PreprocessedWebIDLFile),
('TEST_WEBIDL_FILES', TestWebIDLFile),
('WEBIDL_FILES', WebIDLFile),
('WEBIDL_EXAMPLE_INTERFACES', ExampleWebIDLInterface),
]
for context_var, klass in simple_lists:
for name in context.get(context_var, []):
yield klass(context, name)
for local_include in context.get('LOCAL_INCLUDES', []):
if local_include.startswith('/'):
path = context.config.topsrcdir
relative_include = local_include[1:]
else:
path = context.srcdir
relative_include = local_include
actual_include = os.path.join(path, relative_include)
if not os.path.exists(actual_include):
raise SandboxValidationError(
'Path specified in LOCAL_INCLUDES '
'does not exist: %s (resolved to %s)' %
(local_include, actual_include), context)
yield LocalInclude(context, local_include)
final_target_files = context.get('FINAL_TARGET_FILES')
if final_target_files:
yield FinalTargetFiles(context, final_target_files,
context['FINAL_TARGET'])
branding_files = context.get('BRANDING_FILES')
if branding_files:
yield BrandingFiles(context, branding_files)
host_libname = context.get('HOST_LIBRARY_NAME')
libname = context.get('LIBRARY_NAME')
if host_libname:
if host_libname == libname:
raise SandboxValidationError(
'LIBRARY_NAME and '
'HOST_LIBRARY_NAME must have a different value', context)
lib = HostLibrary(context, host_libname)
self._libs[host_libname].append(lib)
self._linkage.append((context, lib, 'HOST_USE_LIBS'))
final_lib = context.get('FINAL_LIBRARY')
if not libname and final_lib:
# If no LIBRARY_NAME is given, create one.
libname = context.relsrcdir.replace('/', '_')
static_lib = context.get('FORCE_STATIC_LIB')
shared_lib = context.get('FORCE_SHARED_LIB')
static_name = context.get('STATIC_LIBRARY_NAME')
shared_name = context.get('SHARED_LIBRARY_NAME')
is_framework = context.get('IS_FRAMEWORK')
is_component = context.get('IS_COMPONENT')
soname = context.get('SONAME')
lib_defines = context.get('LIBRARY_DEFINES')
shared_args = {}
static_args = {}
if final_lib:
if static_lib:
raise SandboxValidationError(
'FINAL_LIBRARY implies FORCE_STATIC_LIB. '
'Please remove the latter.', context)
if shared_lib:
raise SandboxValidationError(
'FINAL_LIBRARY conflicts with FORCE_SHARED_LIB. '
'Please remove one.', context)
if is_framework:
raise SandboxValidationError(
'FINAL_LIBRARY conflicts with IS_FRAMEWORK. '
'Please remove one.', context)
if is_component:
raise SandboxValidationError(
'FINAL_LIBRARY conflicts with IS_COMPONENT. '
'Please remove one.', context)
static_args['link_into'] = final_lib
static_lib = True
if libname:
if is_component:
if static_lib:
raise SandboxValidationError(
'IS_COMPONENT conflicts with FORCE_STATIC_LIB. '
'Please remove one.', context)
shared_lib = True
shared_args['variant'] = SharedLibrary.COMPONENT
if is_framework:
if soname:
raise SandboxValidationError(
'IS_FRAMEWORK conflicts with SONAME. '
'Please remove one.', context)
shared_lib = True
shared_args['variant'] = SharedLibrary.FRAMEWORK
if not static_lib and not shared_lib:
static_lib = True
if static_name:
if not static_lib:
raise SandboxValidationError(
'STATIC_LIBRARY_NAME requires FORCE_STATIC_LIB',
context)
static_args['real_name'] = static_name
if shared_name:
if not shared_lib:
raise SandboxValidationError(
'SHARED_LIBRARY_NAME requires FORCE_SHARED_LIB',
context)
shared_args['real_name'] = shared_name
if soname:
if not shared_lib:
raise SandboxValidationError(
'SONAME requires FORCE_SHARED_LIB', context)
shared_args['soname'] = soname
# If both a shared and a static library are created, only the
# shared library is meant to be a SDK library.
if context.get('SDK_LIBRARY'):
if shared_lib:
shared_args['is_sdk'] = True
elif static_lib:
static_args['is_sdk'] = True
if shared_lib and static_lib:
if not static_name and not shared_name:
raise SandboxValidationError(
'Both FORCE_STATIC_LIB and FORCE_SHARED_LIB are True, '
'but neither STATIC_LIBRARY_NAME or '
'SHARED_LIBRARY_NAME is set. At least one is required.',
context)
if static_name and not shared_name and static_name == libname:
raise SandboxValidationError(
'Both FORCE_STATIC_LIB and FORCE_SHARED_LIB are True, '
'but STATIC_LIBRARY_NAME is the same as LIBRARY_NAME, '
'and SHARED_LIBRARY_NAME is unset. Please either '
'change STATIC_LIBRARY_NAME or LIBRARY_NAME, or set '
'SHARED_LIBRARY_NAME.', context)
if shared_name and not static_name and shared_name == libname:
raise SandboxValidationError(
'Both FORCE_STATIC_LIB and FORCE_SHARED_LIB are True, '
'but SHARED_LIBRARY_NAME is the same as LIBRARY_NAME, '
'and STATIC_LIBRARY_NAME is unset. Please either '
'change SHARED_LIBRARY_NAME or LIBRARY_NAME, or set '
'STATIC_LIBRARY_NAME.', context)
if shared_name and static_name and shared_name == static_name:
raise SandboxValidationError(
'Both FORCE_STATIC_LIB and FORCE_SHARED_LIB are True, '
'but SHARED_LIBRARY_NAME is the same as '
'STATIC_LIBRARY_NAME. Please change one of them.',
context)
if shared_lib:
lib = SharedLibrary(context, libname, **shared_args)
self._libs[libname].append(lib)
self._linkage.append((context, lib, 'USE_LIBS'))
if static_lib:
lib = StaticLibrary(context, libname, **static_args)
self._libs[libname].append(lib)
self._linkage.append((context, lib, 'USE_LIBS'))
if lib_defines:
if not libname:
raise SandboxValidationError(
'LIBRARY_DEFINES needs a '
'LIBRARY_NAME to take effect', context)
lib.defines.update(lib_defines)
# While there are multiple test manifests, the behavior is very similar
# across them. We enforce this by having common handling of all
# manifests and outputting a single class type with the differences
# described inside the instance.
#
# Keys are variable prefixes and values are tuples describing how these
# manifests should be handled:
#
# (flavor, install_prefix, package_tests)
#
# flavor identifies the flavor of this test.
# install_prefix is the path prefix of where to install the files in
# the tests directory.
# package_tests indicates whether to package test files into the test
# package; suites that compile the test files should not install
# them into the test package.
#
test_manifests = dict(
A11Y=('a11y', 'testing/mochitest', 'a11y', True),
BROWSER_CHROME=('browser-chrome', 'testing/mochitest', 'browser',
True),
ANDROID_INSTRUMENTATION=('instrumentation', 'instrumentation', '.',
False),
JETPACK_PACKAGE=('jetpack-package', 'testing/mochitest',
'jetpack-package', True),
JETPACK_ADDON=('jetpack-addon', 'testing/mochitest',
'jetpack-addon', False),
METRO_CHROME=('metro-chrome', 'testing/mochitest', 'metro', True),
MOCHITEST=('mochitest', 'testing/mochitest', 'tests', True),
MOCHITEST_CHROME=('chrome', 'testing/mochitest', 'chrome', True),
WEBRTC_SIGNALLING_TEST=('steeplechase', 'steeplechase', '.', True),
XPCSHELL_TESTS=('xpcshell', 'xpcshell', '.', True),
)
for prefix, info in test_manifests.items():
for path in context.get('%s_MANIFESTS' % prefix, []):
for obj in self._process_test_manifest(context, info, path):
yield obj
for flavor in ('crashtest', 'reftest'):
for path in context.get('%s_MANIFESTS' % flavor.upper(), []):
for obj in self._process_reftest_manifest(
context, flavor, path):
yield obj
jar_manifests = context.get('JAR_MANIFESTS', [])
if len(jar_manifests) > 1:
raise SandboxValidationError(
'While JAR_MANIFESTS is a list, '
'it is currently limited to one value.', context)
for path in jar_manifests:
yield JARManifest(context, mozpath.join(context.srcdir, path))
# Temporary test to look for jar.mn files that creep in without using
# the new declaration. Before, we didn't require jar.mn files to
# declared anywhere (they were discovered). This will detect people
# relying on the old behavior.
if os.path.exists(os.path.join(context.srcdir, 'jar.mn')):
if 'jar.mn' not in jar_manifests:
raise SandboxValidationError(
'A jar.mn exists but it '
'is not referenced in the moz.build file. '
'Please define JAR_MANIFESTS.', context)
for name, jar in context.get('JAVA_JAR_TARGETS', {}).items():
yield ContextWrapped(context, jar)
for name, data in context.get('ANDROID_ECLIPSE_PROJECT_TARGETS',
{}).items():
yield ContextWrapped(context, data)
if passthru.variables:
yield passthru
def _process_sources(self, context, passthru):
for symbol in ('SOURCES', 'HOST_SOURCES', 'UNIFIED_SOURCES'):
for src in (context[symbol] or []):
if not os.path.exists(mozpath.join(context.srcdir, src)):
raise SandboxValidationError('File listed in %s does not '
'exist: \'%s\'' % (symbol, src), context)
no_pgo = context.get('NO_PGO')
sources = context.get('SOURCES', [])
no_pgo_sources = [f for f in sources if sources[f].no_pgo]
if no_pgo:
if no_pgo_sources:
raise SandboxValidationError('NO_PGO and SOURCES[...].no_pgo '
'cannot be set at the same time', context)
passthru.variables['NO_PROFILE_GUIDED_OPTIMIZE'] = no_pgo
if no_pgo_sources:
passthru.variables['NO_PROFILE_GUIDED_OPTIMIZE'] = no_pgo_sources
# A map from "canonical suffixes" for a particular source file
# language to the range of suffixes associated with that language.
#
# We deliberately don't list the canonical suffix in the suffix list
# in the definition; we'll add it in programmatically after defining
# things.
suffix_map = {
'.s': set(['.asm']),
'.c': set(),
'.m': set(),
'.mm': set(),
'.cpp': set(['.cc', '.cxx']),
'.S': set(),
}
# The inverse of the above, mapping suffixes to their canonical suffix.
canonicalized_suffix_map = {}
for suffix, alternatives in suffix_map.iteritems():
alternatives.add(suffix)
for a in alternatives:
canonicalized_suffix_map[a] = suffix
def canonical_suffix_for_file(f):
return canonicalized_suffix_map[mozpath.splitext(f)[1]]
# A map from moz.build variables to the canonical suffixes of file
# kinds that can be listed therein.
all_suffixes = list(suffix_map.keys())
varmap = dict(
SOURCES=(Sources, all_suffixes),
HOST_SOURCES=(HostSources, ['.c', '.mm', '.cpp']),
UNIFIED_SOURCES=(UnifiedSources, ['.c', '.mm', '.cpp']),
GENERATED_SOURCES=(GeneratedSources, all_suffixes),
)
for variable, (klass, suffixes) in varmap.items():
allowed_suffixes = set().union(*[suffix_map[s] for s in suffixes])
# First ensure that we haven't been given filetypes that we don't
# recognize.
for f in context[variable]:
ext = mozpath.splitext(f)[1]
if ext not in allowed_suffixes:
raise SandboxValidationError(
'%s has an unknown file type.' % f, context)
if variable.startswith('GENERATED_'):
l = passthru.variables.setdefault('GARBAGE', [])
l.append(f)
# Now sort the files to let groupby work.
sorted_files = sorted(context[variable], key=canonical_suffix_for_file)
for canonical_suffix, files in itertools.groupby(sorted_files, canonical_suffix_for_file):
arglist = [context, list(files), canonical_suffix]
if variable.startswith('UNIFIED_') and 'FILES_PER_UNIFIED_FILE' in context:
arglist.append(context['FILES_PER_UNIFIED_FILE'])
yield klass(*arglist)
sources_with_flags = [f for f in sources if sources[f].flags]
for f in sources_with_flags:
ext = mozpath.splitext(f)[1]
yield PerSourceFlag(context, f, sources[f].flags)
def _process_webidl_basename(self, basename):
header = 'mozilla/dom/%sBinding.h' % mozpath.splitext(basename)[0]
self._install_manifests['dist_include'].add_optional_exists(header)
def _process_sources(self, context, passthru):
for symbol in ('SOURCES', 'HOST_SOURCES', 'UNIFIED_SOURCES'):
for src in (context[symbol] or []):
if not os.path.exists(mozpath.join(context.srcdir, src)):
raise SandboxValidationError('File listed in %s does not '
'exist: \'%s\'' % (symbol, src), context)
no_pgo = context.get('NO_PGO')
sources = context.get('SOURCES', [])
no_pgo_sources = [f for f in sources if sources[f].no_pgo]
if no_pgo:
if no_pgo_sources:
raise SandboxValidationError('NO_PGO and SOURCES[...].no_pgo '
'cannot be set at the same time', context)
passthru.variables['NO_PROFILE_GUIDED_OPTIMIZE'] = no_pgo
if no_pgo_sources:
passthru.variables['NO_PROFILE_GUIDED_OPTIMIZE'] = no_pgo_sources
# A map from "canonical suffixes" for a particular source file
# language to the range of suffixes associated with that language.
#
# We deliberately don't list the canonical suffix in the suffix list
# in the definition; we'll add it in programmatically after defining
# things.
suffix_map = {
'.s': set(['.asm']),
'.c': set(),
'.m': set(),
'.mm': set(),
'.cpp': set(['.cc', '.cxx']),
'.S': set(),
}
# The inverse of the above, mapping suffixes to their canonical suffix.
canonicalized_suffix_map = {}
for suffix, alternatives in suffix_map.iteritems():
alternatives.add(suffix)
for a in alternatives:
canonicalized_suffix_map[a] = suffix
def canonical_suffix_for_file(f):
return canonicalized_suffix_map[mozpath.splitext(f)[1]]
# A map from moz.build variables to the canonical suffixes of file
# kinds that can be listed therein.
all_suffixes = list(suffix_map.keys())
varmap = dict(
SOURCES=(Sources, all_suffixes),
HOST_SOURCES=(HostSources, ['.c', '.mm', '.cpp']),
UNIFIED_SOURCES=(UnifiedSources, ['.c', '.mm', '.cpp']),
GENERATED_SOURCES=(GeneratedSources, all_suffixes),
)
for variable, (klass, suffixes) in varmap.items():
allowed_suffixes = set().union(*[suffix_map[s] for s in suffixes])
# First ensure that we haven't been given filetypes that we don't
# recognize.
for f in context[variable]:
ext = mozpath.splitext(f)[1]
if ext not in allowed_suffixes:
raise SandboxValidationError(
'%s has an unknown file type.' % f, context)
if variable.startswith('GENERATED_'):
l = passthru.variables.setdefault('GARBAGE', [])
l.append(f)
# Now sort the files to let groupby work.
sorted_files = sorted(context[variable], key=canonical_suffix_for_file)
for canonical_suffix, files in itertools.groupby(sorted_files, canonical_suffix_for_file):
arglist = [context, list(files), canonical_suffix]
if variable.startswith('UNIFIED_') and 'FILES_PER_UNIFIED_FILE' in context:
arglist.append(context['FILES_PER_UNIFIED_FILE'])
yield klass(*arglist)
sources_with_flags = [f for f in sources if sources[f].flags]
for f in sources_with_flags:
ext = mozpath.splitext(f)[1]
yield PerSourceFlag(context, f, sources[f].flags)
def read_from_gyp(config, path, output, vars, non_unified_sources = set()):
"""Read a gyp configuration and emits GypContexts for the backend to
process.
config is a ConfigEnvironment, path is the path to a root gyp configuration
file, output is the base path under which the objdir for the various gyp
dependencies will be, and vars a dict of variables to pass to the gyp
processor.
"""
# gyp expects plain str instead of unicode. The frontend code gives us
# unicode strings, so convert them.
path = encode(path)
str_vars = dict((name, encode(value)) for name, value in vars.items())
params = {
b'parallel': False,
b'generator_flags': {},
b'build_files': [path],
}
# Files that gyp_chromium always includes
includes = [encode(mozpath.join(script_dir, 'common.gypi'))]
finder = FileFinder(chrome_src, find_executables=False)
includes.extend(encode(mozpath.join(chrome_src, name))
for name, _ in finder.find('*/supplement.gypi'))
# Read the given gyp file and its dependencies.
generator, flat_list, targets, data = \
gyp.Load([path], format=b'mozbuild',
default_variables=str_vars,
includes=includes,
depth=encode(mozpath.dirname(path)),
params=params)
# Process all targets from the given gyp files and its dependencies.
# The path given to AllTargets needs to use os.sep, while the frontend code
# gives us paths normalized with forward slash separator.
for target in gyp.common.AllTargets(flat_list, targets, path.replace(b'/', os.sep)):
build_file, target_name, toolset = gyp.common.ParseQualifiedTarget(target)
# Each target is given its own objdir. The base of that objdir
# is derived from the relative path from the root gyp file path
# to the current build_file, placed under the given output
# directory. Since several targets can be in a given build_file,
# separate them in subdirectories using the build_file basename
# and the target_name.
reldir = mozpath.relpath(mozpath.dirname(build_file),
mozpath.dirname(path))
subdir = '%s_%s' % (
mozpath.splitext(mozpath.basename(build_file))[0],
target_name,
)
# Emit a context for each target.
context = GypContext(config, mozpath.relpath(
mozpath.join(output, reldir, subdir), config.topobjdir))
context.add_source(mozpath.abspath(build_file))
# The list of included files returned by gyp are relative to build_file
for f in data[build_file]['included_files']:
context.add_source(mozpath.abspath(mozpath.join(
mozpath.dirname(build_file), f)))
spec = targets[target]
# Derive which gyp configuration to use based on MOZ_DEBUG.
c = 'Debug' if config.substs['MOZ_DEBUG'] else 'Release'
if c not in spec['configurations']:
raise RuntimeError('Missing %s gyp configuration for target %s '
'in %s' % (c, target_name, build_file))
target_conf = spec['configurations'][c]
if spec['type'] == 'none':
continue
elif spec['type'] == 'static_library':
# Remove leading 'lib' from the target_name if any, and use as
# library name.
name = spec['target_name']
if name.startswith('lib'):
name = name[3:]
# The context expects an unicode string.
context['LIBRARY_NAME'] = name.decode('utf-8')
# gyp files contain headers and asm sources in sources lists.
sources = []
unified_sources = []
extensions = set()
for f in spec.get('sources', []):
ext = mozpath.splitext(f)[-1]
extensions.add(ext)
s = SourcePath(context, f)
if ext == '.h':
continue
if ext != '.S' and s not in non_unified_sources:
unified_sources.append(s)
else:
sources.append(s)
# The context expects alphabetical order when adding sources
context['SOURCES'] = alphabetical_sorted(sources)
context['UNIFIED_SOURCES'] = alphabetical_sorted(unified_sources)
for define in target_conf.get('defines', []):
if '=' in define:
name, value = define.split('=', 1)
context['DEFINES'][name] = value
else:
context['DEFINES'][define] = True
for include in target_conf.get('include_dirs', []):
# moz.build expects all LOCAL_INCLUDES to exist, so ensure they do.
#
# NB: gyp files sometimes have actual absolute paths (e.g.
# /usr/include32) and sometimes paths that moz.build considers
# absolute, i.e. starting from topsrcdir. There's no good way
# to tell them apart here, and the actual absolute paths are
# likely bogus. In any event, actual absolute paths will be
# filtered out by trying to find them in topsrcdir.
if include.startswith('/'):
resolved = mozpath.abspath(mozpath.join(config.topsrcdir, include[1:]))
else:
resolved = mozpath.abspath(mozpath.join(mozpath.dirname(build_file), include))
if not os.path.exists(resolved):
continue
context['LOCAL_INCLUDES'] += [include]
context['ASFLAGS'] = target_conf.get('asflags_mozilla', [])
flags = target_conf.get('cflags_mozilla', [])
if flags:
suffix_map = {
'.c': 'CFLAGS',
'.cpp': 'CXXFLAGS',
'.cc': 'CXXFLAGS',
'.m': 'CMFLAGS',
'.mm': 'CMMFLAGS',
}
variables = (
suffix_map[e]
for e in extensions if e in suffix_map
)
for var in variables:
for f in flags:
# We may be getting make variable references out of the
# gyp data, and we don't want those in emitted data, so
# substitute them with their actual value.
f = expand_variables(f, config.substs)
if not f:
continue
# the result may be a string or a list.
if isinstance(f, types.StringTypes):
context[var].append(f)
else:
context[var].extend(f)
else:
# Ignore other types than static_library because we don't have
# anything using them, and we're not testing them. They can be
# added when that becomes necessary.
raise NotImplementedError('Unsupported gyp target type: %s' % spec['type'])
# Add some features to all contexts. Put here in case LOCAL_INCLUDES
# order matters.
context['LOCAL_INCLUDES'] += [
'!/ipc/ipdl/_ipdlheaders',
'/ipc/chromium/src',
'/ipc/glue',
]
# These get set via VC project file settings for normal GYP builds.
if config.substs['OS_TARGET'] == 'WINNT':
context['DEFINES']['UNICODE'] = True
context['DEFINES']['_UNICODE'] = True
context['DISABLE_STL_WRAPPING'] = True
yield context
def emit_from_context(self, context):
"""Convert a Context to tree metadata objects.
This is a generator of mozbuild.frontend.data.ContextDerived instances.
"""
# We always emit a directory traversal descriptor. This is needed by
# the recursive make backend.
for o in self._emit_directory_traversal_from_context(context): yield o
for path in context['CONFIGURE_SUBST_FILES']:
yield self._create_substitution(ConfigFileSubstitution, context,
path)
for path in context['CONFIGURE_DEFINE_FILES']:
yield self._create_substitution(HeaderFileSubstitution, context,
path)
# XPIDL source files get processed and turned into .h and .xpt files.
# If there are multiple XPIDL files in a directory, they get linked
# together into a final .xpt, which has the name defined by
# XPIDL_MODULE.
xpidl_module = context['XPIDL_MODULE']
if context['XPIDL_SOURCES'] and not xpidl_module:
raise SandboxValidationError('XPIDL_MODULE must be defined if '
'XPIDL_SOURCES is defined.', context)
if xpidl_module and not context['XPIDL_SOURCES']:
raise SandboxValidationError('XPIDL_MODULE cannot be defined '
'unless there are XPIDL_SOURCES', context)
if context['XPIDL_SOURCES'] and context['NO_DIST_INSTALL']:
self.log(logging.WARN, 'mozbuild_warning', dict(
path=context.main_path),
'{path}: NO_DIST_INSTALL has no effect on XPIDL_SOURCES.')
for idl in context['XPIDL_SOURCES']:
yield XPIDLFile(context, mozpath.join(context.srcdir, idl),
xpidl_module)
for symbol in ('SOURCES', 'HOST_SOURCES', 'UNIFIED_SOURCES'):
for src in (context[symbol] or []):
if not os.path.exists(mozpath.join(context.srcdir, src)):
raise SandboxValidationError('File listed in %s does not '
'exist: \'%s\'' % (symbol, src), context)
# Proxy some variables as-is until we have richer classes to represent
# them. We should aim to keep this set small because it violates the
# desired abstraction of the build definition away from makefiles.
passthru = VariablePassthru(context)
varlist = [
'ANDROID_GENERATED_RESFILES',
'ANDROID_RES_DIRS',
'DISABLE_STL_WRAPPING',
'EXTRA_ASSEMBLER_FLAGS',
'EXTRA_COMPILE_FLAGS',
'EXTRA_COMPONENTS',
'EXTRA_DSO_LDOPTS',
'EXTRA_PP_COMPONENTS',
'FAIL_ON_WARNINGS',
'FILES_PER_UNIFIED_FILE',
'USE_STATIC_LIBS',
'GENERATED_FILES',
'IS_GYP_DIR',
'MSVC_ENABLE_PGO',
'NO_DIST_INSTALL',
'PYTHON_UNIT_TESTS',
'RCFILE',
'RESFILE',
'RCINCLUDE',
'DEFFILE',
'WIN32_EXE_LDFLAGS',
'LD_VERSION_SCRIPT',
]
for v in varlist:
if v in context and context[v]:
passthru.variables[v] = context[v]
for v in ['CFLAGS', 'CXXFLAGS', 'CMFLAGS', 'CMMFLAGS', 'LDFLAGS']:
if v in context and context[v]:
passthru.variables['MOZBUILD_' + v] = context[v]
# NO_VISIBILITY_FLAGS is slightly different
if context['NO_VISIBILITY_FLAGS']:
passthru.variables['VISIBILITY_FLAGS'] = ''
if context['DELAYLOAD_DLLS']:
passthru.variables['DELAYLOAD_LDFLAGS'] = [('-DELAYLOAD:%s' % dll)
for dll in context['DELAYLOAD_DLLS']]
passthru.variables['USE_DELAYIMP'] = True
varmap = dict(
SOURCES={
'.s': 'ASFILES',
'.asm': 'ASFILES',
'.c': 'CSRCS',
'.m': 'CMSRCS',
'.mm': 'CMMSRCS',
'.cc': 'CPPSRCS',
'.cpp': 'CPPSRCS',
'.cxx': 'CPPSRCS',
'.S': 'SSRCS',
},
HOST_SOURCES={
'.c': 'HOST_CSRCS',
'.mm': 'HOST_CMMSRCS',
'.cc': 'HOST_CPPSRCS',
'.cpp': 'HOST_CPPSRCS',
'.cxx': 'HOST_CPPSRCS',
},
UNIFIED_SOURCES={
'.c': 'UNIFIED_CSRCS',
'.mm': 'UNIFIED_CMMSRCS',
'.cc': 'UNIFIED_CPPSRCS',
'.cpp': 'UNIFIED_CPPSRCS',
'.cxx': 'UNIFIED_CPPSRCS',
}
)
varmap.update(dict(('GENERATED_%s' % k, v) for k, v in varmap.items()
if k in ('SOURCES', 'UNIFIED_SOURCES')))
for variable, mapping in varmap.items():
for f in context[variable]:
ext = mozpath.splitext(f)[1]
if ext not in mapping:
raise SandboxValidationError(
'%s has an unknown file type.' % f, context)
l = passthru.variables.setdefault(mapping[ext], [])
l.append(f)
if variable.startswith('GENERATED_'):
l = passthru.variables.setdefault('GARBAGE', [])
l.append(f)
no_pgo = context.get('NO_PGO')
sources = context.get('SOURCES', [])
no_pgo_sources = [f for f in sources if sources[f].no_pgo]
if no_pgo:
if no_pgo_sources:
raise SandboxValidationError('NO_PGO and SOURCES[...].no_pgo '
'cannot be set at the same time', context)
passthru.variables['NO_PROFILE_GUIDED_OPTIMIZE'] = no_pgo
if no_pgo_sources:
passthru.variables['NO_PROFILE_GUIDED_OPTIMIZE'] = no_pgo_sources
sources_with_flags = [f for f in sources if sources[f].flags]
for f in sources_with_flags:
ext = mozpath.splitext(f)[1]
yield PerSourceFlag(context, f, sources[f].flags)
exports = context.get('EXPORTS')
if exports:
yield Exports(context, exports,
dist_install=not context.get('NO_DIST_INSTALL', False))
defines = context.get('DEFINES')
if defines:
yield Defines(context, defines)
resources = context.get('RESOURCE_FILES')
if resources:
yield Resources(context, resources, defines)
for kind, cls in [('PROGRAM', Program), ('HOST_PROGRAM', HostProgram)]:
program = context.get(kind)
if program:
if program in self._binaries:
raise SandboxValidationError(
'Cannot use "%s" as %s name, '
'because it is already used in %s' % (program, kind,
self._binaries[program].relativedir), context)
self._binaries[program] = cls(context, program)
self._linkage.append((context, self._binaries[program],
kind.replace('PROGRAM', 'USE_LIBS')))
for kind, cls in [
('SIMPLE_PROGRAMS', SimpleProgram),
('CPP_UNIT_TESTS', SimpleProgram),
('HOST_SIMPLE_PROGRAMS', HostSimpleProgram)]:
for program in context[kind]:
if program in self._binaries:
raise SandboxValidationError(
'Cannot use "%s" in %s, '
'because it is already used in %s' % (program, kind,
self._binaries[program].relativedir), context)
self._binaries[program] = cls(context, program,
is_unit_test=kind == 'CPP_UNIT_TESTS')
self._linkage.append((context, self._binaries[program],
'HOST_USE_LIBS' if kind == 'HOST_SIMPLE_PROGRAMS'
else 'USE_LIBS'))
extra_js_modules = context.get('EXTRA_JS_MODULES')
if extra_js_modules:
yield JavaScriptModules(context, extra_js_modules, 'extra')
extra_pp_js_modules = context.get('EXTRA_PP_JS_MODULES')
if extra_pp_js_modules:
yield JavaScriptModules(context, extra_pp_js_modules, 'extra_pp')
test_js_modules = context.get('TESTING_JS_MODULES')
if test_js_modules:
yield JavaScriptModules(context, test_js_modules, 'testing')
simple_lists = [
('GENERATED_EVENTS_WEBIDL_FILES', GeneratedEventWebIDLFile),
('GENERATED_WEBIDL_FILES', GeneratedWebIDLFile),
('IPDL_SOURCES', IPDLFile),
('GENERATED_INCLUDES', GeneratedInclude),
('PREPROCESSED_TEST_WEBIDL_FILES', PreprocessedTestWebIDLFile),
('PREPROCESSED_WEBIDL_FILES', PreprocessedWebIDLFile),
('TEST_WEBIDL_FILES', TestWebIDLFile),
('WEBIDL_FILES', WebIDLFile),
('WEBIDL_EXAMPLE_INTERFACES', ExampleWebIDLInterface),
]
for context_var, klass in simple_lists:
for name in context.get(context_var, []):
yield klass(context, name)
for local_include in context.get('LOCAL_INCLUDES', []):
if local_include.startswith('/'):
path = context.config.topsrcdir
relative_include = local_include[1:]
else:
path = context.srcdir
relative_include = local_include
actual_include = os.path.join(path, relative_include)
if not os.path.exists(actual_include):
raise SandboxValidationError('Path specified in LOCAL_INCLUDES '
'does not exist: %s (resolved to %s)' % (local_include, actual_include), context)
yield LocalInclude(context, local_include)
if context.get('FINAL_TARGET') or context.get('XPI_NAME') or \
context.get('DIST_SUBDIR'):
yield InstallationTarget(context)
host_libname = context.get('HOST_LIBRARY_NAME')
libname = context.get('LIBRARY_NAME')
if host_libname:
if host_libname == libname:
raise SandboxValidationError('LIBRARY_NAME and '
'HOST_LIBRARY_NAME must have a different value', context)
lib = HostLibrary(context, host_libname)
self._libs[host_libname].append(lib)
self._linkage.append((context, lib, 'HOST_USE_LIBS'))
final_lib = context.get('FINAL_LIBRARY')
if not libname and final_lib:
# If no LIBRARY_NAME is given, create one.
libname = context.relsrcdir.replace('/', '_')
static_lib = context.get('FORCE_STATIC_LIB')
shared_lib = context.get('FORCE_SHARED_LIB')
static_name = context.get('STATIC_LIBRARY_NAME')
shared_name = context.get('SHARED_LIBRARY_NAME')
is_framework = context.get('IS_FRAMEWORK')
is_component = context.get('IS_COMPONENT')
soname = context.get('SONAME')
shared_args = {}
static_args = {}
if final_lib:
if static_lib:
raise SandboxValidationError(
'FINAL_LIBRARY implies FORCE_STATIC_LIB. '
'Please remove the latter.', context)
if shared_lib:
raise SandboxValidationError(
'FINAL_LIBRARY conflicts with FORCE_SHARED_LIB. '
'Please remove one.', context)
if is_framework:
raise SandboxValidationError(
'FINAL_LIBRARY conflicts with IS_FRAMEWORK. '
'Please remove one.', context)
if is_component:
raise SandboxValidationError(
'FINAL_LIBRARY conflicts with IS_COMPONENT. '
'Please remove one.', context)
static_args['link_into'] = final_lib
static_lib = True
if libname:
if is_component:
if shared_lib:
raise SandboxValidationError(
'IS_COMPONENT implies FORCE_SHARED_LIB. '
'Please remove the latter.', context)
if is_framework:
raise SandboxValidationError(
'IS_COMPONENT conflicts with IS_FRAMEWORK. '
'Please remove one.', context)
if static_lib:
raise SandboxValidationError(
'IS_COMPONENT conflicts with FORCE_STATIC_LIB. '
'Please remove one.', context)
shared_lib = True
shared_args['variant'] = SharedLibrary.COMPONENT
if is_framework:
if shared_lib:
raise SandboxValidationError(
'IS_FRAMEWORK implies FORCE_SHARED_LIB. '
'Please remove the latter.', context)
if soname:
raise SandboxValidationError(
'IS_FRAMEWORK conflicts with SONAME. '
'Please remove one.', context)
shared_lib = True
shared_args['variant'] = SharedLibrary.FRAMEWORK
if static_name:
if not static_lib:
raise SandboxValidationError(
'STATIC_LIBRARY_NAME requires FORCE_STATIC_LIB',
context)
static_args['real_name'] = static_name
if shared_name:
if not shared_lib:
raise SandboxValidationError(
'SHARED_LIBRARY_NAME requires FORCE_SHARED_LIB',
context)
shared_args['real_name'] = shared_name
if soname:
if not shared_lib:
raise SandboxValidationError(
'SONAME requires FORCE_SHARED_LIB', context)
shared_args['soname'] = soname
if not static_lib and not shared_lib:
static_lib = True
# If both a shared and a static library are created, only the
# shared library is meant to be a SDK library.
if context.get('SDK_LIBRARY'):
if shared_lib:
shared_args['is_sdk'] = True
elif static_lib:
static_args['is_sdk'] = True
if shared_lib and static_lib:
if not static_name and not shared_name:
raise SandboxValidationError(
'Both FORCE_STATIC_LIB and FORCE_SHARED_LIB are True, '
'but neither STATIC_LIBRARY_NAME or '
'SHARED_LIBRARY_NAME is set. At least one is required.',
context)
if static_name and not shared_name and static_name == libname:
raise SandboxValidationError(
'Both FORCE_STATIC_LIB and FORCE_SHARED_LIB are True, '
'but STATIC_LIBRARY_NAME is the same as LIBRARY_NAME, '
'and SHARED_LIBRARY_NAME is unset. Please either '
'change STATIC_LIBRARY_NAME or LIBRARY_NAME, or set '
'SHARED_LIBRARY_NAME.', context)
if shared_name and not static_name and shared_name == libname:
raise SandboxValidationError(
'Both FORCE_STATIC_LIB and FORCE_SHARED_LIB are True, '
'but SHARED_LIBRARY_NAME is the same as LIBRARY_NAME, '
'and STATIC_LIBRARY_NAME is unset. Please either '
'change SHARED_LIBRARY_NAME or LIBRARY_NAME, or set '
'STATIC_LIBRARY_NAME.', context)
if shared_name and static_name and shared_name == static_name:
raise SandboxValidationError(
'Both FORCE_STATIC_LIB and FORCE_SHARED_LIB are True, '
'but SHARED_LIBRARY_NAME is the same as '
'STATIC_LIBRARY_NAME. Please change one of them.',
context)
if shared_lib:
lib = SharedLibrary(context, libname, **shared_args)
self._libs[libname].append(lib)
self._linkage.append((context, lib, 'USE_LIBS'))
if static_lib:
lib = StaticLibrary(context, libname, **static_args)
self._libs[libname].append(lib)
self._linkage.append((context, lib, 'USE_LIBS'))
# While there are multiple test manifests, the behavior is very similar
# across them. We enforce this by having common handling of all
# manifests and outputting a single class type with the differences
# described inside the instance.
#
# Keys are variable prefixes and values are tuples describing how these
# manifests should be handled:
#
# (flavor, install_prefix, active)
#
# flavor identifies the flavor of this test.
# install_prefix is the path prefix of where to install the files in
# the tests directory.
# active indicates whether to filter out inactive tests from the
# manifest.
#
# We ideally don't filter out inactive tests. However, not every test
# harness can yet deal with test filtering. Once all harnesses can do
# this, this feature can be dropped.
test_manifests = dict(
A11Y=('a11y', 'testing/mochitest', 'a11y', True),
BROWSER_CHROME=('browser-chrome', 'testing/mochitest', 'browser', True),
METRO_CHROME=('metro-chrome', 'testing/mochitest', 'metro', True),
MOCHITEST=('mochitest', 'testing/mochitest', 'tests', True),
MOCHITEST_CHROME=('chrome', 'testing/mochitest', 'chrome', True),
MOCHITEST_WEBAPPRT_CHROME=('webapprt-chrome', 'testing/mochitest', 'webapprtChrome', True),
WEBRTC_SIGNALLING_TEST=('steeplechase', 'steeplechase', '.', True),
XPCSHELL_TESTS=('xpcshell', 'xpcshell', '.', False),
)
for prefix, info in test_manifests.items():
for path in context.get('%s_MANIFESTS' % prefix, []):
for obj in self._process_test_manifest(context, info, path):
yield obj
for flavor in ('crashtest', 'reftest'):
for path in context.get('%s_MANIFESTS' % flavor.upper(), []):
for obj in self._process_reftest_manifest(context, flavor, path):
yield obj
jar_manifests = context.get('JAR_MANIFESTS', [])
if len(jar_manifests) > 1:
raise SandboxValidationError('While JAR_MANIFESTS is a list, '
'it is currently limited to one value.', context)
for path in jar_manifests:
yield JARManifest(context, mozpath.join(context.srcdir, path))
# Temporary test to look for jar.mn files that creep in without using
# the new declaration. Before, we didn't require jar.mn files to
# declared anywhere (they were discovered). This will detect people
# relying on the old behavior.
if os.path.exists(os.path.join(context.srcdir, 'jar.mn')):
if 'jar.mn' not in jar_manifests:
raise SandboxValidationError('A jar.mn exists but it '
'is not referenced in the moz.build file. '
'Please define JAR_MANIFESTS.', context)
for name, jar in context.get('JAVA_JAR_TARGETS', {}).items():
yield ContextWrapped(context, jar)
for name, data in context.get('ANDROID_ECLIPSE_PROJECT_TARGETS', {}).items():
yield ContextWrapped(context, data)
if passthru.variables:
yield passthru
def all_regular_bindinggen_stems(self):
for stem in self.all_regular_stems():
yield '%sBinding' % stem
for source in self.generated_events_sources:
yield mozpath.splitext(mozpath.basename(source))[0]
def read_from_gyp(config, path, output, vars, non_unified_sources=set()):
"""Read a gyp configuration and emits GypContexts for the backend to
process.
config is a ConfigEnvironment, path is the path to a root gyp configuration
file, output is the base path under which the objdir for the various gyp
dependencies will be, and vars a dict of variables to pass to the gyp
processor.
"""
time_start = time.time()
# gyp expects plain str instead of unicode. The frontend code gives us
# unicode strings, so convert them.
path = encode(path)
str_vars = dict((name, encode(value)) for name, value in vars.items())
params = {b"parallel": False, b"generator_flags": {}, b"build_files": [path]}
# Files that gyp_chromium always includes
includes = [encode(mozpath.join(script_dir, "common.gypi"))]
finder = FileFinder(chrome_src, find_executables=False)
includes.extend(encode(mozpath.join(chrome_src, name)) for name, _ in finder.find("*/supplement.gypi"))
# Read the given gyp file and its dependencies.
generator, flat_list, targets, data = gyp.Load(
[path],
format=b"mozbuild",
default_variables=str_vars,
includes=includes,
depth=encode(mozpath.dirname(path)),
params=params,
)
# Process all targets from the given gyp files and its dependencies.
# The path given to AllTargets needs to use os.sep, while the frontend code
# gives us paths normalized with forward slash separator.
for target in gyp.common.AllTargets(flat_list, targets, path.replace(b"/", os.sep)):
build_file, target_name, toolset = gyp.common.ParseQualifiedTarget(target)
# Each target is given its own objdir. The base of that objdir
# is derived from the relative path from the root gyp file path
# to the current build_file, placed under the given output
# directory. Since several targets can be in a given build_file,
# separate them in subdirectories using the build_file basename
# and the target_name.
reldir = mozpath.relpath(mozpath.dirname(build_file), mozpath.dirname(path))
subdir = "%s_%s" % (mozpath.splitext(mozpath.basename(build_file))[0], target_name)
# Emit a context for each target.
context = GypContext(config, mozpath.relpath(mozpath.join(output, reldir, subdir), config.topobjdir))
context.add_source(mozpath.abspath(build_file))
# The list of included files returned by gyp are relative to build_file
for f in data[build_file]["included_files"]:
context.add_source(mozpath.abspath(mozpath.join(mozpath.dirname(build_file), f)))
spec = targets[target]
# Derive which gyp configuration to use based on MOZ_DEBUG.
c = "Debug" if config.substs["MOZ_DEBUG"] else "Release"
if c not in spec["configurations"]:
raise RuntimeError("Missing %s gyp configuration for target %s " "in %s" % (c, target_name, build_file))
target_conf = spec["configurations"][c]
if spec["type"] == "none":
continue
elif spec["type"] == "static_library":
# Remove leading 'lib' from the target_name if any, and use as
# library name.
name = spec["target_name"]
if name.startswith("lib"):
name = name[3:]
# The context expects an unicode string.
context["LIBRARY_NAME"] = name.decode("utf-8")
# gyp files contain headers and asm sources in sources lists.
sources = []
unified_sources = []
extensions = set()
for f in spec.get("sources", []):
ext = mozpath.splitext(f)[-1]
extensions.add(ext)
s = SourcePath(context, f)
if ext == ".h":
continue
if ext != ".S" and s not in non_unified_sources:
unified_sources.append(s)
else:
sources.append(s)
# The context expects alphabetical order when adding sources
context["SOURCES"] = alphabetical_sorted(sources)
context["UNIFIED_SOURCES"] = alphabetical_sorted(unified_sources)
for define in target_conf.get("defines", []):
if "=" in define:
name, value = define.split("=", 1)
context["DEFINES"][name] = value
else:
context["DEFINES"][define] = True
for include in target_conf.get("include_dirs", []):
# moz.build expects all LOCAL_INCLUDES to exist, so ensure they do.
#
# NB: gyp files sometimes have actual absolute paths (e.g.
# /usr/include32) and sometimes paths that moz.build considers
# absolute, i.e. starting from topsrcdir. There's no good way
# to tell them apart here, and the actual absolute paths are
# likely bogus. In any event, actual absolute paths will be
# filtered out by trying to find them in topsrcdir.
if include.startswith("/"):
resolved = mozpath.abspath(mozpath.join(config.topsrcdir, include[1:]))
else:
resolved = mozpath.abspath(mozpath.join(mozpath.dirname(build_file), include))
if not os.path.exists(resolved):
continue
context["LOCAL_INCLUDES"] += [include]
context["ASFLAGS"] = target_conf.get("asflags_mozilla", [])
flags = target_conf.get("cflags_mozilla", [])
if flags:
suffix_map = {".c": "CFLAGS", ".cpp": "CXXFLAGS", ".cc": "CXXFLAGS", ".m": "CMFLAGS", ".mm": "CMMFLAGS"}
variables = (suffix_map[e] for e in extensions if e in suffix_map)
for var in variables:
context[var].extend(flags)
else:
# Ignore other types than static_library because we don't have
# anything using them, and we're not testing them. They can be
# added when that becomes necessary.
raise NotImplementedError("Unsupported gyp target type: %s" % spec["type"])
# Add some features to all contexts. Put here in case LOCAL_INCLUDES
# order matters.
context["LOCAL_INCLUDES"] += ["/ipc/chromium/src", "/ipc/glue"]
context["GENERATED_INCLUDES"] += ["/ipc/ipdl/_ipdlheaders"]
# These get set via VC project file settings for normal GYP builds.
if config.substs["OS_TARGET"] == "WINNT":
context["DEFINES"]["UNICODE"] = True
context["DEFINES"]["_UNICODE"] = True
context["DISABLE_STL_WRAPPING"] = True
context.execution_time = time.time() - time_start
yield context
time_start = time.time()
def emit_from_sandbox(self, sandbox):
"""Convert a MozbuildSandbox to tree metadata objects.
This is a generator of mozbuild.frontend.data.SandboxDerived instances.
"""
# We always emit a directory traversal descriptor. This is needed by
# the recursive make backend.
for o in self._emit_directory_traversal_from_sandbox(sandbox):
yield o
for path in sandbox["CONFIGURE_SUBST_FILES"]:
yield self._create_substitution(ConfigFileSubstitution, sandbox, path)
for path in sandbox["CONFIGURE_DEFINE_FILES"]:
yield self._create_substitution(HeaderFileSubstitution, sandbox, path)
# XPIDL source files get processed and turned into .h and .xpt files.
# If there are multiple XPIDL files in a directory, they get linked
# together into a final .xpt, which has the name defined by
# XPIDL_MODULE.
xpidl_module = sandbox["XPIDL_MODULE"]
if sandbox["XPIDL_SOURCES"] and not xpidl_module:
raise SandboxValidationError("XPIDL_MODULE must be defined if " "XPIDL_SOURCES is defined.")
if sandbox["XPIDL_SOURCES"] and sandbox["NO_DIST_INSTALL"]:
self.log(
logging.WARN,
"mozbuild_warning",
dict(path=sandbox.main_path),
"{path}: NO_DIST_INSTALL has no effect on XPIDL_SOURCES.",
)
for idl in sandbox["XPIDL_SOURCES"]:
yield XPIDLFile(sandbox, mozpath.join(sandbox["SRCDIR"], idl), xpidl_module)
for symbol in ("SOURCES", "HOST_SOURCES", "UNIFIED_SOURCES"):
for src in sandbox[symbol] or []:
if not os.path.exists(mozpath.join(sandbox["SRCDIR"], src)):
raise SandboxValidationError(
"Reference to a file that "
"doesn't exist in %s (%s) in %s" % (symbol, src, sandbox["RELATIVEDIR"])
)
if sandbox.get("LIBXUL_LIBRARY") and sandbox.get("FORCE_STATIC_LIB"):
raise SandboxValidationError("LIBXUL_LIBRARY implies FORCE_STATIC_LIB")
# Proxy some variables as-is until we have richer classes to represent
# them. We should aim to keep this set small because it violates the
# desired abstraction of the build definition away from makefiles.
passthru = VariablePassthru(sandbox)
varmap = dict(
# Makefile.in : moz.build
ANDROID_GENERATED_RESFILES="ANDROID_GENERATED_RESFILES",
ANDROID_RESFILES="ANDROID_RESFILES",
CPP_UNIT_TESTS="CPP_UNIT_TESTS",
EXPORT_LIBRARY="EXPORT_LIBRARY",
EXTRA_COMPONENTS="EXTRA_COMPONENTS",
EXTRA_JS_MODULES="EXTRA_JS_MODULES",
EXTRA_PP_COMPONENTS="EXTRA_PP_COMPONENTS",
EXTRA_PP_JS_MODULES="EXTRA_PP_JS_MODULES",
FAIL_ON_WARNINGS="FAIL_ON_WARNINGS",
FILES_PER_UNIFIED_FILE="FILES_PER_UNIFIED_FILE",
FORCE_SHARED_LIB="FORCE_SHARED_LIB",
FORCE_STATIC_LIB="FORCE_STATIC_LIB",
GENERATED_FILES="GENERATED_FILES",
HOST_LIBRARY_NAME="HOST_LIBRARY_NAME",
IS_COMPONENT="IS_COMPONENT",
JS_MODULES_PATH="JS_MODULES_PATH",
LIBS="LIBS",
LIBXUL_LIBRARY="LIBXUL_LIBRARY",
MSVC_ENABLE_PGO="MSVC_ENABLE_PGO",
NO_DIST_INSTALL="NO_DIST_INSTALL",
OS_LIBS="OS_LIBS",
SDK_LIBRARY="SDK_LIBRARY",
)
for mak, moz in varmap.items():
if sandbox[moz]:
passthru.variables[mak] = sandbox[moz]
# NO_VISIBILITY_FLAGS is slightly different
if sandbox["NO_VISIBILITY_FLAGS"]:
passthru.variables["VISIBILITY_FLAGS"] = ""
varmap = dict(
SOURCES={
".s": "ASFILES",
".asm": "ASFILES",
".c": "CSRCS",
".m": "CMSRCS",
".mm": "CMMSRCS",
".cc": "CPPSRCS",
".cpp": "CPPSRCS",
".S": "SSRCS",
},
HOST_SOURCES={".c": "HOST_CSRCS", ".mm": "HOST_CMMSRCS", ".cc": "HOST_CPPSRCS", ".cpp": "HOST_CPPSRCS"},
UNIFIED_SOURCES={
".c": "UNIFIED_CSRCS",
".mm": "UNIFIED_CMMSRCS",
".cc": "UNIFIED_CPPSRCS",
".cpp": "UNIFIED_CPPSRCS",
},
)
varmap.update(dict(("GENERATED_%s" % k, v) for k, v in varmap.items() if k in ("SOURCES", "UNIFIED_SOURCES")))
for variable, mapping in varmap.items():
for f in sandbox[variable]:
ext = mozpath.splitext(f)[1]
if ext not in mapping:
raise SandboxValidationError("%s has an unknown file type in %s" % (f, sandbox["RELATIVEDIR"]))
l = passthru.variables.setdefault(mapping[ext], [])
l.append(f)
if variable.startswith("GENERATED_"):
l = passthru.variables.setdefault("GARBAGE", [])
l.append(f)
no_pgo = sandbox.get("NO_PGO")
sources = sandbox.get("SOURCES", [])
no_pgo_sources = [f for f in sources if sources[f].no_pgo]
if no_pgo:
if no_pgo_sources:
raise SandboxValidationError("NO_PGO and SOURCES[...].no_pgo cannot be set at the same time")
passthru.variables["NO_PROFILE_GUIDED_OPTIMIZE"] = no_pgo
if no_pgo_sources:
passthru.variables["NO_PROFILE_GUIDED_OPTIMIZE"] = no_pgo_sources
exports = sandbox.get("EXPORTS")
if exports:
yield Exports(sandbox, exports, dist_install=not sandbox.get("NO_DIST_INSTALL", False))
defines = sandbox.get("DEFINES")
if defines:
yield Defines(sandbox, defines)
program = sandbox.get("PROGRAM")
if program:
yield Program(sandbox, program, sandbox["CONFIG"]["BIN_SUFFIX"])
program = sandbox.get("HOST_PROGRAM")
if program:
yield HostProgram(sandbox, program, sandbox["CONFIG"]["HOST_BIN_SUFFIX"])
for program in sandbox["SIMPLE_PROGRAMS"]:
yield SimpleProgram(sandbox, program, sandbox["CONFIG"]["BIN_SUFFIX"])
for program in sandbox["HOST_SIMPLE_PROGRAMS"]:
yield HostSimpleProgram(sandbox, program, sandbox["CONFIG"]["HOST_BIN_SUFFIX"])
simple_lists = [
("GENERATED_EVENTS_WEBIDL_FILES", GeneratedEventWebIDLFile),
("GENERATED_WEBIDL_FILES", GeneratedWebIDLFile),
("IPDL_SOURCES", IPDLFile),
("LOCAL_INCLUDES", LocalInclude),
("GENERATED_INCLUDES", GeneratedInclude),
("PREPROCESSED_TEST_WEBIDL_FILES", PreprocessedTestWebIDLFile),
("PREPROCESSED_WEBIDL_FILES", PreprocessedWebIDLFile),
("TEST_WEBIDL_FILES", TestWebIDLFile),
("WEBIDL_FILES", WebIDLFile),
]
for sandbox_var, klass in simple_lists:
for name in sandbox.get(sandbox_var, []):
yield klass(sandbox, name)
if sandbox.get("FINAL_TARGET") or sandbox.get("XPI_NAME") or sandbox.get("DIST_SUBDIR"):
yield InstallationTarget(sandbox)
libname = sandbox.get("LIBRARY_NAME")
final_lib = sandbox.get("FINAL_LIBRARY")
if not libname and final_lib:
# If no LIBRARY_NAME is given, create one.
libname = sandbox["RELATIVEDIR"].replace("/", "_")
if libname:
self._libs.setdefault(libname, {})[sandbox["OBJDIR"]] = LibraryDefinition(sandbox, libname)
if final_lib:
if sandbox.get("FORCE_STATIC_LIB"):
raise SandboxValidationError("FINAL_LIBRARY implies FORCE_STATIC_LIB")
self._final_libs.append((sandbox["OBJDIR"], libname, final_lib))
passthru.variables["FORCE_STATIC_LIB"] = True
# While there are multiple test manifests, the behavior is very similar
# across them. We enforce this by having common handling of all
# manifests and outputting a single class type with the differences
# described inside the instance.
#
# Keys are variable prefixes and values are tuples describing how these
# manifests should be handled:
#
# (flavor, install_prefix, active)
#
# flavor identifies the flavor of this test.
# install_prefix is the path prefix of where to install the files in
# the tests directory.
# active indicates whether to filter out inactive tests from the
# manifest.
#
# We ideally don't filter out inactive tests. However, not every test
# harness can yet deal with test filtering. Once all harnesses can do
# this, this feature can be dropped.
test_manifests = dict(
A11Y=("a11y", "testing/mochitest/a11y", True),
BROWSER_CHROME=("browser-chrome", "testing/mochitest/browser", True),
METRO_CHROME=("metro-chrome", "testing/mochitest/metro", True),
MOCHITEST=("mochitest", "testing/mochitest/tests", True),
MOCHITEST_CHROME=("chrome", "testing/mochitest/chrome", True),
MOCHITEST_WEBAPPRT_CHROME=("webapprt-chrome", "testing/mochitest/webapprtChrome", True),
WEBRTC_SIGNALLING_TEST=("steeplechase", "steeplechase", True),
XPCSHELL_TESTS=("xpcshell", "xpcshell", False),
)
for prefix, info in test_manifests.items():
for path in sandbox.get("%s_MANIFESTS" % prefix, []):
for obj in self._process_test_manifest(sandbox, info, path):
yield obj
for name, jar in sandbox.get("JAVA_JAR_TARGETS", {}).items():
yield SandboxWrapped(sandbox, jar)
if passthru.variables:
yield passthru
def emit_from_sandbox(self, sandbox):
"""Convert a MozbuildSandbox to tree metadata objects.
This is a generator of mozbuild.frontend.data.SandboxDerived instances.
"""
# We always emit a directory traversal descriptor. This is needed by
# the recursive make backend.
for o in self._emit_directory_traversal_from_sandbox(sandbox):
yield o
for path in sandbox["CONFIGURE_SUBST_FILES"]:
yield self._create_substitution(ConfigFileSubstitution, sandbox, path)
for path in sandbox["CONFIGURE_DEFINE_FILES"]:
yield self._create_substitution(HeaderFileSubstitution, sandbox, path)
# XPIDL source files get processed and turned into .h and .xpt files.
# If there are multiple XPIDL files in a directory, they get linked
# together into a final .xpt, which has the name defined by
# XPIDL_MODULE.
xpidl_module = sandbox["XPIDL_MODULE"]
if sandbox["XPIDL_SOURCES"] and not xpidl_module:
raise SandboxValidationError("XPIDL_MODULE must be defined if " "XPIDL_SOURCES is defined.")
if xpidl_module and not sandbox["XPIDL_SOURCES"]:
raise SandboxValidationError(
"XPIDL_MODULE cannot be defined " "unless there are XPIDL_SOURCES: %s" % sandbox["RELATIVEDIR"]
)
if sandbox["XPIDL_SOURCES"] and sandbox["NO_DIST_INSTALL"]:
self.log(
logging.WARN,
"mozbuild_warning",
dict(path=sandbox.main_path),
"{path}: NO_DIST_INSTALL has no effect on XPIDL_SOURCES.",
)
for idl in sandbox["XPIDL_SOURCES"]:
yield XPIDLFile(sandbox, mozpath.join(sandbox["SRCDIR"], idl), xpidl_module)
for symbol in ("SOURCES", "HOST_SOURCES", "UNIFIED_SOURCES"):
for src in sandbox[symbol] or []:
if not os.path.exists(mozpath.join(sandbox["SRCDIR"], src)):
raise SandboxValidationError(
"Reference to a file that "
"doesn't exist in %s (%s) in %s" % (symbol, src, sandbox["RELATIVEDIR"])
)
# Proxy some variables as-is until we have richer classes to represent
# them. We should aim to keep this set small because it violates the
# desired abstraction of the build definition away from makefiles.
passthru = VariablePassthru(sandbox)
varlist = [
"ANDROID_GENERATED_RESFILES",
"ANDROID_RES_DIRS",
"CPP_UNIT_TESTS",
"DISABLE_STL_WRAPPING",
"EXPORT_LIBRARY",
"EXTRA_ASSEMBLER_FLAGS",
"EXTRA_COMPILE_FLAGS",
"EXTRA_COMPONENTS",
"EXTRA_DSO_LDOPTS",
"EXTRA_JS_MODULES",
"EXTRA_PP_COMPONENTS",
"EXTRA_PP_JS_MODULES",
"FAIL_ON_WARNINGS",
"FILES_PER_UNIFIED_FILE",
"FORCE_SHARED_LIB",
"FORCE_STATIC_LIB",
"USE_STATIC_LIBS",
"GENERATED_FILES",
"HOST_LIBRARY_NAME",
"IS_COMPONENT",
"IS_GYP_DIR",
"JS_MODULES_PATH",
"LIBS",
"MSVC_ENABLE_PGO",
"NO_DIST_INSTALL",
"OS_LIBS",
"RCFILE",
"RESFILE",
"RCINCLUDE",
"DEFFILE",
"SDK_LIBRARY",
"WIN32_EXE_LDFLAGS",
"LD_VERSION_SCRIPT",
]
for v in varlist:
if v in sandbox and sandbox[v]:
passthru.variables[v] = sandbox[v]
for v in ["CFLAGS", "CXXFLAGS", "CMFLAGS", "CMMFLAGS", "LDFLAGS"]:
if v in sandbox and sandbox[v]:
passthru.variables["MOZBUILD_" + v] = sandbox[v]
# NO_VISIBILITY_FLAGS is slightly different
if sandbox["NO_VISIBILITY_FLAGS"]:
passthru.variables["VISIBILITY_FLAGS"] = ""
if sandbox["DELAYLOAD_DLLS"]:
passthru.variables["DELAYLOAD_LDFLAGS"] = [("-DELAYLOAD:%s" % dll) for dll in sandbox["DELAYLOAD_DLLS"]]
passthru.variables["USE_DELAYIMP"] = True
varmap = dict(
SOURCES={
".s": "ASFILES",
".asm": "ASFILES",
".c": "CSRCS",
".m": "CMSRCS",
".mm": "CMMSRCS",
".cc": "CPPSRCS",
".cpp": "CPPSRCS",
".cxx": "CPPSRCS",
".S": "SSRCS",
},
HOST_SOURCES={
".c": "HOST_CSRCS",
".mm": "HOST_CMMSRCS",
".cc": "HOST_CPPSRCS",
".cpp": "HOST_CPPSRCS",
".cxx": "HOST_CPPSRCS",
},
UNIFIED_SOURCES={
".c": "UNIFIED_CSRCS",
".mm": "UNIFIED_CMMSRCS",
".cc": "UNIFIED_CPPSRCS",
".cpp": "UNIFIED_CPPSRCS",
".cxx": "UNIFIED_CPPSRCS",
},
)
varmap.update(dict(("GENERATED_%s" % k, v) for k, v in varmap.items() if k in ("SOURCES", "UNIFIED_SOURCES")))
for variable, mapping in varmap.items():
for f in sandbox[variable]:
ext = mozpath.splitext(f)[1]
if ext not in mapping:
raise SandboxValidationError("%s has an unknown file type in %s" % (f, sandbox["RELATIVEDIR"]))
l = passthru.variables.setdefault(mapping[ext], [])
l.append(f)
if variable.startswith("GENERATED_"):
l = passthru.variables.setdefault("GARBAGE", [])
l.append(f)
no_pgo = sandbox.get("NO_PGO")
sources = sandbox.get("SOURCES", [])
no_pgo_sources = [f for f in sources if sources[f].no_pgo]
if no_pgo:
if no_pgo_sources:
raise SandboxValidationError("NO_PGO and SOURCES[...].no_pgo cannot be set at the same time")
passthru.variables["NO_PROFILE_GUIDED_OPTIMIZE"] = no_pgo
if no_pgo_sources:
passthru.variables["NO_PROFILE_GUIDED_OPTIMIZE"] = no_pgo_sources
sources_with_flags = [f for f in sources if sources[f].flags]
for f in sources_with_flags:
ext = mozpath.splitext(f)[1]
yield PerSourceFlag(sandbox, f, sources[f].flags)
exports = sandbox.get("EXPORTS")
if exports:
yield Exports(sandbox, exports, dist_install=not sandbox.get("NO_DIST_INSTALL", False))
defines = sandbox.get("DEFINES")
if defines:
yield Defines(sandbox, defines)
resources = sandbox.get("RESOURCE_FILES")
if resources:
yield Resources(sandbox, resources, defines)
program = sandbox.get("PROGRAM")
if program:
yield Program(sandbox, program, sandbox["CONFIG"]["BIN_SUFFIX"])
program = sandbox.get("HOST_PROGRAM")
if program:
yield HostProgram(sandbox, program, sandbox["CONFIG"]["HOST_BIN_SUFFIX"])
for program in sandbox["SIMPLE_PROGRAMS"]:
yield SimpleProgram(sandbox, program, sandbox["CONFIG"]["BIN_SUFFIX"])
for program in sandbox["HOST_SIMPLE_PROGRAMS"]:
yield HostSimpleProgram(sandbox, program, sandbox["CONFIG"]["HOST_BIN_SUFFIX"])
simple_lists = [
("GENERATED_EVENTS_WEBIDL_FILES", GeneratedEventWebIDLFile),
("GENERATED_WEBIDL_FILES", GeneratedWebIDLFile),
("IPDL_SOURCES", IPDLFile),
("LOCAL_INCLUDES", LocalInclude),
("GENERATED_INCLUDES", GeneratedInclude),
("PREPROCESSED_TEST_WEBIDL_FILES", PreprocessedTestWebIDLFile),
("PREPROCESSED_WEBIDL_FILES", PreprocessedWebIDLFile),
("TEST_WEBIDL_FILES", TestWebIDLFile),
("WEBIDL_FILES", WebIDLFile),
("WEBIDL_EXAMPLE_INTERFACES", ExampleWebIDLInterface),
]
for sandbox_var, klass in simple_lists:
for name in sandbox.get(sandbox_var, []):
yield klass(sandbox, name)
if sandbox.get("FINAL_TARGET") or sandbox.get("XPI_NAME") or sandbox.get("DIST_SUBDIR"):
yield InstallationTarget(sandbox)
libname = sandbox.get("LIBRARY_NAME")
final_lib = sandbox.get("FINAL_LIBRARY")
if not libname and final_lib:
# If no LIBRARY_NAME is given, create one.
libname = sandbox["RELATIVEDIR"].replace("/", "_")
if libname:
self._libs.setdefault(libname, {})[sandbox["OBJDIR"]] = LibraryDefinition(sandbox, libname)
if final_lib:
if isinstance(sandbox, MozbuildSandbox) and sandbox.get("FORCE_STATIC_LIB"):
raise SandboxValidationError("FINAL_LIBRARY implies FORCE_STATIC_LIB")
self._final_libs.append((sandbox["OBJDIR"], libname, final_lib))
passthru.variables["FORCE_STATIC_LIB"] = True
# While there are multiple test manifests, the behavior is very similar
# across them. We enforce this by having common handling of all
# manifests and outputting a single class type with the differences
# described inside the instance.
#
# Keys are variable prefixes and values are tuples describing how these
# manifests should be handled:
#
# (flavor, install_prefix, active)
#
# flavor identifies the flavor of this test.
# install_prefix is the path prefix of where to install the files in
# the tests directory.
# active indicates whether to filter out inactive tests from the
# manifest.
#
# We ideally don't filter out inactive tests. However, not every test
# harness can yet deal with test filtering. Once all harnesses can do
# this, this feature can be dropped.
test_manifests = dict(
A11Y=("a11y", "testing/mochitest", "a11y", True),
BROWSER_CHROME=("browser-chrome", "testing/mochitest", "browser", True),
METRO_CHROME=("metro-chrome", "testing/mochitest", "metro", True),
MOCHITEST=("mochitest", "testing/mochitest", "tests", True),
MOCHITEST_CHROME=("chrome", "testing/mochitest", "chrome", True),
MOCHITEST_WEBAPPRT_CHROME=("webapprt-chrome", "testing/mochitest", "webapprtChrome", True),
WEBRTC_SIGNALLING_TEST=("steeplechase", "steeplechase", ".", True),
XPCSHELL_TESTS=("xpcshell", "xpcshell", ".", False),
)
for prefix, info in test_manifests.items():
for path in sandbox.get("%s_MANIFESTS" % prefix, []):
for obj in self._process_test_manifest(sandbox, info, path):
yield obj
for flavor in ("crashtest", "reftest"):
for path in sandbox.get("%s_MANIFESTS" % flavor.upper(), []):
for obj in self._process_reftest_manifest(sandbox, flavor, path):
yield obj
jar_manifests = sandbox.get("JAR_MANIFESTS", [])
if len(jar_manifests) > 1:
raise SandboxValidationError("While JAR_MANIFESTS is a list, " "it is currently limited to one value.")
for path in jar_manifests:
yield JARManifest(sandbox, mozpath.join(sandbox["SRCDIR"], path))
# Temporary test to look for jar.mn files that creep in without using
# the new declaration. Before, we didn't require jar.mn files to
# declared anywhere (they were discovered). This will detect people
# relying on the old behavior.
if os.path.exists(os.path.join(sandbox["SRCDIR"], "jar.mn")):
if "jar.mn" not in jar_manifests:
raise SandboxValidationError(
"A jar.mn exists in %s but it "
"is not referenced in the corresponding moz.build file. "
"Please define JAR_MANIFESTS in the moz.build file." % sandbox["SRCDIR"]
)
for name, jar in sandbox.get("JAVA_JAR_TARGETS", {}).items():
yield SandboxWrapped(sandbox, jar)
for name, data in sandbox.get("ANDROID_ECLIPSE_PROJECT_TARGETS", {}).items():
yield SandboxWrapped(sandbox, data)
if passthru.variables:
yield passthru
def canonical_suffix_for_file(f):
return canonicalized_suffix_map[mozpath.splitext(f)[1]]
def read_from_gyp(config, path, output, vars, non_unified_sources=set()):
"""Read a gyp configuration and emits GypSandboxes for the backend to
process.
config is a ConfigEnvironment, path is the path to a root gyp configuration
file, output is the base path under which the objdir for the various gyp
dependencies will be, and vars a dict of variables to pass to the gyp
processor.
"""
time_start = time.time()
all_sources = set()
# gyp expects plain str instead of unicode. The frontend code gives us
# unicode strings, so convert them.
path = encode(path)
str_vars = dict((name, encode(value)) for name, value in vars.items())
params = {
b'parallel': False,
b'generator_flags': {},
b'build_files': [path],
}
# Files that gyp_chromium always includes
includes = [encode(mozpath.join(script_dir, 'common.gypi'))]
finder = FileFinder(chrome_src, find_executables=False)
includes.extend(
encode(mozpath.join(chrome_src, name))
for name, _ in finder.find('*/supplement.gypi'))
# Read the given gyp file and its dependencies.
generator, flat_list, targets, data = \
gyp.Load([path], format=b'mozbuild',
default_variables=str_vars,
includes=includes,
depth=encode(mozpath.dirname(path)),
params=params)
# Process all targets from the given gyp files and its dependencies.
# The path given to AllTargets needs to use os.sep, while the frontend code
# gives us paths normalized with forward slash separator.
for target in gyp.common.AllTargets(flat_list, targets,
path.replace(b'/', os.sep)):
build_file, target_name, toolset = gyp.common.ParseQualifiedTarget(
target)
# The list of included files returned by gyp are relative to build_file
included_files = [
mozpath.abspath(mozpath.join(mozpath.dirname(build_file), f))
for f in data[build_file]['included_files']
]
# Emit a sandbox for each target.
sandbox = GypSandbox(mozpath.abspath(build_file), included_files)
with sandbox.allow_all_writes() as d:
topsrcdir = d['TOPSRCDIR'] = config.topsrcdir
d['TOPOBJDIR'] = config.topobjdir
relsrcdir = d['RELATIVEDIR'] = mozpath.relpath(
mozpath.dirname(build_file), config.topsrcdir)
d['SRCDIR'] = mozpath.join(topsrcdir, relsrcdir)
# Each target is given its own objdir. The base of that objdir
# is derived from the relative path from the root gyp file path
# to the current build_file, placed under the given output
# directory. Since several targets can be in a given build_file,
# separate them in subdirectories using the build_file basename
# and the target_name.
reldir = mozpath.relpath(mozpath.dirname(build_file),
mozpath.dirname(path))
subdir = '%s_%s' % (
mozpath.splitext(mozpath.basename(build_file))[0],
target_name,
)
d['OBJDIR'] = mozpath.join(output, reldir, subdir)
d['IS_GYP_DIR'] = True
spec = targets[target]
# Derive which gyp configuration to use based on MOZ_DEBUG.
c = 'Debug' if config.substs['MOZ_DEBUG'] else 'Release'
if c not in spec['configurations']:
raise RuntimeError('Missing %s gyp configuration for target %s '
'in %s' % (c, target_name, build_file))
target_conf = spec['configurations'][c]
if spec['type'] == 'none':
continue
elif spec['type'] == 'static_library':
sandbox['FORCE_STATIC_LIB'] = True
# Remove leading 'lib' from the target_name if any, and use as
# library name.
name = spec['target_name']
if name.startswith('lib'):
name = name[3:]
# The sandbox expects an unicode string.
sandbox['LIBRARY_NAME'] = name.decode('utf-8')
# gyp files contain headers and asm sources in sources lists.
sources = set(
mozpath.normpath(mozpath.join(sandbox['SRCDIR'], f))
for f in spec.get('sources', [])
if mozpath.splitext(f)[-1] != '.h')
asm_sources = set(f for f in sources if f.endswith('.S'))
unified_sources = sources - non_unified_sources - asm_sources
sources -= unified_sources
all_sources |= sources
# The sandbox expects alphabetical order when adding sources
sandbox['SOURCES'] = alphabetical_sorted(sources)
sandbox['UNIFIED_SOURCES'] = alphabetical_sorted(unified_sources)
for define in target_conf.get('defines', []):
if '=' in define:
name, value = define.split('=', 1)
sandbox['DEFINES'][name] = value
else:
sandbox['DEFINES'][define] = True
for include in target_conf.get('include_dirs', []):
sandbox['LOCAL_INCLUDES'] += [include]
with sandbox.allow_all_writes() as d:
d['EXTRA_ASSEMBLER_FLAGS'] = target_conf.get(
'asflags_mozilla', [])
d['EXTRA_COMPILE_FLAGS'] = target_conf.get(
'cflags_mozilla', [])
else:
# Ignore other types than static_library because we don't have
# anything using them, and we're not testing them. They can be
# added when that becomes necessary.
raise NotImplementedError('Unsupported gyp target type: %s' %
spec['type'])
sandbox.execution_time = time.time() - time_start
yield sandbox
time_start = time.time()
def test_splitext(self):
self.assertEqual(splitext(os.path.join("foo", "bar", "baz.qux")), ("foo/bar/baz", ".qux"))
def test_splitext(self):
self.assertEqual(splitext(self.SEP.join(("foo", "bar", "baz.qux"))),
("foo/bar/baz", ".qux"))
def emit_from_sandbox(self, sandbox):
"""Convert a MozbuildSandbox to tree metadata objects.
This is a generator of mozbuild.frontend.data.SandboxDerived instances.
"""
# We always emit a directory traversal descriptor. This is needed by
# the recursive make backend.
for o in self._emit_directory_traversal_from_sandbox(sandbox): yield o
for path in sandbox['CONFIGURE_SUBST_FILES']:
yield self._create_substitution(ConfigFileSubstitution, sandbox,
path)
for path in sandbox['CONFIGURE_DEFINE_FILES']:
yield self._create_substitution(HeaderFileSubstitution, sandbox,
path)
# XPIDL source files get processed and turned into .h and .xpt files.
# If there are multiple XPIDL files in a directory, they get linked
# together into a final .xpt, which has the name defined by
# XPIDL_MODULE.
xpidl_module = sandbox['XPIDL_MODULE']
if sandbox['XPIDL_SOURCES'] and not xpidl_module:
raise SandboxValidationError('XPIDL_MODULE must be defined if '
'XPIDL_SOURCES is defined.')
if xpidl_module and not sandbox['XPIDL_SOURCES']:
raise SandboxValidationError('XPIDL_MODULE cannot be defined '
'unless there are XPIDL_SOURCES: %s' % sandbox['RELATIVEDIR'])
if sandbox['XPIDL_SOURCES'] and sandbox['NO_DIST_INSTALL']:
self.log(logging.WARN, 'mozbuild_warning', dict(
path=sandbox.main_path),
'{path}: NO_DIST_INSTALL has no effect on XPIDL_SOURCES.')
for idl in sandbox['XPIDL_SOURCES']:
yield XPIDLFile(sandbox, mozpath.join(sandbox['SRCDIR'], idl),
xpidl_module)
for symbol in ('SOURCES', 'HOST_SOURCES', 'UNIFIED_SOURCES'):
for src in (sandbox[symbol] or []):
if not os.path.exists(mozpath.join(sandbox['SRCDIR'], src)):
raise SandboxValidationError('Reference to a file that '
'doesn\'t exist in %s (%s) in %s'
% (symbol, src, sandbox['RELATIVEDIR']))
if sandbox.get('LIBXUL_LIBRARY') and sandbox.get('FORCE_STATIC_LIB'):
raise SandboxValidationError('LIBXUL_LIBRARY implies FORCE_STATIC_LIB')
# Proxy some variables as-is until we have richer classes to represent
# them. We should aim to keep this set small because it violates the
# desired abstraction of the build definition away from makefiles.
passthru = VariablePassthru(sandbox)
varlist = [
'ANDROID_GENERATED_RESFILES',
'ANDROID_RES_DIRS',
'CPP_UNIT_TESTS',
'EXPORT_LIBRARY',
'EXTRA_ASSEMBLER_FLAGS',
'EXTRA_COMPILE_FLAGS',
'EXTRA_COMPONENTS',
'EXTRA_JS_MODULES',
'EXTRA_PP_COMPONENTS',
'EXTRA_PP_JS_MODULES',
'FAIL_ON_WARNINGS',
'FILES_PER_UNIFIED_FILE',
'FORCE_SHARED_LIB',
'FORCE_STATIC_LIB',
'USE_STATIC_LIBS',
'GENERATED_FILES',
'HOST_LIBRARY_NAME',
'IS_COMPONENT',
'IS_GYP_DIR',
'JS_MODULES_PATH',
'LIBS',
'LIBXUL_LIBRARY',
'MSVC_ENABLE_PGO',
'NO_DIST_INSTALL',
'OS_LIBS',
'RCFILE',
'RESFILE',
'DEFFILE',
'SDK_LIBRARY',
'CFLAGS',
'CXXFLAGS',
'LDFLAGS',
'WIN32_EXE_LDFLAGS',
]
for v in varlist:
if v in sandbox and sandbox[v]:
passthru.variables[v] = sandbox[v]
# NO_VISIBILITY_FLAGS is slightly different
if sandbox['NO_VISIBILITY_FLAGS']:
passthru.variables['VISIBILITY_FLAGS'] = ''
if sandbox['DELAYLOAD_DLLS']:
passthru.variables['DELAYLOAD_LDFLAGS'] = [('-DELAYLOAD:%s' % dll) for dll in sandbox['DELAYLOAD_DLLS']]
passthru.variables['USE_DELAYIMP'] = True
varmap = dict(
SOURCES={
'.s': 'ASFILES',
'.asm': 'ASFILES',
'.c': 'CSRCS',
'.m': 'CMSRCS',
'.mm': 'CMMSRCS',
'.cc': 'CPPSRCS',
'.cpp': 'CPPSRCS',
'.S': 'SSRCS',
},
HOST_SOURCES={
'.c': 'HOST_CSRCS',
'.mm': 'HOST_CMMSRCS',
'.cc': 'HOST_CPPSRCS',
'.cpp': 'HOST_CPPSRCS',
},
UNIFIED_SOURCES={
'.c': 'UNIFIED_CSRCS',
'.mm': 'UNIFIED_CMMSRCS',
'.cc': 'UNIFIED_CPPSRCS',
'.cpp': 'UNIFIED_CPPSRCS',
}
)
varmap.update(dict(('GENERATED_%s' % k, v) for k, v in varmap.items()
if k in ('SOURCES', 'UNIFIED_SOURCES')))
for variable, mapping in varmap.items():
for f in sandbox[variable]:
ext = mozpath.splitext(f)[1]
if ext not in mapping:
raise SandboxValidationError('%s has an unknown file type in %s' % (f, sandbox['RELATIVEDIR']))
l = passthru.variables.setdefault(mapping[ext], [])
l.append(f)
if variable.startswith('GENERATED_'):
l = passthru.variables.setdefault('GARBAGE', [])
l.append(f)
no_pgo = sandbox.get('NO_PGO')
sources = sandbox.get('SOURCES', [])
no_pgo_sources = [f for f in sources if sources[f].no_pgo]
if no_pgo:
if no_pgo_sources:
raise SandboxValidationError('NO_PGO and SOURCES[...].no_pgo cannot be set at the same time')
passthru.variables['NO_PROFILE_GUIDED_OPTIMIZE'] = no_pgo
if no_pgo_sources:
passthru.variables['NO_PROFILE_GUIDED_OPTIMIZE'] = no_pgo_sources
exports = sandbox.get('EXPORTS')
if exports:
yield Exports(sandbox, exports,
dist_install=not sandbox.get('NO_DIST_INSTALL', False))
defines = sandbox.get('DEFINES')
if defines:
yield Defines(sandbox, defines)
resources = sandbox.get('RESOURCE_FILES')
if resources:
yield Resources(sandbox, resources, defines)
program = sandbox.get('PROGRAM')
if program:
yield Program(sandbox, program, sandbox['CONFIG']['BIN_SUFFIX'])
program = sandbox.get('HOST_PROGRAM')
if program:
yield HostProgram(sandbox, program, sandbox['CONFIG']['HOST_BIN_SUFFIX'])
for program in sandbox['SIMPLE_PROGRAMS']:
yield SimpleProgram(sandbox, program, sandbox['CONFIG']['BIN_SUFFIX'])
for program in sandbox['HOST_SIMPLE_PROGRAMS']:
yield HostSimpleProgram(sandbox, program, sandbox['CONFIG']['HOST_BIN_SUFFIX'])
simple_lists = [
('GENERATED_EVENTS_WEBIDL_FILES', GeneratedEventWebIDLFile),
('GENERATED_WEBIDL_FILES', GeneratedWebIDLFile),
('IPDL_SOURCES', IPDLFile),
('LOCAL_INCLUDES', LocalInclude),
('GENERATED_INCLUDES', GeneratedInclude),
('PREPROCESSED_TEST_WEBIDL_FILES', PreprocessedTestWebIDLFile),
('PREPROCESSED_WEBIDL_FILES', PreprocessedWebIDLFile),
('TEST_WEBIDL_FILES', TestWebIDLFile),
('WEBIDL_FILES', WebIDLFile),
('WEBIDL_EXAMPLE_INTERFACES', ExampleWebIDLInterface),
]
for sandbox_var, klass in simple_lists:
for name in sandbox.get(sandbox_var, []):
yield klass(sandbox, name)
if sandbox.get('FINAL_TARGET') or sandbox.get('XPI_NAME') or \
sandbox.get('DIST_SUBDIR'):
yield InstallationTarget(sandbox)
libname = sandbox.get('LIBRARY_NAME')
final_lib = sandbox.get('FINAL_LIBRARY')
if not libname and final_lib:
# If no LIBRARY_NAME is given, create one.
libname = sandbox['RELATIVEDIR'].replace('/', '_')
if libname:
self._libs.setdefault(libname, {})[sandbox['OBJDIR']] = \
LibraryDefinition(sandbox, libname)
if final_lib:
if isinstance(sandbox, MozbuildSandbox) and sandbox.get('FORCE_STATIC_LIB'):
raise SandboxValidationError('FINAL_LIBRARY implies FORCE_STATIC_LIB')
self._final_libs.append((sandbox['OBJDIR'], libname, final_lib))
passthru.variables['FORCE_STATIC_LIB'] = True
# While there are multiple test manifests, the behavior is very similar
# across them. We enforce this by having common handling of all
# manifests and outputting a single class type with the differences
# described inside the instance.
#
# Keys are variable prefixes and values are tuples describing how these
# manifests should be handled:
#
# (flavor, install_prefix, active)
#
# flavor identifies the flavor of this test.
# install_prefix is the path prefix of where to install the files in
# the tests directory.
# active indicates whether to filter out inactive tests from the
# manifest.
#
# We ideally don't filter out inactive tests. However, not every test
# harness can yet deal with test filtering. Once all harnesses can do
# this, this feature can be dropped.
test_manifests = dict(
A11Y=('a11y', 'testing/mochitest', 'a11y', True),
BROWSER_CHROME=('browser-chrome', 'testing/mochitest', 'browser', True),
METRO_CHROME=('metro-chrome', 'testing/mochitest', 'metro', True),
MOCHITEST=('mochitest', 'testing/mochitest', 'tests', True),
MOCHITEST_CHROME=('chrome', 'testing/mochitest', 'chrome', True),
MOCHITEST_WEBAPPRT_CHROME=('webapprt-chrome', 'testing/mochitest', 'webapprtChrome', True),
WEBRTC_SIGNALLING_TEST=('steeplechase', 'steeplechase', '.', True),
XPCSHELL_TESTS=('xpcshell', 'xpcshell', '.', False),
)
for prefix, info in test_manifests.items():
for path in sandbox.get('%s_MANIFESTS' % prefix, []):
for obj in self._process_test_manifest(sandbox, info, path):
yield obj
jar_manifests = sandbox.get('JAR_MANIFESTS', [])
if len(jar_manifests) > 1:
raise SandboxValidationError('While JAR_MANIFESTS is a list, '
'it is currently limited to one value.')
for path in jar_manifests:
yield JARManifest(sandbox, mozpath.join(sandbox['SRCDIR'], path))
# Temporary test to look for jar.mn files that creep in without using
# the new declaration. Before, we didn't require jar.mn files to
# declared anywhere (they were discovered). This will detect people
# relying on the old behavior.
if os.path.exists(os.path.join(sandbox['SRCDIR'], 'jar.mn')):
if 'jar.mn' not in jar_manifests:
raise SandboxValidationError('A jar.mn exists in %s but it '
'is not referenced in the corresponding moz.build file. '
'Please define JAR_MANIFESTS in the moz.build file.' %
sandbox['SRCDIR'])
for name, jar in sandbox.get('JAVA_JAR_TARGETS', {}).items():
yield SandboxWrapped(sandbox, jar)
for name, data in sandbox.get('ANDROID_ECLIPSE_PROJECT_TARGETS', {}).items():
yield SandboxWrapped(sandbox, data)
if passthru.variables:
yield passthru
def process_gn_config(gn_config, srcdir, config, output, non_unified_sources,
sandbox_vars, mozilla_flags):
# Translates a json gn config into attributes that can be used to write out
# moz.build files for this configuration.
# Much of this code is based on similar functionality in `gyp_reader.py`.
mozbuild_attrs = {'mozbuild_args': gn_config.get('mozbuild_args', None),
'dirs': {}}
targets = gn_config["targets"]
project_relsrcdir = mozpath.relpath(srcdir, config.topsrcdir)
def target_info(fullname):
path, name = target_fullname.split(':')
# Stripping '//' gives us a path relative to the project root,
# adding a suffix avoids name collisions with libraries already
# in the tree (like "webrtc").
return path.lstrip('//'), name + '_gn'
# Process all targets from the given gn project and its dependencies.
for target_fullname, spec in targets.iteritems():
target_path, target_name = target_info(target_fullname)
context_attrs = {}
# Remove leading 'lib' from the target_name if any, and use as
# library name.
name = target_name
if spec['type'] in ('static_library', 'shared_library', 'source_set'):
if name.startswith('lib'):
name = name[3:]
context_attrs['LIBRARY_NAME'] = name.decode('utf-8')
else:
raise Exception('The following GN target type is not currently '
'consumed by moz.build: "%s". It may need to be '
'added, or you may need to re-run the '
'`GnConfigGen` step.' % spec['type'])
if spec['type'] == 'shared_library':
context_attrs['FORCE_SHARED_LIB'] = True
sources = []
unified_sources = []
extensions = set()
use_defines_in_asflags = False
for f in spec.get('sources', []):
f = f.lstrip("//")
ext = mozpath.splitext(f)[-1]
extensions.add(ext)
src = '%s/%s' % (project_relsrcdir, f)
if ext == '.h':
continue
elif ext == '.def':
context_attrs['SYMBOLS_FILE'] = src
elif ext != '.S' and src not in non_unified_sources:
unified_sources.append('/%s' % src)
else:
sources.append('/%s' % src)
# The Mozilla build system doesn't use DEFINES for building
# ASFILES.
if ext == '.s':
use_defines_in_asflags = True
context_attrs['SOURCES'] = sources
context_attrs['UNIFIED_SOURCES'] = unified_sources
context_attrs['DEFINES'] = {}
for define in spec.get('defines', []):
if '=' in define:
name, value = define.split('=', 1)
context_attrs['DEFINES'][name] = value
else:
context_attrs['DEFINES'][define] = True
context_attrs['LOCAL_INCLUDES'] = []
for include in spec.get('include_dirs', []):
# GN will have resolved all these paths relative to the root of
# the project indicated by "//".
if include.startswith('//'):
include = include[2:]
# moz.build expects all LOCAL_INCLUDES to exist, so ensure they do.
if include.startswith('/'):
resolved = mozpath.abspath(mozpath.join(config.topsrcdir, include[1:]))
else:
resolved = mozpath.abspath(mozpath.join(srcdir, include))
if not os.path.exists(resolved):
# GN files may refer to include dirs that are outside of the
# tree or we simply didn't vendor. Print a warning in this case.
if not resolved.endswith('gn-output/gen'):
print("Included path: '%s' does not exist, dropping include from GN "
"configuration." % resolved, file=sys.stderr)
continue
if not include.startswith('/'):
include = '/%s/%s' % (project_relsrcdir, include)
context_attrs['LOCAL_INCLUDES'] += [include]
context_attrs['ASFLAGS'] = spec.get('asflags_mozilla', [])
if use_defines_in_asflags and defines:
context_attrs['ASFLAGS'] += ['-D' + d for d in defines]
flags = [f for f in spec.get('cflags', []) if f in mozilla_flags]
if flags:
suffix_map = {
'.c': 'CFLAGS',
'.cpp': 'CXXFLAGS',
'.cc': 'CXXFLAGS',
'.m': 'CMFLAGS',
'.mm': 'CMMFLAGS',
}
variables = (suffix_map[e] for e in extensions if e in suffix_map)
for var in variables:
for f in flags:
# We may be getting make variable references out of the
# gn data, and we don't want those in emitted data, so
# substitute them with their actual value.
f = expand_variables(f, config.substs).split()
if not f:
continue
# the result may be a string or a list.
if isinstance(f, types.StringTypes):
context_attrs.setdefault(var, []).append(f)
else:
context_attrs.setdefault(var, []).extend(f)
context_attrs['OS_LIBS'] = []
for lib in spec.get('libs', []):
lib_name = os.path.splitext(lib)[0]
if lib.endswith('.framework'):
context_attrs['OS_LIBS'] += ['-framework ' + lib_name]
else:
context_attrs['OS_LIBS'] += [lib_name]
# Add some features to all contexts. Put here in case LOCAL_INCLUDES
# order matters.
context_attrs['LOCAL_INCLUDES'] += [
'!/ipc/ipdl/_ipdlheaders',
'/ipc/chromium/src',
'/ipc/glue',
]
# These get set via VC project file settings for normal GYP builds.
# TODO: Determine if these defines are needed for GN builds.
if gn_config['mozbuild_args']['OS_TARGET'] == 'WINNT':
context_attrs['DEFINES']['UNICODE'] = True
context_attrs['DEFINES']['_UNICODE'] = True
context_attrs['COMPILE_FLAGS'] = {
'STL': [],
'OS_INCLUDES': [],
}
for key, value in sandbox_vars.items():
if context_attrs.get(key) and isinstance(context_attrs[key], list):
# If we have a key from sandbox_vars that's also been
# populated here we use the value from sandbox_vars as our
# basis rather than overriding outright.
context_attrs[key] = value + context_attrs[key]
elif context_attrs.get(key) and isinstance(context_attrs[key], dict):
context_attrs[key].update(value)
else:
context_attrs[key] = value
target_relsrcdir = mozpath.join(project_relsrcdir, target_path, target_name)
mozbuild_attrs['dirs'][target_relsrcdir] = context_attrs
return mozbuild_attrs
def emit_from_context(self, context):
"""Convert a Context to tree metadata objects.
This is a generator of mozbuild.frontend.data.ContextDerived instances.
"""
# We only want to emit an InstallationTarget if one of the consulted
# variables is defined. Later on, we look up FINAL_TARGET, which has
# the side-effect of populating it. So, we need to do this lookup
# early.
if any(k in context for k in ('FINAL_TARGET', 'XPI_NAME', 'DIST_SUBDIR')):
yield InstallationTarget(context)
# We always emit a directory traversal descriptor. This is needed by
# the recursive make backend.
for o in self._emit_directory_traversal_from_context(context): yield o
for path in context['CONFIGURE_SUBST_FILES']:
yield self._create_substitution(ConfigFileSubstitution, context,
path)
for path in context['CONFIGURE_DEFINE_FILES']:
yield self._create_substitution(HeaderFileSubstitution, context,
path)
# XPIDL source files get processed and turned into .h and .xpt files.
# If there are multiple XPIDL files in a directory, they get linked
# together into a final .xpt, which has the name defined by
# XPIDL_MODULE.
xpidl_module = context['XPIDL_MODULE']
if context['XPIDL_SOURCES'] and not xpidl_module:
raise SandboxValidationError('XPIDL_MODULE must be defined if '
'XPIDL_SOURCES is defined.', context)
if xpidl_module and not context['XPIDL_SOURCES']:
raise SandboxValidationError('XPIDL_MODULE cannot be defined '
'unless there are XPIDL_SOURCES', context)
if context['XPIDL_SOURCES'] and context['NO_DIST_INSTALL']:
self.log(logging.WARN, 'mozbuild_warning', dict(
path=context.main_path),
'{path}: NO_DIST_INSTALL has no effect on XPIDL_SOURCES.')
for idl in context['XPIDL_SOURCES']:
yield XPIDLFile(context, mozpath.join(context.srcdir, idl),
xpidl_module)
for symbol in ('SOURCES', 'HOST_SOURCES', 'UNIFIED_SOURCES'):
for src in (context[symbol] or []):
if not os.path.exists(mozpath.join(context.srcdir, src)):
raise SandboxValidationError('File listed in %s does not '
'exist: \'%s\'' % (symbol, src), context)
# Proxy some variables as-is until we have richer classes to represent
# them. We should aim to keep this set small because it violates the
# desired abstraction of the build definition away from makefiles.
passthru = VariablePassthru(context)
varlist = [
'ANDROID_GENERATED_RESFILES',
'ANDROID_RES_DIRS',
'DISABLE_STL_WRAPPING',
'EXTRA_ASSEMBLER_FLAGS',
'EXTRA_COMPILE_FLAGS',
'EXTRA_COMPONENTS',
'EXTRA_DSO_LDOPTS',
'EXTRA_PP_COMPONENTS',
'FAIL_ON_WARNINGS',
'USE_STATIC_LIBS',
'IS_GYP_DIR',
'MSVC_ENABLE_PGO',
'NO_DIST_INSTALL',
'PYTHON_UNIT_TESTS',
'RCFILE',
'RESFILE',
'RCINCLUDE',
'DEFFILE',
'WIN32_EXE_LDFLAGS',
'LD_VERSION_SCRIPT',
]
for v in varlist:
if v in context and context[v]:
passthru.variables[v] = context[v]
if context.config.substs.get('OS_TARGET') == 'WINNT' and \
context['DELAYLOAD_DLLS']:
context['LDFLAGS'].extend([('-DELAYLOAD:%s' % dll)
for dll in context['DELAYLOAD_DLLS']])
context['OS_LIBS'].append('delayimp')
for v in ['CFLAGS', 'CXXFLAGS', 'CMFLAGS', 'CMMFLAGS', 'LDFLAGS']:
if v in context and context[v]:
passthru.variables['MOZBUILD_' + v] = context[v]
# NO_VISIBILITY_FLAGS is slightly different
if context['NO_VISIBILITY_FLAGS']:
passthru.variables['VISIBILITY_FLAGS'] = ''
no_pgo = context.get('NO_PGO')
sources = context.get('SOURCES', [])
no_pgo_sources = [f for f in sources if sources[f].no_pgo]
if no_pgo:
if no_pgo_sources:
raise SandboxValidationError('NO_PGO and SOURCES[...].no_pgo '
'cannot be set at the same time', context)
passthru.variables['NO_PROFILE_GUIDED_OPTIMIZE'] = no_pgo
if no_pgo_sources:
passthru.variables['NO_PROFILE_GUIDED_OPTIMIZE'] = no_pgo_sources
# A map from "canonical suffixes" for a particular source file
# language to the range of suffixes associated with that language.
#
# We deliberately don't list the canonical suffix in the suffix list
# in the definition; we'll add it in programmatically after defining
# things.
suffix_map = {
'.s': set(['.asm']),
'.c': set(),
'.m': set(),
'.mm': set(),
'.cpp': set(['.cc', '.cxx']),
'.S': set(),
}
# The inverse of the above, mapping suffixes to their canonical suffix.
canonicalized_suffix_map = {}
for suffix, alternatives in suffix_map.iteritems():
alternatives.add(suffix)
for a in alternatives:
canonicalized_suffix_map[a] = suffix
def canonical_suffix_for_file(f):
return canonicalized_suffix_map[mozpath.splitext(f)[1]]
# A map from moz.build variables to the canonical suffixes of file
# kinds that can be listed therein.
all_suffixes = list(suffix_map.keys())
varmap = dict(
SOURCES=(Sources, all_suffixes),
HOST_SOURCES=(HostSources, ['.c', '.mm', '.cpp']),
UNIFIED_SOURCES=(UnifiedSources, ['.c', '.mm', '.cpp']),
GENERATED_SOURCES=(GeneratedSources, all_suffixes),
)
for variable, (klass, suffixes) in varmap.items():
allowed_suffixes = set().union(*[suffix_map[s] for s in suffixes])
# First ensure that we haven't been given filetypes that we don't
# recognize.
for f in context[variable]:
ext = mozpath.splitext(f)[1]
if ext not in allowed_suffixes:
raise SandboxValidationError(
'%s has an unknown file type.' % f, context)
if variable.startswith('GENERATED_'):
l = passthru.variables.setdefault('GARBAGE', [])
l.append(f)
# Now sort the files to let groupby work.
sorted_files = sorted(context[variable], key=canonical_suffix_for_file)
for canonical_suffix, files in itertools.groupby(sorted_files, canonical_suffix_for_file):
arglist = [context, list(files), canonical_suffix]
if variable.startswith('UNIFIED_') and 'FILES_PER_UNIFIED_FILE' in context:
arglist.append(context['FILES_PER_UNIFIED_FILE'])
yield klass(*arglist)
sources_with_flags = [f for f in sources if sources[f].flags]
for f in sources_with_flags:
ext = mozpath.splitext(f)[1]
yield PerSourceFlag(context, f, sources[f].flags)
exports = context.get('EXPORTS')
if exports:
yield Exports(context, exports,
dist_install=not context.get('NO_DIST_INSTALL', False))
generated_files = context.get('GENERATED_FILES')
if generated_files:
for f in generated_files:
flags = generated_files[f]
output = f
if flags.script:
script = mozpath.join(context.srcdir, flags.script)
inputs = [mozpath.join(context.srcdir, i) for i in flags.inputs]
if not os.path.exists(script):
raise SandboxValidationError(
'Script for generating %s does not exist: %s'
% (f, script), context)
if os.path.splitext(script)[1] != '.py':
raise SandboxValidationError(
'Script for generating %s does not end in .py: %s'
% (f, script), context)
for i in inputs:
if not os.path.exists(i):
raise SandboxValidationError(
'Input for generating %s does not exist: %s'
% (f, i), context)
else:
script = None
inputs = []
yield GeneratedFile(context, script, output, inputs)
test_harness_files = context.get('TEST_HARNESS_FILES')
if test_harness_files:
srcdir_files = defaultdict(list)
srcdir_pattern_files = defaultdict(list)
objdir_files = defaultdict(list)
for path, strings in test_harness_files.walk():
if not path and strings:
raise SandboxValidationError(
'Cannot install files to the root of TEST_HARNESS_FILES', context)
for s in strings:
if context.is_objdir_path(s):
if s.startswith('!/'):
objdir_files[path].append('$(DEPTH)/%s' % s[2:])
else:
objdir_files[path].append(s[1:])
else:
resolved = context.resolve_path(s)
if '*' in s:
srcdir_pattern_files[path].append(s);
elif not os.path.exists(resolved):
raise SandboxValidationError(
'File listed in TEST_HARNESS_FILES does not exist: %s' % s, context)
else:
srcdir_files[path].append(resolved)
yield TestHarnessFiles(context, srcdir_files,
srcdir_pattern_files, objdir_files)
defines = context.get('DEFINES')
if defines:
yield Defines(context, defines)
resources = context.get('RESOURCE_FILES')
if resources:
yield Resources(context, resources, defines)
for pref in sorted(context['JS_PREFERENCE_FILES']):
yield JsPreferenceFile(context, pref)
for kind, cls in [('PROGRAM', Program), ('HOST_PROGRAM', HostProgram)]:
program = context.get(kind)
if program:
if program in self._binaries:
raise SandboxValidationError(
'Cannot use "%s" as %s name, '
'because it is already used in %s' % (program, kind,
self._binaries[program].relativedir), context)
self._binaries[program] = cls(context, program)
self._linkage.append((context, self._binaries[program],
kind.replace('PROGRAM', 'USE_LIBS')))
for kind, cls in [
('SIMPLE_PROGRAMS', SimpleProgram),
('CPP_UNIT_TESTS', SimpleProgram),
('HOST_SIMPLE_PROGRAMS', HostSimpleProgram)]:
for program in context[kind]:
if program in self._binaries:
raise SandboxValidationError(
'Cannot use "%s" in %s, '
'because it is already used in %s' % (program, kind,
self._binaries[program].relativedir), context)
self._binaries[program] = cls(context, program,
is_unit_test=kind == 'CPP_UNIT_TESTS')
self._linkage.append((context, self._binaries[program],
'HOST_USE_LIBS' if kind == 'HOST_SIMPLE_PROGRAMS'
else 'USE_LIBS'))
extra_js_modules = context.get('EXTRA_JS_MODULES')
if extra_js_modules:
yield JavaScriptModules(context, extra_js_modules, 'extra')
extra_pp_js_modules = context.get('EXTRA_PP_JS_MODULES')
if extra_pp_js_modules:
yield JavaScriptModules(context, extra_pp_js_modules, 'extra_pp')
test_js_modules = context.get('TESTING_JS_MODULES')
if test_js_modules:
yield JavaScriptModules(context, test_js_modules, 'testing')
simple_lists = [
('GENERATED_EVENTS_WEBIDL_FILES', GeneratedEventWebIDLFile),
('GENERATED_WEBIDL_FILES', GeneratedWebIDLFile),
('IPDL_SOURCES', IPDLFile),
('GENERATED_INCLUDES', GeneratedInclude),
('PREPROCESSED_TEST_WEBIDL_FILES', PreprocessedTestWebIDLFile),
('PREPROCESSED_WEBIDL_FILES', PreprocessedWebIDLFile),
('TEST_WEBIDL_FILES', TestWebIDLFile),
('WEBIDL_FILES', WebIDLFile),
('WEBIDL_EXAMPLE_INTERFACES', ExampleWebIDLInterface),
]
for context_var, klass in simple_lists:
for name in context.get(context_var, []):
yield klass(context, name)
for local_include in context.get('LOCAL_INCLUDES', []):
if local_include.startswith('/'):
path = context.config.topsrcdir
relative_include = local_include[1:]
else:
path = context.srcdir
relative_include = local_include
actual_include = os.path.join(path, relative_include)
if not os.path.exists(actual_include):
raise SandboxValidationError('Path specified in LOCAL_INCLUDES '
'does not exist: %s (resolved to %s)' % (local_include, actual_include), context)
yield LocalInclude(context, local_include)
final_target_files = context.get('FINAL_TARGET_FILES')
if final_target_files:
yield FinalTargetFiles(context, final_target_files)
host_libname = context.get('HOST_LIBRARY_NAME')
libname = context.get('LIBRARY_NAME')
if host_libname:
if host_libname == libname:
raise SandboxValidationError('LIBRARY_NAME and '
'HOST_LIBRARY_NAME must have a different value', context)
lib = HostLibrary(context, host_libname)
self._libs[host_libname].append(lib)
self._linkage.append((context, lib, 'HOST_USE_LIBS'))
final_lib = context.get('FINAL_LIBRARY')
if not libname and final_lib:
# If no LIBRARY_NAME is given, create one.
libname = context.relsrcdir.replace('/', '_')
static_lib = context.get('FORCE_STATIC_LIB')
shared_lib = context.get('FORCE_SHARED_LIB')
static_name = context.get('STATIC_LIBRARY_NAME')
shared_name = context.get('SHARED_LIBRARY_NAME')
is_framework = context.get('IS_FRAMEWORK')
is_component = context.get('IS_COMPONENT')
soname = context.get('SONAME')
lib_defines = context.get('LIBRARY_DEFINES')
shared_args = {}
static_args = {}
if final_lib:
if static_lib:
raise SandboxValidationError(
'FINAL_LIBRARY implies FORCE_STATIC_LIB. '
'Please remove the latter.', context)
if shared_lib:
raise SandboxValidationError(
'FINAL_LIBRARY conflicts with FORCE_SHARED_LIB. '
'Please remove one.', context)
if is_framework:
raise SandboxValidationError(
'FINAL_LIBRARY conflicts with IS_FRAMEWORK. '
'Please remove one.', context)
if is_component:
raise SandboxValidationError(
'FINAL_LIBRARY conflicts with IS_COMPONENT. '
'Please remove one.', context)
static_args['link_into'] = final_lib
static_lib = True
if libname:
if is_component:
if static_lib:
raise SandboxValidationError(
'IS_COMPONENT conflicts with FORCE_STATIC_LIB. '
'Please remove one.', context)
shared_lib = True
shared_args['variant'] = SharedLibrary.COMPONENT
if is_framework:
if soname:
raise SandboxValidationError(
'IS_FRAMEWORK conflicts with SONAME. '
'Please remove one.', context)
shared_lib = True
shared_args['variant'] = SharedLibrary.FRAMEWORK
if not static_lib and not shared_lib:
static_lib = True
if static_name:
if not static_lib:
raise SandboxValidationError(
'STATIC_LIBRARY_NAME requires FORCE_STATIC_LIB',
context)
static_args['real_name'] = static_name
if shared_name:
if not shared_lib:
raise SandboxValidationError(
'SHARED_LIBRARY_NAME requires FORCE_SHARED_LIB',
context)
shared_args['real_name'] = shared_name
if soname:
if not shared_lib:
raise SandboxValidationError(
'SONAME requires FORCE_SHARED_LIB', context)
shared_args['soname'] = soname
# If both a shared and a static library are created, only the
# shared library is meant to be a SDK library.
if context.get('SDK_LIBRARY'):
if shared_lib:
shared_args['is_sdk'] = True
elif static_lib:
static_args['is_sdk'] = True
if shared_lib and static_lib:
if not static_name and not shared_name:
raise SandboxValidationError(
'Both FORCE_STATIC_LIB and FORCE_SHARED_LIB are True, '
'but neither STATIC_LIBRARY_NAME or '
'SHARED_LIBRARY_NAME is set. At least one is required.',
context)
if static_name and not shared_name and static_name == libname:
raise SandboxValidationError(
'Both FORCE_STATIC_LIB and FORCE_SHARED_LIB are True, '
'but STATIC_LIBRARY_NAME is the same as LIBRARY_NAME, '
'and SHARED_LIBRARY_NAME is unset. Please either '
'change STATIC_LIBRARY_NAME or LIBRARY_NAME, or set '
'SHARED_LIBRARY_NAME.', context)
if shared_name and not static_name and shared_name == libname:
raise SandboxValidationError(
'Both FORCE_STATIC_LIB and FORCE_SHARED_LIB are True, '
'but SHARED_LIBRARY_NAME is the same as LIBRARY_NAME, '
'and STATIC_LIBRARY_NAME is unset. Please either '
'change SHARED_LIBRARY_NAME or LIBRARY_NAME, or set '
'STATIC_LIBRARY_NAME.', context)
if shared_name and static_name and shared_name == static_name:
raise SandboxValidationError(
'Both FORCE_STATIC_LIB and FORCE_SHARED_LIB are True, '
'but SHARED_LIBRARY_NAME is the same as '
'STATIC_LIBRARY_NAME. Please change one of them.',
context)
if shared_lib:
lib = SharedLibrary(context, libname, **shared_args)
self._libs[libname].append(lib)
self._linkage.append((context, lib, 'USE_LIBS'))
if static_lib:
lib = StaticLibrary(context, libname, **static_args)
self._libs[libname].append(lib)
self._linkage.append((context, lib, 'USE_LIBS'))
if lib_defines:
if not libname:
raise SandboxValidationError('LIBRARY_DEFINES needs a '
'LIBRARY_NAME to take effect', context)
lib.defines.update(lib_defines)
# While there are multiple test manifests, the behavior is very similar
# across them. We enforce this by having common handling of all
# manifests and outputting a single class type with the differences
# described inside the instance.
#
# Keys are variable prefixes and values are tuples describing how these
# manifests should be handled:
#
# (flavor, install_prefix, package_tests)
#
# flavor identifies the flavor of this test.
# install_prefix is the path prefix of where to install the files in
# the tests directory.
# package_tests indicates whether to package test files into the test
# package; suites that compile the test files should not install
# them into the test package.
#
test_manifests = dict(
A11Y=('a11y', 'testing/mochitest', 'a11y', True),
BROWSER_CHROME=('browser-chrome', 'testing/mochitest', 'browser', True),
ANDROID_INSTRUMENTATION=('instrumentation', 'instrumentation', '.', False),
JETPACK_PACKAGE=('jetpack-package', 'testing/mochitest', 'jetpack-package', True),
JETPACK_ADDON=('jetpack-addon', 'testing/mochitest', 'jetpack-addon', False),
METRO_CHROME=('metro-chrome', 'testing/mochitest', 'metro', True),
MOCHITEST=('mochitest', 'testing/mochitest', 'tests', True),
MOCHITEST_CHROME=('chrome', 'testing/mochitest', 'chrome', True),
MOCHITEST_WEBAPPRT_CONTENT=('webapprt-content', 'testing/mochitest', 'webapprtContent', True),
MOCHITEST_WEBAPPRT_CHROME=('webapprt-chrome', 'testing/mochitest', 'webapprtChrome', True),
WEBRTC_SIGNALLING_TEST=('steeplechase', 'steeplechase', '.', True),
XPCSHELL_TESTS=('xpcshell', 'xpcshell', '.', True),
)
for prefix, info in test_manifests.items():
for path in context.get('%s_MANIFESTS' % prefix, []):
for obj in self._process_test_manifest(context, info, path):
yield obj
for flavor in ('crashtest', 'reftest'):
for path in context.get('%s_MANIFESTS' % flavor.upper(), []):
for obj in self._process_reftest_manifest(context, flavor, path):
yield obj
jar_manifests = context.get('JAR_MANIFESTS', [])
if len(jar_manifests) > 1:
raise SandboxValidationError('While JAR_MANIFESTS is a list, '
'it is currently limited to one value.', context)
for path in jar_manifests:
yield JARManifest(context, mozpath.join(context.srcdir, path))
# Temporary test to look for jar.mn files that creep in without using
# the new declaration. Before, we didn't require jar.mn files to
# declared anywhere (they were discovered). This will detect people
# relying on the old behavior.
if os.path.exists(os.path.join(context.srcdir, 'jar.mn')):
if 'jar.mn' not in jar_manifests:
raise SandboxValidationError('A jar.mn exists but it '
'is not referenced in the moz.build file. '
'Please define JAR_MANIFESTS.', context)
for name, jar in context.get('JAVA_JAR_TARGETS', {}).items():
yield ContextWrapped(context, jar)
for name, data in context.get('ANDROID_ECLIPSE_PROJECT_TARGETS', {}).items():
yield ContextWrapped(context, data)
if passthru.variables:
yield passthru
def read_from_gyp(config, path, output, vars, non_unified_sources=set()):
"""Read a gyp configuration and emits GypContexts for the backend to
process.
config is a ConfigEnvironment, path is the path to a root gyp configuration
file, output is the base path under which the objdir for the various gyp
dependencies will be, and vars a dict of variables to pass to the gyp
processor.
"""
time_start = time.time()
# gyp expects plain str instead of unicode. The frontend code gives us
# unicode strings, so convert them.
path = encode(path)
str_vars = dict((name, encode(value)) for name, value in vars.items())
params = {
b'parallel': False,
b'generator_flags': {},
b'build_files': [path],
}
# Files that gyp_chromium always includes
includes = [encode(mozpath.join(script_dir, 'common.gypi'))]
finder = FileFinder(chrome_src, find_executables=False)
includes.extend(
encode(mozpath.join(chrome_src, name))
for name, _ in finder.find('*/supplement.gypi'))
# Read the given gyp file and its dependencies.
generator, flat_list, targets, data = \
gyp.Load([path], format=b'mozbuild',
default_variables=str_vars,
includes=includes,
depth=encode(mozpath.dirname(path)),
params=params)
# Process all targets from the given gyp files and its dependencies.
# The path given to AllTargets needs to use os.sep, while the frontend code
# gives us paths normalized with forward slash separator.
for target in gyp.common.AllTargets(flat_list, targets,
path.replace(b'/', os.sep)):
build_file, target_name, toolset = gyp.common.ParseQualifiedTarget(
target)
# Each target is given its own objdir. The base of that objdir
# is derived from the relative path from the root gyp file path
# to the current build_file, placed under the given output
# directory. Since several targets can be in a given build_file,
# separate them in subdirectories using the build_file basename
# and the target_name.
reldir = mozpath.relpath(mozpath.dirname(build_file),
mozpath.dirname(path))
subdir = '%s_%s' % (
mozpath.splitext(mozpath.basename(build_file))[0],
target_name,
)
# Emit a context for each target.
context = GypContext(
config,
mozpath.relpath(mozpath.join(output, reldir, subdir),
config.topobjdir))
context.add_source(mozpath.abspath(build_file))
# The list of included files returned by gyp are relative to build_file
for f in data[build_file]['included_files']:
context.add_source(
mozpath.abspath(mozpath.join(mozpath.dirname(build_file), f)))
spec = targets[target]
# Derive which gyp configuration to use based on MOZ_DEBUG.
c = 'Debug' if config.substs['MOZ_DEBUG'] else 'Release'
if c not in spec['configurations']:
raise RuntimeError('Missing %s gyp configuration for target %s '
'in %s' % (c, target_name, build_file))
target_conf = spec['configurations'][c]
if spec['type'] == 'none':
continue
elif spec['type'] == 'static_library':
# Remove leading 'lib' from the target_name if any, and use as
# library name.
name = spec['target_name']
if name.startswith('lib'):
name = name[3:]
# The context expects an unicode string.
context['LIBRARY_NAME'] = name.decode('utf-8')
# gyp files contain headers and asm sources in sources lists.
sources = []
unified_sources = []
extensions = set()
for f in spec.get('sources', []):
ext = mozpath.splitext(f)[-1]
extensions.add(ext)
s = SourcePath(context, f)
if ext == '.h':
continue
if ext != '.S' and s not in non_unified_sources:
unified_sources.append(s)
else:
sources.append(s)
# The context expects alphabetical order when adding sources
context['SOURCES'] = alphabetical_sorted(sources)
context['UNIFIED_SOURCES'] = alphabetical_sorted(unified_sources)
for define in target_conf.get('defines', []):
if '=' in define:
name, value = define.split('=', 1)
context['DEFINES'][name] = value
else:
context['DEFINES'][define] = True
for include in target_conf.get('include_dirs', []):
# moz.build expects all LOCAL_INCLUDES to exist, so ensure they do.
#
# NB: gyp files sometimes have actual absolute paths (e.g.
# /usr/include32) and sometimes paths that moz.build considers
# absolute, i.e. starting from topsrcdir. There's no good way
# to tell them apart here, and the actual absolute paths are
# likely bogus. In any event, actual absolute paths will be
# filtered out by trying to find them in topsrcdir.
if include.startswith('/'):
resolved = mozpath.abspath(
mozpath.join(config.topsrcdir, include[1:]))
else:
resolved = mozpath.abspath(
mozpath.join(mozpath.dirname(build_file), include))
if not os.path.exists(resolved):
continue
context['LOCAL_INCLUDES'] += [include]
context['ASFLAGS'] = target_conf.get('asflags_mozilla', [])
flags = target_conf.get('cflags_mozilla', [])
if flags:
suffix_map = {
'.c': 'CFLAGS',
'.cpp': 'CXXFLAGS',
'.cc': 'CXXFLAGS',
'.m': 'CMFLAGS',
'.mm': 'CMMFLAGS',
}
variables = (suffix_map[e] for e in extensions
if e in suffix_map)
for var in variables:
context[var].extend(flags)
else:
# Ignore other types than static_library because we don't have
# anything using them, and we're not testing them. They can be
# added when that becomes necessary.
raise NotImplementedError('Unsupported gyp target type: %s' %
spec['type'])
# Add some features to all contexts. Put here in case LOCAL_INCLUDES
# order matters.
context['LOCAL_INCLUDES'] += [
'/ipc/chromium/src',
'/ipc/glue',
]
context['GENERATED_INCLUDES'] += ['/ipc/ipdl/_ipdlheaders']
# These get set via VC project file settings for normal GYP builds.
if config.substs['OS_TARGET'] == 'WINNT':
context['DEFINES']['UNICODE'] = True
context['DEFINES']['_UNICODE'] = True
context['DISABLE_STL_WRAPPING'] = True
context.execution_time = time.time() - time_start
yield context
time_start = time.time()
def source_files(self):
for srcs in self.sources.values():
for f in srcs:
if mozpath.basename(mozpath.splitext(f)[0]) == mozpath.splitext(self.program)[0]:
return [f]
return []
def read_from_gyp(config,
path,
output,
vars,
no_chromium,
no_unified,
action_overrides,
non_unified_sources=set()):
"""Read a gyp configuration and emits GypContexts for the backend to
process.
config is a ConfigEnvironment, path is the path to a root gyp configuration
file, output is the base path under which the objdir for the various gyp
dependencies will be, and vars a dict of variables to pass to the gyp
processor.
"""
is_win = config.substs['OS_TARGET'] == 'WINNT'
is_msvc = bool(config.substs['_MSC_VER'])
# gyp expects plain str instead of unicode. The frontend code gives us
# unicode strings, so convert them.
path = encode(path)
str_vars = dict((name, encode(value)) for name, value in vars.items())
if is_msvc:
# This isn't actually used anywhere in this generator, but it's needed
# to override the registry detection of VC++ in gyp.
os.environ['GYP_MSVS_OVERRIDE_PATH'] = 'fake_path'
os.environ['GYP_MSVS_VERSION'] = config.substs['MSVS_VERSION']
params = {
b'parallel': False,
b'generator_flags': {},
b'build_files': [path],
b'root_targets': None,
}
if no_chromium:
includes = []
depth = mozpath.dirname(path)
else:
depth = chrome_src
# Files that gyp_chromium always includes
includes = [encode(mozpath.join(script_dir, 'common.gypi'))]
finder = FileFinder(chrome_src, find_executables=False)
includes.extend(
encode(mozpath.join(chrome_src, name))
for name, _ in finder.find('*/supplement.gypi'))
# Read the given gyp file and its dependencies.
generator, flat_list, targets, data = \
gyp.Load([path], format=b'mozbuild',
default_variables=str_vars,
includes=includes,
depth=encode(depth),
params=params)
# Process all targets from the given gyp files and its dependencies.
# The path given to AllTargets needs to use os.sep, while the frontend code
# gives us paths normalized with forward slash separator.
for target in gyp.common.AllTargets(flat_list, targets,
path.replace(b'/', os.sep)):
build_file, target_name, toolset = gyp.common.ParseQualifiedTarget(
target)
# Each target is given its own objdir. The base of that objdir
# is derived from the relative path from the root gyp file path
# to the current build_file, placed under the given output
# directory. Since several targets can be in a given build_file,
# separate them in subdirectories using the build_file basename
# and the target_name.
reldir = mozpath.relpath(mozpath.dirname(build_file),
mozpath.dirname(path))
subdir = '%s_%s' % (
mozpath.splitext(mozpath.basename(build_file))[0],
target_name,
)
# Emit a context for each target.
context = GypContext(
config,
mozpath.relpath(mozpath.join(output, reldir, subdir),
config.topobjdir))
context.add_source(mozpath.abspath(build_file))
# The list of included files returned by gyp are relative to build_file
for f in data[build_file]['included_files']:
context.add_source(
mozpath.abspath(mozpath.join(mozpath.dirname(build_file), f)))
spec = targets[target]
# Derive which gyp configuration to use based on MOZ_DEBUG.
c = 'Debug' if config.substs['MOZ_DEBUG'] else 'Release'
if c not in spec['configurations']:
raise RuntimeError('Missing %s gyp configuration for target %s '
'in %s' % (c, target_name, build_file))
target_conf = spec['configurations'][c]
if 'actions' in spec:
handle_actions(spec['actions'], context, action_overrides)
if 'copies' in spec:
handle_copies(spec['copies'], context)
use_libs = []
libs = []
def add_deps(s):
for t in s.get('dependencies', []) + s.get('dependencies_original',
[]):
ty = targets[t]['type']
if ty in ('static_library', 'shared_library'):
use_libs.append(targets[t]['target_name'])
# Manually expand out transitive dependencies--
# gyp won't do this for static libs or none targets.
if ty in ('static_library', 'none'):
add_deps(targets[t])
libs.extend(spec.get('libraries', []))
#XXX: this sucks, but webrtc breaks with this right now because
# it builds a library called 'gtest' and we just get lucky
# that it isn't in USE_LIBS by that name anywhere.
if no_chromium:
add_deps(spec)
os_libs = []
for l in libs:
if l.startswith('-'):
os_libs.append(l)
elif l.endswith('.lib'):
os_libs.append(l[:-4])
elif l:
# For library names passed in from moz.build.
use_libs.append(os.path.basename(l))
if spec['type'] == 'none':
if not ('actions' in spec or 'copies' in spec):
continue
elif spec['type'] in ('static_library', 'shared_library',
'executable'):
# Remove leading 'lib' from the target_name if any, and use as
# library name.
name = spec['target_name']
if spec['type'] in ('static_library', 'shared_library'):
if name.startswith('lib'):
name = name[3:]
# The context expects an unicode string.
context['LIBRARY_NAME'] = name.decode('utf-8')
else:
context['PROGRAM'] = name.decode('utf-8')
if spec['type'] == 'shared_library':
context['FORCE_SHARED_LIB'] = True
elif spec['type'] == 'static_library' and spec.get(
'variables', {}).get('no_expand_libs', '0') == '1':
# PSM links a NSS static library, but our folded libnss
# doesn't actually export everything that all of the
# objects within would need, so that one library
# should be built as a real static library.
context['NO_EXPAND_LIBS'] = True
if use_libs:
context['USE_LIBS'] = sorted(use_libs, key=lambda s: s.lower())
if os_libs:
context['OS_LIBS'] = os_libs
# gyp files contain headers and asm sources in sources lists.
sources = []
unified_sources = []
extensions = set()
use_defines_in_asflags = False
for f in spec.get('sources', []):
ext = mozpath.splitext(f)[-1]
extensions.add(ext)
if f.startswith('$INTERMEDIATE_DIR/'):
s = ObjDirPath(context, f.replace('$INTERMEDIATE_DIR/',
'!'))
else:
s = SourcePath(context, f)
if ext == '.h':
continue
if ext == '.def':
context['SYMBOLS_FILE'] = s
elif ext != '.S' and not no_unified and s not in non_unified_sources:
unified_sources.append(s)
else:
sources.append(s)
# The Mozilla build system doesn't use DEFINES for building
# ASFILES.
if ext == '.s':
use_defines_in_asflags = True
# The context expects alphabetical order when adding sources
context['SOURCES'] = alphabetical_sorted(sources)
context['UNIFIED_SOURCES'] = alphabetical_sorted(unified_sources)
defines = target_conf.get('defines', [])
if is_msvc and no_chromium:
msvs_settings = gyp.msvs_emulation.MsvsSettings(spec, {})
defines.extend(msvs_settings.GetComputedDefines(c))
for define in defines:
if '=' in define:
name, value = define.split('=', 1)
context['DEFINES'][name] = value
else:
context['DEFINES'][define] = True
product_dir_dist = '$PRODUCT_DIR/dist/'
for include in target_conf.get('include_dirs', []):
if include.startswith(product_dir_dist):
# special-case includes of <(PRODUCT_DIR)/dist/ to match
# handle_copies above. This is used for NSS' exports.
include = '!/dist/include/' + include[len(product_dir_dist
):]
elif include.startswith(config.topobjdir):
# NSPR_INCLUDE_DIR gets passed into the NSS build this way.
include = '!/' + mozpath.relpath(include, config.topobjdir)
else:
# moz.build expects all LOCAL_INCLUDES to exist, so ensure they do.
#
# NB: gyp files sometimes have actual absolute paths (e.g.
# /usr/include32) and sometimes paths that moz.build considers
# absolute, i.e. starting from topsrcdir. There's no good way
# to tell them apart here, and the actual absolute paths are
# likely bogus. In any event, actual absolute paths will be
# filtered out by trying to find them in topsrcdir.
if include.startswith('/'):
resolved = mozpath.abspath(
mozpath.join(config.topsrcdir, include[1:]))
else:
resolved = mozpath.abspath(
mozpath.join(mozpath.dirname(build_file), include))
if not os.path.exists(resolved):
continue
context['LOCAL_INCLUDES'] += [include]
context['ASFLAGS'] = target_conf.get('asflags_mozilla', [])
if use_defines_in_asflags and defines:
context['ASFLAGS'] += ['-D' + d for d in defines]
flags = target_conf.get('cflags_mozilla', [])
if flags:
suffix_map = {
'.c': 'CFLAGS',
'.cpp': 'CXXFLAGS',
'.cc': 'CXXFLAGS',
'.m': 'CMFLAGS',
'.mm': 'CMMFLAGS',
}
variables = (suffix_map[e] for e in extensions
if e in suffix_map)
for var in variables:
for f in flags:
# We may be getting make variable references out of the
# gyp data, and we don't want those in emitted data, so
# substitute them with their actual value.
f = expand_variables(f, config.substs).split()
if not f:
continue
# the result may be a string or a list.
if isinstance(f, types.StringTypes):
context[var].append(f)
else:
context[var].extend(f)
else:
# Ignore other types because we don't have
# anything using them, and we're not testing them. They can be
# added when that becomes necessary.
raise NotImplementedError('Unsupported gyp target type: %s' %
spec['type'])
if not no_chromium:
# Add some features to all contexts. Put here in case LOCAL_INCLUDES
# order matters.
context['LOCAL_INCLUDES'] += [
'!/ipc/ipdl/_ipdlheaders',
'/ipc/chromium/src',
'/ipc/glue',
]
# These get set via VC project file settings for normal GYP builds.
if is_win:
context['DEFINES']['UNICODE'] = True
context['DEFINES']['_UNICODE'] = True
context['DISABLE_STL_WRAPPING'] = True
yield context
def process_gn_config(gn_config, srcdir, config, output, non_unified_sources,
sandbox_vars, mozilla_flags):
# Translates a json gn config into attributes that can be used to write out
# moz.build files for this configuration.
# Much of this code is based on similar functionality in `gyp_reader.py`.
mozbuild_attrs = {
"mozbuild_args": gn_config.get("mozbuild_args", None),
"dirs": {}
}
targets = gn_config["targets"]
project_relsrcdir = mozpath.relpath(srcdir, config.topsrcdir)
non_unified_sources = set(
[mozpath.normpath(s) for s in non_unified_sources])
def target_info(fullname):
path, name = target_fullname.split(":")
# Stripping '//' gives us a path relative to the project root,
# adding a suffix avoids name collisions with libraries already
# in the tree (like "webrtc").
return path.lstrip("//"), name + "_gn"
# Process all targets from the given gn project and its dependencies.
for target_fullname, spec in six.iteritems(targets):
target_path, target_name = target_info(target_fullname)
context_attrs = {}
# Remove leading 'lib' from the target_name if any, and use as
# library name.
name = target_name
if spec["type"] in ("static_library", "shared_library", "source_set"):
if name.startswith("lib"):
name = name[3:]
context_attrs["LIBRARY_NAME"] = six.ensure_text(name)
else:
raise Exception("The following GN target type is not currently "
'consumed by moz.build: "%s". It may need to be '
"added, or you may need to re-run the "
"`GnConfigGen` step." % spec["type"])
if spec["type"] == "shared_library":
context_attrs["FORCE_SHARED_LIB"] = True
sources = []
unified_sources = []
extensions = set()
use_defines_in_asflags = False
for f in spec.get("sources", []):
f = f.lstrip("//")
ext = mozpath.splitext(f)[-1]
extensions.add(ext)
src = "%s/%s" % (project_relsrcdir, f)
if ext == ".h":
continue
elif ext == ".def":
context_attrs["SYMBOLS_FILE"] = src
elif ext != ".S" and src not in non_unified_sources:
unified_sources.append("/%s" % src)
else:
sources.append("/%s" % src)
# The Mozilla build system doesn't use DEFINES for building
# ASFILES.
if ext == ".s":
use_defines_in_asflags = True
context_attrs["SOURCES"] = sources
context_attrs["UNIFIED_SOURCES"] = unified_sources
context_attrs["DEFINES"] = {}
for define in spec.get("defines", []):
if "=" in define:
name, value = define.split("=", 1)
context_attrs["DEFINES"][name] = value
else:
context_attrs["DEFINES"][define] = True
context_attrs["LOCAL_INCLUDES"] = []
for include in spec.get("include_dirs", []):
# GN will have resolved all these paths relative to the root of
# the project indicated by "//".
if include.startswith("//"):
include = include[2:]
# moz.build expects all LOCAL_INCLUDES to exist, so ensure they do.
if include.startswith("/"):
resolved = mozpath.abspath(
mozpath.join(config.topsrcdir, include[1:]))
else:
resolved = mozpath.abspath(mozpath.join(srcdir, include))
if not os.path.exists(resolved):
# GN files may refer to include dirs that are outside of the
# tree or we simply didn't vendor. Print a warning in this case.
if not resolved.endswith("gn-output/gen"):
print(
"Included path: '%s' does not exist, dropping include from GN "
"configuration." % resolved,
file=sys.stderr,
)
continue
if not include.startswith("/"):
include = "/%s/%s" % (project_relsrcdir, include)
context_attrs["LOCAL_INCLUDES"] += [include]
context_attrs["ASFLAGS"] = spec.get("asflags_mozilla", [])
if use_defines_in_asflags and context_attrs["DEFINES"]:
context_attrs["ASFLAGS"] += [
"-D" + d for d in context_attrs["DEFINES"]
]
flags = [_f for _f in spec.get("cflags", []) if _f in mozilla_flags]
if flags:
suffix_map = {
".c": "CFLAGS",
".cpp": "CXXFLAGS",
".cc": "CXXFLAGS",
".m": "CMFLAGS",
".mm": "CMMFLAGS",
}
variables = (suffix_map[e] for e in extensions if e in suffix_map)
for var in variables:
for f in flags:
# We may be getting make variable references out of the
# gn data, and we don't want those in emitted data, so
# substitute them with their actual value.
f = expand_variables(f, config.substs).split()
if not f:
continue
# the result may be a string or a list.
if isinstance(f, six.string_types):
context_attrs.setdefault(var, []).append(f)
else:
context_attrs.setdefault(var, []).extend(f)
context_attrs["OS_LIBS"] = []
for lib in spec.get("libs", []):
lib_name = os.path.splitext(lib)[0]
if lib.endswith(".framework"):
context_attrs["OS_LIBS"] += ["-framework " + lib_name]
else:
context_attrs["OS_LIBS"] += [lib_name]
# Add some features to all contexts. Put here in case LOCAL_INCLUDES
# order matters.
context_attrs["LOCAL_INCLUDES"] += [
"!/ipc/ipdl/_ipdlheaders",
"/ipc/chromium/src",
"/ipc/glue",
"/tools/profiler/public",
]
# These get set via VC project file settings for normal GYP builds.
# TODO: Determine if these defines are needed for GN builds.
if gn_config["mozbuild_args"]["OS_TARGET"] == "WINNT":
context_attrs["DEFINES"]["UNICODE"] = True
context_attrs["DEFINES"]["_UNICODE"] = True
context_attrs["COMPILE_FLAGS"] = {"OS_INCLUDES": []}
for key, value in sandbox_vars.items():
if context_attrs.get(key) and isinstance(context_attrs[key], list):
# If we have a key from sandbox_vars that's also been
# populated here we use the value from sandbox_vars as our
# basis rather than overriding outright.
context_attrs[key] = value + context_attrs[key]
elif context_attrs.get(key) and isinstance(context_attrs[key],
dict):
context_attrs[key].update(value)
else:
context_attrs[key] = value
target_relsrcdir = mozpath.join(project_relsrcdir, target_path,
target_name)
mozbuild_attrs["dirs"][target_relsrcdir] = context_attrs
return mozbuild_attrs
def generated_events_stems(self):
return [mozpath.splitext(b)[0] for b in self.generated_events_basenames()]
def all_regular_bindinggen_stems(self):
for stem in self.all_regular_stems():
yield '%sBinding' % stem
for source in self.generated_events_sources:
yield mozpath.splitext(mozpath.basename(source))[0]
def process_gn_config(gn_config, srcdir, config, output, non_unified_sources,
sandbox_vars, mozilla_flags):
# Translates a json gn config into attributes that can be used to write out
# moz.build files for this configuration.
# Much of this code is based on similar functionality in `gyp_reader.py`.
mozbuild_attrs = {
'mozbuild_args': gn_config.get('mozbuild_args', None),
'dirs': {}
}
targets = gn_config["targets"]
project_relsrcdir = mozpath.relpath(srcdir, config.topsrcdir)
def target_info(fullname):
path, name = target_fullname.split(':')
# Stripping '//' gives us a path relative to the project root,
# adding a suffix avoids name collisions with libraries already
# in the tree (like "webrtc").
return path.lstrip('//'), name + '_gn'
# Process all targets from the given gn project and its dependencies.
for target_fullname, spec in targets.iteritems():
target_path, target_name = target_info(target_fullname)
context_attrs = {}
# Remove leading 'lib' from the target_name if any, and use as
# library name.
name = target_name
if spec['type'] in ('static_library', 'shared_library', 'source_set'):
if name.startswith('lib'):
name = name[3:]
context_attrs['LIBRARY_NAME'] = name.decode('utf-8')
else:
raise Exception('The following GN target type is not currently '
'consumed by moz.build: "%s". It may need to be '
'added, or you may need to re-run the '
'`GnConfigGen` step.' % spec['type'])
if spec['type'] == 'shared_library':
context_attrs['FORCE_SHARED_LIB'] = True
sources = []
unified_sources = []
extensions = set()
use_defines_in_asflags = False
for f in spec.get('sources', []):
f = f.lstrip("//")
ext = mozpath.splitext(f)[-1]
extensions.add(ext)
src = '%s/%s' % (project_relsrcdir, f)
if ext == '.h':
continue
elif ext == '.def':
context_attrs['SYMBOLS_FILE'] = src
elif ext != '.S' and src not in non_unified_sources:
unified_sources.append('/%s' % src)
else:
sources.append('/%s' % src)
# The Mozilla build system doesn't use DEFINES for building
# ASFILES.
if ext == '.s':
use_defines_in_asflags = True
context_attrs['SOURCES'] = sources
context_attrs['UNIFIED_SOURCES'] = unified_sources
context_attrs['DEFINES'] = {}
for define in spec.get('defines', []):
if '=' in define:
name, value = define.split('=', 1)
context_attrs['DEFINES'][name] = value
else:
context_attrs['DEFINES'][define] = True
context_attrs['LOCAL_INCLUDES'] = []
for include in spec.get('include_dirs', []):
# GN will have resolved all these paths relative to the root of
# the project indicated by "//".
if include.startswith('//'):
include = include[2:]
# moz.build expects all LOCAL_INCLUDES to exist, so ensure they do.
if include.startswith('/'):
resolved = mozpath.abspath(
mozpath.join(config.topsrcdir, include[1:]))
else:
resolved = mozpath.abspath(mozpath.join(srcdir, include))
if not os.path.exists(resolved):
# GN files may refer to include dirs that are outside of the
# tree or we simply didn't vendor. Print a warning in this case.
if not resolved.endswith('gn-output/gen'):
print(
"Included path: '%s' does not exist, dropping include from GN "
"configuration." % resolved,
file=sys.stderr)
continue
if not include.startswith('/'):
include = '/%s/%s' % (project_relsrcdir, include)
context_attrs['LOCAL_INCLUDES'] += [include]
context_attrs['ASFLAGS'] = spec.get('asflags_mozilla', [])
if use_defines_in_asflags and defines:
context_attrs['ASFLAGS'] += ['-D' + d for d in defines]
flags = [f for f in spec.get('cflags', []) if f in mozilla_flags]
if flags:
suffix_map = {
'.c': 'CFLAGS',
'.cpp': 'CXXFLAGS',
'.cc': 'CXXFLAGS',
'.m': 'CMFLAGS',
'.mm': 'CMMFLAGS',
}
variables = (suffix_map[e] for e in extensions if e in suffix_map)
for var in variables:
for f in flags:
# We may be getting make variable references out of the
# gn data, and we don't want those in emitted data, so
# substitute them with their actual value.
f = expand_variables(f, config.substs).split()
if not f:
continue
# the result may be a string or a list.
if isinstance(f, types.StringTypes):
context_attrs.setdefault(var, []).append(f)
else:
context_attrs.setdefault(var, []).extend(f)
context_attrs['OS_LIBS'] = []
for lib in spec.get('libs', []):
lib_name = os.path.splitext(lib)[0]
if lib.endswith('.framework'):
context_attrs['OS_LIBS'] += ['-framework ' + lib_name]
else:
context_attrs['OS_LIBS'] += [lib_name]
# Add some features to all contexts. Put here in case LOCAL_INCLUDES
# order matters.
context_attrs['LOCAL_INCLUDES'] += [
'!/ipc/ipdl/_ipdlheaders',
'/ipc/chromium/src',
'/ipc/glue',
]
# These get set via VC project file settings for normal GYP builds.
# TODO: Determine if these defines are needed for GN builds.
if gn_config['mozbuild_args']['OS_TARGET'] == 'WINNT':
context_attrs['DEFINES']['UNICODE'] = True
context_attrs['DEFINES']['_UNICODE'] = True
context_attrs['COMPILE_FLAGS'] = {
'STL': [],
'OS_INCLUDES': [],
}
for key, value in sandbox_vars.items():
if context_attrs.get(key) and isinstance(context_attrs[key], list):
# If we have a key from sandbox_vars that's also been
# populated here we use the value from sandbox_vars as our
# basis rather than overriding outright.
context_attrs[key] = value + context_attrs[key]
elif context_attrs.get(key) and isinstance(context_attrs[key],
dict):
context_attrs[key].update(value)
else:
context_attrs[key] = value
target_relsrcdir = mozpath.join(project_relsrcdir, target_path,
target_name)
mozbuild_attrs['dirs'][target_relsrcdir] = context_attrs
return mozbuild_attrs
def process_gyp_result(gyp_result, gyp_dir_attrs, path, config, output,
non_unified_sources, action_overrides):
flat_list, targets, data = gyp_result
no_chromium = gyp_dir_attrs.no_chromium
no_unified = gyp_dir_attrs.no_unified
# Process all targets from the given gyp files and its dependencies.
# The path given to AllTargets needs to use os.sep, while the frontend code
# gives us paths normalized with forward slash separator.
for target in gyp.common.AllTargets(flat_list, targets,
path.replace(b'/', os.sep)):
build_file, target_name, toolset = gyp.common.ParseQualifiedTarget(
target)
# Each target is given its own objdir. The base of that objdir
# is derived from the relative path from the root gyp file path
# to the current build_file, placed under the given output
# directory. Since several targets can be in a given build_file,
# separate them in subdirectories using the build_file basename
# and the target_name.
reldir = mozpath.relpath(mozpath.dirname(build_file),
mozpath.dirname(path))
subdir = '%s_%s' % (
mozpath.splitext(mozpath.basename(build_file))[0],
target_name,
)
# Emit a context for each target.
context = GypContext(
config,
mozpath.relpath(mozpath.join(output, reldir, subdir),
config.topobjdir))
context.add_source(mozpath.abspath(build_file))
# The list of included files returned by gyp are relative to build_file
for f in data[build_file]['included_files']:
context.add_source(
mozpath.abspath(mozpath.join(mozpath.dirname(build_file), f)))
spec = targets[target]
# Derive which gyp configuration to use based on MOZ_DEBUG.
c = 'Debug' if config.substs.get('MOZ_DEBUG') else 'Release'
if c not in spec['configurations']:
raise RuntimeError('Missing %s gyp configuration for target %s '
'in %s' % (c, target_name, build_file))
target_conf = spec['configurations'][c]
if 'actions' in spec:
handle_actions(spec['actions'], context, action_overrides)
if 'copies' in spec:
handle_copies(spec['copies'], context)
use_libs = []
libs = []
def add_deps(s):
for t in s.get('dependencies', []) + s.get('dependencies_original',
[]):
ty = targets[t]['type']
if ty in ('static_library', 'shared_library'):
use_libs.append(targets[t]['target_name'])
# Manually expand out transitive dependencies--
# gyp won't do this for static libs or none targets.
if ty in ('static_library', 'none'):
add_deps(targets[t])
libs.extend(spec.get('libraries', []))
#XXX: this sucks, but webrtc breaks with this right now because
# it builds a library called 'gtest' and we just get lucky
# that it isn't in USE_LIBS by that name anywhere.
if no_chromium:
add_deps(spec)
os_libs = []
for l in libs:
if l.startswith('-'):
os_libs.append(l)
elif l.endswith('.lib'):
os_libs.append(l[:-4])
elif l:
# For library names passed in from moz.build.
use_libs.append(os.path.basename(l))
if spec['type'] == 'none':
if not ('actions' in spec or 'copies' in spec):
continue
elif spec['type'] in ('static_library', 'shared_library',
'executable'):
# Remove leading 'lib' from the target_name if any, and use as
# library name.
name = spec['target_name']
if spec['type'] in ('static_library', 'shared_library'):
if name.startswith('lib'):
name = name[3:]
# The context expects an unicode string.
context['LIBRARY_NAME'] = name.decode('utf-8')
else:
context['PROGRAM'] = name.decode('utf-8')
if spec['type'] == 'shared_library':
context['FORCE_SHARED_LIB'] = True
elif spec['type'] == 'static_library' and spec.get(
'variables', {}).get('no_expand_libs', '0') == '1':
# PSM links a NSS static library, but our folded libnss
# doesn't actually export everything that all of the
# objects within would need, so that one library
# should be built as a real static library.
context['NO_EXPAND_LIBS'] = True
if use_libs:
context['USE_LIBS'] = sorted(use_libs, key=lambda s: s.lower())
if os_libs:
context['OS_LIBS'] = os_libs
# gyp files contain headers and asm sources in sources lists.
sources = []
unified_sources = []
extensions = set()
use_defines_in_asflags = False
for f in spec.get('sources', []):
ext = mozpath.splitext(f)[-1]
extensions.add(ext)
if f.startswith('$INTERMEDIATE_DIR/'):
s = ObjDirPath(context, f.replace('$INTERMEDIATE_DIR/',
'!'))
else:
s = SourcePath(context, f)
if ext == '.h':
continue
if ext == '.def':
context['SYMBOLS_FILE'] = s
elif ext != '.S' and not no_unified and s not in non_unified_sources:
unified_sources.append(s)
else:
sources.append(s)
# The Mozilla build system doesn't use DEFINES for building
# ASFILES.
if ext == '.s':
use_defines_in_asflags = True
# The context expects alphabetical order when adding sources
context['SOURCES'] = alphabetical_sorted(sources)
context['UNIFIED_SOURCES'] = alphabetical_sorted(unified_sources)
defines = target_conf.get('defines', [])
if config.substs['CC_TYPE'] == 'clang-cl' and no_chromium:
msvs_settings = gyp.msvs_emulation.MsvsSettings(spec, {})
defines.extend(msvs_settings.GetComputedDefines(c))
for define in defines:
if '=' in define:
name, value = define.split('=', 1)
# The NSS gyp file doesn't expose a way to override this
# currently, so we do so here.
if name == 'NSS_ALLOW_SSLKEYLOGFILE' and config.substs.get(
'RELEASE_OR_BETA', False):
continue
context['DEFINES'][name] = value
else:
context['DEFINES'][define] = True
product_dir_dist = '$PRODUCT_DIR/dist/'
for include in target_conf.get('include_dirs', []):
if include.startswith(product_dir_dist):
# special-case includes of <(PRODUCT_DIR)/dist/ to match
# handle_copies above. This is used for NSS' exports.
include = '!/dist/include/' + include[len(product_dir_dist
):]
elif include.startswith(config.topobjdir):
# NSPR_INCLUDE_DIR gets passed into the NSS build this way.
include = '!/' + mozpath.relpath(include, config.topobjdir)
else:
# moz.build expects all LOCAL_INCLUDES to exist, so ensure they do.
#
# NB: gyp files sometimes have actual absolute paths (e.g.
# /usr/include32) and sometimes paths that moz.build considers
# absolute, i.e. starting from topsrcdir. There's no good way
# to tell them apart here, and the actual absolute paths are
# likely bogus. In any event, actual absolute paths will be
# filtered out by trying to find them in topsrcdir.
#
# We do allow !- and %-prefixed paths, assuming they come
# from moz.build and will be handled the same way as if they
# were given to LOCAL_INCLUDES in moz.build.
if include.startswith('/'):
resolved = mozpath.abspath(
mozpath.join(config.topsrcdir, include[1:]))
elif not include.startswith(('!', '%')):
resolved = mozpath.abspath(
mozpath.join(mozpath.dirname(build_file), include))
if not include.startswith(
('!', '%')) and not os.path.exists(resolved):
continue
context['LOCAL_INCLUDES'] += [include]
context['ASFLAGS'] = target_conf.get('asflags_mozilla', [])
if use_defines_in_asflags and defines:
context['ASFLAGS'] += ['-D' + d for d in defines]
flags = target_conf.get('cflags_mozilla', [])
if flags:
suffix_map = {
'.c': 'CFLAGS',
'.cpp': 'CXXFLAGS',
'.cc': 'CXXFLAGS',
'.m': 'CMFLAGS',
'.mm': 'CMMFLAGS',
}
variables = (suffix_map[e] for e in extensions
if e in suffix_map)
for var in variables:
for f in flags:
# We may be getting make variable references out of the
# gyp data, and we don't want those in emitted data, so
# substitute them with their actual value.
f = expand_variables(f, config.substs).split()
if not f:
continue
# the result may be a string or a list.
if isinstance(f, types.StringTypes):
context[var].append(f)
else:
context[var].extend(f)
else:
# Ignore other types because we don't have
# anything using them, and we're not testing them. They can be
# added when that becomes necessary.
raise NotImplementedError('Unsupported gyp target type: %s' %
spec['type'])
if not no_chromium:
# Add some features to all contexts. Put here in case LOCAL_INCLUDES
# order matters.
context['LOCAL_INCLUDES'] += [
'!/ipc/ipdl/_ipdlheaders',
'/ipc/chromium/src',
'/ipc/glue',
]
# These get set via VC project file settings for normal GYP builds.
if config.substs['OS_TARGET'] == 'WINNT':
context['DEFINES']['UNICODE'] = True
context['DEFINES']['_UNICODE'] = True
context['COMPILE_FLAGS']['OS_INCLUDES'] = []
for key, value in gyp_dir_attrs.sandbox_vars.items():
if context.get(key) and isinstance(context[key], list):
# If we have a key from sanbox_vars that's also been
# populated here we use the value from sandbox_vars as our
# basis rather than overriding outright.
context[key] = value + context[key]
elif context.get(key) and isinstance(context[key], dict):
context[key].update(value)
else:
context[key] = value
yield context
def add_idls(self, idls):
self.idl_files.update(idl.full_path for idl in idls)
self.directories.update(
mozpath.dirname(idl.full_path) for idl in idls)
self._stems.update(
mozpath.splitext(mozpath.basename(idl))[0] for idl in idls)
def all_stems(self):
return [mozpath.splitext(b)[0] for b in self.all_basenames()]
def all_stems(self):
return [mozpath.splitext(b)[0] for b in self.all_basenames()]
def emit_from_sandbox(self, sandbox):
"""Convert a MozbuildSandbox to tree metadata objects.
This is a generator of mozbuild.frontend.data.SandboxDerived instances.
"""
# We always emit a directory traversal descriptor. This is needed by
# the recursive make backend.
for o in self._emit_directory_traversal_from_sandbox(sandbox): yield o
for path in sandbox['CONFIGURE_SUBST_FILES']:
yield self._create_substitution(ConfigFileSubstitution, sandbox,
path)
for path in sandbox['CONFIGURE_DEFINE_FILES']:
yield self._create_substitution(HeaderFileSubstitution, sandbox,
path)
# XPIDL source files get processed and turned into .h and .xpt files.
# If there are multiple XPIDL files in a directory, they get linked
# together into a final .xpt, which has the name defined by
# XPIDL_MODULE.
xpidl_module = sandbox['XPIDL_MODULE']
if sandbox['XPIDL_SOURCES'] and not xpidl_module:
raise SandboxValidationError('XPIDL_MODULE must be defined if '
'XPIDL_SOURCES is defined.', sandbox)
if xpidl_module and not sandbox['XPIDL_SOURCES']:
raise SandboxValidationError('XPIDL_MODULE cannot be defined '
'unless there are XPIDL_SOURCES', sandbox)
if sandbox['XPIDL_SOURCES'] and sandbox['NO_DIST_INSTALL']:
self.log(logging.WARN, 'mozbuild_warning', dict(
path=sandbox.main_path),
'{path}: NO_DIST_INSTALL has no effect on XPIDL_SOURCES.')
for idl in sandbox['XPIDL_SOURCES']:
yield XPIDLFile(sandbox, mozpath.join(sandbox['SRCDIR'], idl),
xpidl_module)
for symbol in ('SOURCES', 'HOST_SOURCES', 'UNIFIED_SOURCES'):
for src in (sandbox[symbol] or []):
if not os.path.exists(mozpath.join(sandbox['SRCDIR'], src)):
raise SandboxValidationError('Reference to a file that '
'doesn\'t exist in %s (%s)'
% (symbol, src), sandbox)
# Proxy some variables as-is until we have richer classes to represent
# them. We should aim to keep this set small because it violates the
# desired abstraction of the build definition away from makefiles.
passthru = VariablePassthru(sandbox)
varlist = [
'ANDROID_GENERATED_RESFILES',
'ANDROID_RES_DIRS',
'DISABLE_STL_WRAPPING',
'EXTRA_ASSEMBLER_FLAGS',
'EXTRA_COMPILE_FLAGS',
'EXTRA_COMPONENTS',
'EXTRA_DSO_LDOPTS',
'EXTRA_PP_COMPONENTS',
'FAIL_ON_WARNINGS',
'FILES_PER_UNIFIED_FILE',
'USE_STATIC_LIBS',
'GENERATED_FILES',
'IS_GYP_DIR',
'MSVC_ENABLE_PGO',
'NO_DIST_INSTALL',
'PYTHON_UNIT_TESTS',
'RCFILE',
'RESFILE',
'RCINCLUDE',
'DEFFILE',
'WIN32_EXE_LDFLAGS',
'LD_VERSION_SCRIPT',
]
for v in varlist:
if v in sandbox and sandbox[v]:
passthru.variables[v] = sandbox[v]
for v in ['CFLAGS', 'CXXFLAGS', 'CMFLAGS', 'CMMFLAGS', 'LDFLAGS']:
if v in sandbox and sandbox[v]:
passthru.variables['MOZBUILD_' + v] = sandbox[v]
# NO_VISIBILITY_FLAGS is slightly different
if sandbox['NO_VISIBILITY_FLAGS']:
passthru.variables['VISIBILITY_FLAGS'] = ''
if sandbox['DELAYLOAD_DLLS']:
passthru.variables['DELAYLOAD_LDFLAGS'] = [('-DELAYLOAD:%s' % dll) for dll in sandbox['DELAYLOAD_DLLS']]
passthru.variables['USE_DELAYIMP'] = True
varmap = dict(
SOURCES={
'.s': 'ASFILES',
'.asm': 'ASFILES',
'.c': 'CSRCS',
'.m': 'CMSRCS',
'.mm': 'CMMSRCS',
'.cc': 'CPPSRCS',
'.cpp': 'CPPSRCS',
'.cxx': 'CPPSRCS',
'.S': 'SSRCS',
},
HOST_SOURCES={
'.c': 'HOST_CSRCS',
'.mm': 'HOST_CMMSRCS',
'.cc': 'HOST_CPPSRCS',
'.cpp': 'HOST_CPPSRCS',
'.cxx': 'HOST_CPPSRCS',
},
UNIFIED_SOURCES={
'.c': 'UNIFIED_CSRCS',
'.mm': 'UNIFIED_CMMSRCS',
'.cc': 'UNIFIED_CPPSRCS',
'.cpp': 'UNIFIED_CPPSRCS',
'.cxx': 'UNIFIED_CPPSRCS',
}
)
varmap.update(dict(('GENERATED_%s' % k, v) for k, v in varmap.items()
if k in ('SOURCES', 'UNIFIED_SOURCES')))
for variable, mapping in varmap.items():
for f in sandbox[variable]:
ext = mozpath.splitext(f)[1]
if ext not in mapping:
raise SandboxValidationError(
'%s has an unknown file type.' % f, sandbox)
l = passthru.variables.setdefault(mapping[ext], [])
l.append(f)
if variable.startswith('GENERATED_'):
l = passthru.variables.setdefault('GARBAGE', [])
l.append(f)
no_pgo = sandbox.get('NO_PGO')
sources = sandbox.get('SOURCES', [])
no_pgo_sources = [f for f in sources if sources[f].no_pgo]
if no_pgo:
if no_pgo_sources:
raise SandboxValidationError('NO_PGO and SOURCES[...].no_pgo '
'cannot be set at the same time', sandbox)
passthru.variables['NO_PROFILE_GUIDED_OPTIMIZE'] = no_pgo
if no_pgo_sources:
passthru.variables['NO_PROFILE_GUIDED_OPTIMIZE'] = no_pgo_sources
sources_with_flags = [f for f in sources if sources[f].flags]
for f in sources_with_flags:
ext = mozpath.splitext(f)[1]
yield PerSourceFlag(sandbox, f, sources[f].flags)
exports = sandbox.get('EXPORTS')
if exports:
yield Exports(sandbox, exports,
dist_install=not sandbox.get('NO_DIST_INSTALL', False))
defines = sandbox.get('DEFINES')
if defines:
yield Defines(sandbox, defines)
resources = sandbox.get('RESOURCE_FILES')
if resources:
yield Resources(sandbox, resources, defines)
for kind, cls in [('PROGRAM', Program), ('HOST_PROGRAM', HostProgram)]:
program = sandbox.get(kind)
if program:
if program in self._binaries:
raise SandboxValidationError(
'Cannot use "%s" as %s name, '
'because it is already used in %s' % (program, kind,
self._binaries[program].relativedir), sandbox)
self._binaries[program] = cls(sandbox, program)
self._linkage.append((sandbox, self._binaries[program],
kind.replace('PROGRAM', 'USE_LIBS')))
for kind, cls in [
('SIMPLE_PROGRAMS', SimpleProgram),
('CPP_UNIT_TESTS', SimpleProgram),
('HOST_SIMPLE_PROGRAMS', HostSimpleProgram)]:
for program in sandbox[kind]:
if program in self._binaries:
raise SandboxValidationError(
'Cannot use "%s" in %s, '
'because it is already used in %s' % (program, kind,
self._binaries[program].relativedir), sandbox)
self._binaries[program] = cls(sandbox, program,
is_unit_test=kind == 'CPP_UNIT_TESTS')
self._linkage.append((sandbox, self._binaries[program],
'HOST_USE_LIBS' if kind == 'HOST_SIMPLE_PROGRAMS'
else 'USE_LIBS'))
extra_js_modules = sandbox.get('EXTRA_JS_MODULES')
if extra_js_modules:
yield JavaScriptModules(sandbox, extra_js_modules, 'extra')
extra_pp_js_modules = sandbox.get('EXTRA_PP_JS_MODULES')
if extra_pp_js_modules:
yield JavaScriptModules(sandbox, extra_pp_js_modules, 'extra_pp')
test_js_modules = sandbox.get('TESTING_JS_MODULES')
if test_js_modules:
yield JavaScriptModules(sandbox, test_js_modules, 'testing')
simple_lists = [
('GENERATED_EVENTS_WEBIDL_FILES', GeneratedEventWebIDLFile),
('GENERATED_WEBIDL_FILES', GeneratedWebIDLFile),
('IPDL_SOURCES', IPDLFile),
('LOCAL_INCLUDES', LocalInclude),
('GENERATED_INCLUDES', GeneratedInclude),
('PREPROCESSED_TEST_WEBIDL_FILES', PreprocessedTestWebIDLFile),
('PREPROCESSED_WEBIDL_FILES', PreprocessedWebIDLFile),
('TEST_WEBIDL_FILES', TestWebIDLFile),
('WEBIDL_FILES', WebIDLFile),
('WEBIDL_EXAMPLE_INTERFACES', ExampleWebIDLInterface),
]
for sandbox_var, klass in simple_lists:
for name in sandbox.get(sandbox_var, []):
yield klass(sandbox, name)
if sandbox.get('FINAL_TARGET') or sandbox.get('XPI_NAME') or \
sandbox.get('DIST_SUBDIR'):
yield InstallationTarget(sandbox)
host_libname = sandbox.get('HOST_LIBRARY_NAME')
libname = sandbox.get('LIBRARY_NAME')
if host_libname:
if host_libname == libname:
raise SandboxValidationError('LIBRARY_NAME and '
'HOST_LIBRARY_NAME must have a different value', sandbox)
lib = HostLibrary(sandbox, host_libname)
self._libs[host_libname].append(lib)
self._linkage.append((sandbox, lib, 'HOST_USE_LIBS'))
final_lib = sandbox.get('FINAL_LIBRARY')
if not libname and final_lib:
# If no LIBRARY_NAME is given, create one.
libname = sandbox['RELATIVEDIR'].replace('/', '_')
static_lib = sandbox.get('FORCE_STATIC_LIB')
shared_lib = sandbox.get('FORCE_SHARED_LIB')
static_name = sandbox.get('STATIC_LIBRARY_NAME')
shared_name = sandbox.get('SHARED_LIBRARY_NAME')
is_framework = sandbox.get('IS_FRAMEWORK')
is_component = sandbox.get('IS_COMPONENT')
soname = sandbox.get('SONAME')
shared_args = {}
static_args = {}
if final_lib:
if isinstance(sandbox, MozbuildSandbox):
if static_lib:
raise SandboxValidationError(
'FINAL_LIBRARY implies FORCE_STATIC_LIB. '
'Please remove the latter.', sandbox)
if shared_lib:
raise SandboxValidationError(
'FINAL_LIBRARY conflicts with FORCE_SHARED_LIB. '
'Please remove one.', sandbox)
if is_framework:
raise SandboxValidationError(
'FINAL_LIBRARY conflicts with IS_FRAMEWORK. '
'Please remove one.', sandbox)
if is_component:
raise SandboxValidationError(
'FINAL_LIBRARY conflicts with IS_COMPONENT. '
'Please remove one.', sandbox)
static_args['link_into'] = final_lib
static_lib = True
if libname:
if is_component:
if shared_lib:
raise SandboxValidationError(
'IS_COMPONENT implies FORCE_SHARED_LIB. '
'Please remove the latter.', sandbox)
if is_framework:
raise SandboxValidationError(
'IS_COMPONENT conflicts with IS_FRAMEWORK. '
'Please remove one.', sandbox)
if static_lib:
raise SandboxValidationError(
'IS_COMPONENT conflicts with FORCE_STATIC_LIB. '
'Please remove one.', sandbox)
shared_lib = True
shared_args['variant'] = SharedLibrary.COMPONENT
if is_framework:
if shared_lib:
raise SandboxValidationError(
'IS_FRAMEWORK implies FORCE_SHARED_LIB. '
'Please remove the latter.', sandbox)
if soname:
raise SandboxValidationError(
'IS_FRAMEWORK conflicts with SONAME. '
'Please remove one.', sandbox)
shared_lib = True
shared_args['variant'] = SharedLibrary.FRAMEWORK
if static_name:
if not static_lib:
raise SandboxValidationError(
'STATIC_LIBRARY_NAME requires FORCE_STATIC_LIB', sandbox)
static_args['real_name'] = static_name
if shared_name:
if not shared_lib:
raise SandboxValidationError(
'SHARED_LIBRARY_NAME requires FORCE_SHARED_LIB', sandbox)
shared_args['real_name'] = shared_name
if soname:
if not shared_lib:
raise SandboxValidationError(
'SONAME requires FORCE_SHARED_LIB', sandbox)
shared_args['soname'] = soname
if not static_lib and not shared_lib:
static_lib = True
# If both a shared and a static library are created, only the
# shared library is meant to be a SDK library.
if sandbox.get('SDK_LIBRARY'):
if shared_lib:
shared_args['is_sdk'] = True
elif static_lib:
static_args['is_sdk'] = True
if shared_lib and static_lib:
if not static_name and not shared_name:
raise SandboxValidationError(
'Both FORCE_STATIC_LIB and FORCE_SHARED_LIB are True, '
'but neither STATIC_LIBRARY_NAME or '
'SHARED_LIBRARY_NAME is set. At least one is required.',
sandbox)
if static_name and not shared_name and static_name == libname:
raise SandboxValidationError(
'Both FORCE_STATIC_LIB and FORCE_SHARED_LIB are True, '
'but STATIC_LIBRARY_NAME is the same as LIBRARY_NAME, '
'and SHARED_LIBRARY_NAME is unset. Please either '
'change STATIC_LIBRARY_NAME or LIBRARY_NAME, or set '
'SHARED_LIBRARY_NAME.', sandbox)
if shared_name and not static_name and shared_name == libname:
raise SandboxValidationError(
'Both FORCE_STATIC_LIB and FORCE_SHARED_LIB are True, '
'but SHARED_LIBRARY_NAME is the same as LIBRARY_NAME, '
'and STATIC_LIBRARY_NAME is unset. Please either '
'change SHARED_LIBRARY_NAME or LIBRARY_NAME, or set '
'STATIC_LIBRARY_NAME.', sandbox)
if shared_name and static_name and shared_name == static_name:
raise SandboxValidationError(
'Both FORCE_STATIC_LIB and FORCE_SHARED_LIB are True, '
'but SHARED_LIBRARY_NAME is the same as '
'STATIC_LIBRARY_NAME. Please change one of them.',
sandbox)
if shared_lib:
lib = SharedLibrary(sandbox, libname, **shared_args)
self._libs[libname].append(lib)
self._linkage.append((sandbox, lib, 'USE_LIBS'))
if static_lib:
lib = StaticLibrary(sandbox, libname, **static_args)
self._libs[libname].append(lib)
self._linkage.append((sandbox, lib, 'USE_LIBS'))
# While there are multiple test manifests, the behavior is very similar
# across them. We enforce this by having common handling of all
# manifests and outputting a single class type with the differences
# described inside the instance.
#
# Keys are variable prefixes and values are tuples describing how these
# manifests should be handled:
#
# (flavor, install_prefix, active)
#
# flavor identifies the flavor of this test.
# install_prefix is the path prefix of where to install the files in
# the tests directory.
# active indicates whether to filter out inactive tests from the
# manifest.
#
# We ideally don't filter out inactive tests. However, not every test
# harness can yet deal with test filtering. Once all harnesses can do
# this, this feature can be dropped.
test_manifests = dict(
A11Y=('a11y', 'testing/mochitest', 'a11y', True),
BROWSER_CHROME=('browser-chrome', 'testing/mochitest', 'browser', True),
METRO_CHROME=('metro-chrome', 'testing/mochitest', 'metro', True),
MOCHITEST=('mochitest', 'testing/mochitest', 'tests', True),
MOCHITEST_CHROME=('chrome', 'testing/mochitest', 'chrome', True),
MOCHITEST_WEBAPPRT_CHROME=('webapprt-chrome', 'testing/mochitest', 'webapprtChrome', True),
WEBRTC_SIGNALLING_TEST=('steeplechase', 'steeplechase', '.', True),
XPCSHELL_TESTS=('xpcshell', 'xpcshell', '.', False),
)
for prefix, info in test_manifests.items():
for path in sandbox.get('%s_MANIFESTS' % prefix, []):
for obj in self._process_test_manifest(sandbox, info, path):
yield obj
for flavor in ('crashtest', 'reftest'):
for path in sandbox.get('%s_MANIFESTS' % flavor.upper(), []):
for obj in self._process_reftest_manifest(sandbox, flavor, path):
yield obj
jar_manifests = sandbox.get('JAR_MANIFESTS', [])
if len(jar_manifests) > 1:
raise SandboxValidationError('While JAR_MANIFESTS is a list, '
'it is currently limited to one value.', sandbox)
for path in jar_manifests:
yield JARManifest(sandbox, mozpath.join(sandbox['SRCDIR'], path))
# Temporary test to look for jar.mn files that creep in without using
# the new declaration. Before, we didn't require jar.mn files to
# declared anywhere (they were discovered). This will detect people
# relying on the old behavior.
if os.path.exists(os.path.join(sandbox['SRCDIR'], 'jar.mn')):
if 'jar.mn' not in jar_manifests:
raise SandboxValidationError('A jar.mn exists but it '
'is not referenced in the moz.build file. '
'Please define JAR_MANIFESTS.', sandbox)
for name, jar in sandbox.get('JAVA_JAR_TARGETS', {}).items():
yield SandboxWrapped(sandbox, jar)
for name, data in sandbox.get('ANDROID_ECLIPSE_PROJECT_TARGETS', {}).items():
yield SandboxWrapped(sandbox, data)
if passthru.variables:
yield passthru
def process_gyp_result(
gyp_result,
gyp_dir_attrs,
path,
config,
output,
non_unified_sources,
action_overrides,
):
flat_list, targets, data = gyp_result
no_chromium = gyp_dir_attrs.no_chromium
no_unified = gyp_dir_attrs.no_unified
# Process all targets from the given gyp files and its dependencies.
# The path given to AllTargets needs to use os.sep, while the frontend code
# gives us paths normalized with forward slash separator.
for target in sorted(
gyp.common.AllTargets(flat_list, targets,
path.replace("/", os.sep))):
build_file, target_name, toolset = gyp.common.ParseQualifiedTarget(
target)
# Each target is given its own objdir. The base of that objdir
# is derived from the relative path from the root gyp file path
# to the current build_file, placed under the given output
# directory. Since several targets can be in a given build_file,
# separate them in subdirectories using the build_file basename
# and the target_name.
reldir = mozpath.relpath(mozpath.dirname(build_file),
mozpath.dirname(path))
subdir = "%s_%s" % (
mozpath.splitext(mozpath.basename(build_file))[0],
target_name,
)
# Emit a context for each target.
context = GypContext(
config,
mozpath.relpath(mozpath.join(output, reldir, subdir),
config.topobjdir),
)
context.add_source(mozpath.abspath(build_file))
# The list of included files returned by gyp are relative to build_file
for f in data[build_file]["included_files"]:
context.add_source(
mozpath.abspath(mozpath.join(mozpath.dirname(build_file), f)))
spec = targets[target]
# Derive which gyp configuration to use based on MOZ_DEBUG.
c = "Debug" if config.substs.get("MOZ_DEBUG") else "Release"
if c not in spec["configurations"]:
raise RuntimeError("Missing %s gyp configuration for target %s "
"in %s" % (c, target_name, build_file))
target_conf = spec["configurations"][c]
if "actions" in spec:
handle_actions(spec["actions"], context, action_overrides)
if "copies" in spec:
handle_copies(spec["copies"], context)
use_libs = []
libs = []
def add_deps(s):
for t in s.get("dependencies", []) + s.get("dependencies_original",
[]):
ty = targets[t]["type"]
if ty in ("static_library", "shared_library"):
l = targets[t]["target_name"]
if l not in use_libs:
use_libs.append(l)
# Manually expand out transitive dependencies--
# gyp won't do this for static libs or none targets.
if ty in ("static_library", "none"):
add_deps(targets[t])
libs.extend(spec.get("libraries", []))
# XXX: this sucks, but webrtc breaks with this right now because
# it builds a library called 'gtest' and we just get lucky
# that it isn't in USE_LIBS by that name anywhere.
if no_chromium:
add_deps(spec)
os_libs = []
for l in libs:
if l.startswith("-"):
if l not in os_libs:
os_libs.append(l)
elif l.endswith(".lib"):
l = l[:-4]
if l not in os_libs:
os_libs.append(l)
elif l:
# For library names passed in from moz.build.
l = os.path.basename(l)
if l not in use_libs:
use_libs.append(l)
if spec["type"] == "none":
if not ("actions" in spec or "copies" in spec):
continue
elif spec["type"] in ("static_library", "shared_library",
"executable"):
# Remove leading 'lib' from the target_name if any, and use as
# library name.
name = six.ensure_text(spec["target_name"])
if spec["type"] in ("static_library", "shared_library"):
if name.startswith("lib"):
name = name[3:]
context["LIBRARY_NAME"] = name
else:
context["PROGRAM"] = name
if spec["type"] == "shared_library":
context["FORCE_SHARED_LIB"] = True
elif (spec["type"] == "static_library" and spec.get(
"variables", {}).get("no_expand_libs", "0") == "1"):
# PSM links a NSS static library, but our folded libnss
# doesn't actually export everything that all of the
# objects within would need, so that one library
# should be built as a real static library.
context["NO_EXPAND_LIBS"] = True
if use_libs:
context["USE_LIBS"] = sorted(use_libs, key=lambda s: s.lower())
if os_libs:
context["OS_LIBS"] = os_libs
# gyp files contain headers and asm sources in sources lists.
sources = []
unified_sources = []
extensions = set()
use_defines_in_asflags = False
for f in spec.get("sources", []):
ext = mozpath.splitext(f)[-1]
extensions.add(ext)
if f.startswith("$INTERMEDIATE_DIR/"):
s = ObjDirPath(context, f.replace("$INTERMEDIATE_DIR/",
"!"))
else:
s = SourcePath(context, f)
if ext == ".h":
continue
if ext == ".def":
context["SYMBOLS_FILE"] = s
elif ext != ".S" and not no_unified and s not in non_unified_sources:
unified_sources.append(s)
else:
sources.append(s)
# The Mozilla build system doesn't use DEFINES for building
# ASFILES.
if ext == ".s":
use_defines_in_asflags = True
# The context expects alphabetical order when adding sources
context["SOURCES"] = alphabetical_sorted(sources)
context["UNIFIED_SOURCES"] = alphabetical_sorted(unified_sources)
defines = target_conf.get("defines", [])
if config.substs["CC_TYPE"] == "clang-cl" and no_chromium:
msvs_settings = gyp.msvs_emulation.MsvsSettings(spec, {})
# Hack: MsvsSettings._TargetConfig tries to compare a str to an int,
# so convert manually.
msvs_settings.vs_version.short_name = int(
msvs_settings.vs_version.short_name)
defines.extend(msvs_settings.GetComputedDefines(c))
for define in defines:
if "=" in define:
name, value = define.split("=", 1)
context["DEFINES"][name] = value
else:
context["DEFINES"][define] = True
product_dir_dist = "$PRODUCT_DIR/dist/"
for include in target_conf.get("include_dirs", []):
if include.startswith(product_dir_dist):
# special-case includes of <(PRODUCT_DIR)/dist/ to match
# handle_copies above. This is used for NSS' exports.
include = "!/dist/include/" + include[len(product_dir_dist
):]
elif include.startswith(config.topobjdir):
# NSPR_INCLUDE_DIR gets passed into the NSS build this way.
include = "!/" + mozpath.relpath(include, config.topobjdir)
else:
# moz.build expects all LOCAL_INCLUDES to exist, so ensure they do.
#
# NB: gyp files sometimes have actual absolute paths (e.g.
# /usr/include32) and sometimes paths that moz.build considers
# absolute, i.e. starting from topsrcdir. There's no good way
# to tell them apart here, and the actual absolute paths are
# likely bogus. In any event, actual absolute paths will be
# filtered out by trying to find them in topsrcdir.
#
# We do allow !- and %-prefixed paths, assuming they come
# from moz.build and will be handled the same way as if they
# were given to LOCAL_INCLUDES in moz.build.
if include.startswith("/"):
resolved = mozpath.abspath(
mozpath.join(config.topsrcdir, include[1:]))
elif not include.startswith(("!", "%")):
resolved = mozpath.abspath(
mozpath.join(mozpath.dirname(build_file), include))
if not include.startswith(
("!", "%")) and not os.path.exists(resolved):
continue
context["LOCAL_INCLUDES"] += [include]
context["ASFLAGS"] = target_conf.get("asflags_mozilla", [])
if use_defines_in_asflags and defines:
context["ASFLAGS"] += ["-D" + d for d in defines]
if config.substs["OS_TARGET"] == "SunOS":
context["LDFLAGS"] = target_conf.get("ldflags", [])
flags = target_conf.get("cflags_mozilla", [])
if flags:
suffix_map = {
".c": "CFLAGS",
".cpp": "CXXFLAGS",
".cc": "CXXFLAGS",
".m": "CMFLAGS",
".mm": "CMMFLAGS",
}
variables = (suffix_map[e] for e in extensions
if e in suffix_map)
for var in variables:
for f in flags:
# We may be getting make variable references out of the
# gyp data, and we don't want those in emitted data, so
# substitute them with their actual value.
f = expand_variables(f, config.substs).split()
if not f:
continue
# the result may be a string or a list.
if isinstance(f, six.string_types):
context[var].append(f)
else:
context[var].extend(f)
else:
# Ignore other types because we don't have
# anything using them, and we're not testing them. They can be
# added when that becomes necessary.
raise NotImplementedError("Unsupported gyp target type: %s" %
spec["type"])
if not no_chromium:
# Add some features to all contexts. Put here in case LOCAL_INCLUDES
# order matters.
context["LOCAL_INCLUDES"] += [
"!/ipc/ipdl/_ipdlheaders",
"/ipc/chromium/src",
"/ipc/glue",
]
# These get set via VC project file settings for normal GYP builds.
if config.substs["OS_TARGET"] == "WINNT":
context["DEFINES"]["UNICODE"] = True
context["DEFINES"]["_UNICODE"] = True
context["COMPILE_FLAGS"]["OS_INCLUDES"] = []
for key, value in gyp_dir_attrs.sandbox_vars.items():
if context.get(key) and isinstance(context[key], list):
# If we have a key from sanbox_vars that's also been
# populated here we use the value from sandbox_vars as our
# basis rather than overriding outright.
context[key] = value + context[key]
elif context.get(key) and isinstance(context[key], dict):
context[key].update(value)
else:
context[key] = value
yield context
