def test_defaults(self):
test = OrderedDefaultDict(bool, {'foo': 1 })
self.assertEqual(test['foo'], 1)
self.assertEqual(test['qux'], False)
self.assertEqual(test.keys(), ['foo', 'qux' ])
def test_simple(self):
original = OrderedDict(foo=1, bar=2)
test = OrderedDefaultDict(bool, original)
self.assertEqual(original, test)
self.assertEqual(test['foo'], 1)
self.assertEqual(test.keys(), ['foo', 'bar' ])
class FasterMakeBackend(CommonBackend):
def _init(self):
super(FasterMakeBackend, self)._init()
self._seen_directories = set()
self._defines = dict()
self._manifest_entries = OrderedDefaultDict(set)
self._install_manifests = OrderedDefaultDict(InstallManifest)
self._dependencies = OrderedDefaultDict(list)
self._has_xpidl = False
def _add_preprocess(self, obj, path, dest, target=None, **kwargs):
if target is None:
target = mozpath.basename(path)
# This matches what PP_TARGETS do in config/rules.
if target.endswith('.in'):
target = target[:-3]
depfile = mozpath.join(
self.environment.topobjdir, 'faster', '.deps',
mozpath.join(obj.install_target, dest, target).replace('/', '_'))
self._install_manifests[obj.install_target].add_preprocess(
mozpath.join(obj.srcdir, path),
mozpath.join(dest, target),
depfile,
**kwargs)
def consume_object(self, obj):
if not isinstance(obj, Defines) and isinstance(obj, ContextDerived):
defines = self._defines.get(obj.objdir, {})
if defines:
defines = defines.defines
if isinstance(obj, Defines):
self._defines[obj.objdir] = obj
# We're assuming below that Defines come first for a given objdir,
# which is kind of set in stone from the order things are treated
# in emitter.py.
assert obj.objdir not in self._seen_directories
elif isinstance(obj, JARManifest) and \
obj.install_target.startswith('dist/bin'):
self._consume_jar_manifest(obj, defines)
elif isinstance(obj, (FinalTargetFiles,
FinalTargetPreprocessedFiles)) and \
obj.install_target.startswith('dist/bin'):
for path, files in obj.files.walk():
for f in files:
if isinstance(obj, FinalTargetPreprocessedFiles):
self._add_preprocess(obj, f.full_path, path,
defines=defines)
else:
self._install_manifests[obj.install_target].add_symlink(
f.full_path,
mozpath.join(path, mozpath.basename(f))
)
elif isinstance(obj, ChromeManifestEntry) and \
obj.install_target.startswith('dist/bin'):
top_level = mozpath.join(obj.install_target, 'chrome.manifest')
if obj.path != top_level:
entry = 'manifest %s' % mozpath.relpath(obj.path,
obj.install_target)
self._manifest_entries[top_level].add(entry)
self._manifest_entries[obj.path].add(str(obj.entry))
elif isinstance(obj, XPIDLFile):
self._has_xpidl = True
# XPIDL are emitted before Defines, which breaks the assert in the
# branch for Defines. OTOH, we don't actually care about the
# XPIDLFile objects just yet, so we can just pretend we didn't see
# an object in the directory yet.
return True
else:
# We currently ignore a lot of object types, so just acknowledge
# everything.
return True
self._seen_directories.add(obj.objdir)
return True
def _consume_jar_manifest(self, obj, defines):
# Ideally, this would all be handled somehow in the emitter, but
# this would require all the magic surrounding l10n and addons in
# the recursive make backend to die, which is not going to happen
# any time soon enough.
# Notably missing:
# - DEFINES from config/config.mk
# - L10n support
# - The equivalent of -e when USE_EXTENSION_MANIFEST is set in
# moz.build, but it doesn't matter in dist/bin.
pp = Preprocessor()
pp.context.update(defines)
pp.context.update(self.environment.defines)
pp.context.update(
AB_CD='en-US',
BUILD_FASTER=1,
)
pp.out = JarManifestParser()
pp.do_include(obj.path)
self.backend_input_files |= pp.includes
for jarinfo in pp.out:
install_target = obj.install_target
if jarinfo.base:
install_target = mozpath.normpath(
mozpath.join(install_target, jarinfo.base))
for e in jarinfo.entries:
if e.is_locale:
if jarinfo.relativesrcdir:
path = mozpath.join(self.environment.topsrcdir,
jarinfo.relativesrcdir)
else:
path = mozpath.dirname(obj.path)
src = mozpath.join( path, 'en-US', e.source)
elif e.source.startswith('/'):
src = mozpath.join(self.environment.topsrcdir,
e.source[1:])
else:
src = mozpath.join(mozpath.dirname(obj.path), e.source)
if '*' in e.source:
if e.preprocess:
raise Exception('%s: Wildcards are not supported with '
'preprocessing' % obj.path)
def _prefix(s):
for p in s.split('/'):
if '*' not in p:
yield p + '/'
prefix = ''.join(_prefix(src))
self._install_manifests[install_target] \
.add_pattern_symlink(
prefix,
src[len(prefix):],
mozpath.join(jarinfo.name, e.output))
continue
if not os.path.exists(src):
if e.is_locale:
raise Exception(
'%s: Cannot find %s' % (obj.path, e.source))
if e.source.startswith('/'):
src = mozpath.join(self.environment.topobjdir,
e.source[1:])
else:
# This actually gets awkward if the jar.mn is not
# in the same directory as the moz.build declaring
# it, but it's how it works in the recursive make,
# not that anything relies on that, but it's simpler.
src = mozpath.join(obj.objdir, e.source)
self._dependencies['install-%s' % install_target] \
.append(mozpath.relpath(
src, self.environment.topobjdir))
if e.preprocess:
kwargs = {}
if src.endswith('.css'):
kwargs['marker'] = '%'
self._add_preprocess(
obj,
src,
mozpath.join(jarinfo.name, mozpath.dirname(e.output)),
mozpath.basename(e.output),
defines=defines,
**kwargs)
else:
self._install_manifests[install_target].add_symlink(
src,
mozpath.join(jarinfo.name, e.output))
manifest = mozpath.normpath(mozpath.join(install_target,
jarinfo.name))
manifest += '.manifest'
for m in jarinfo.chrome_manifests:
self._manifest_entries[manifest].add(
m.replace('%', mozpath.basename(jarinfo.name) + '/'))
if jarinfo.name != 'chrome':
manifest = mozpath.normpath(mozpath.join(install_target,
'chrome.manifest'))
entry = 'manifest %s.manifest' % jarinfo.name
self._manifest_entries[manifest].add(entry)
def consume_finished(self):
mk = Makefile()
# Add the default rule at the very beginning.
mk.create_rule(['default'])
mk.add_statement('TOPSRCDIR = %s' % self.environment.topsrcdir)
mk.add_statement('TOPOBJDIR = %s' % self.environment.topobjdir)
mk.add_statement('BACKEND = %s' % self._backend_output_list_file)
if not self._has_xpidl:
mk.add_statement('NO_XPIDL = 1')
# Add a few necessary variables inherited from configure
for var in (
'PYTHON',
'ACDEFINES',
'MOZ_BUILD_APP',
'MOZ_WIDGET_TOOLKIT',
):
mk.add_statement('%s = %s' % (var, self.environment.substs[var]))
install_manifests_bases = self._install_manifests.keys()
# Add information for chrome manifest generation
manifest_targets = []
for target, entries in self._manifest_entries.iteritems():
manifest_targets.append(target)
install_target = mozpath.basedir(target, install_manifests_bases)
self._install_manifests[install_target].add_content(
''.join('%s\n' % e for e in sorted(entries)),
mozpath.relpath(target, install_target))
# Add information for install manifests.
mk.add_statement('INSTALL_MANIFESTS = %s'
% ' '.join(self._install_manifests.keys()))
# Add dependencies we infered:
for target, deps in self._dependencies.iteritems():
mk.create_rule([target]).add_dependencies(
'$(TOPOBJDIR)/%s' % d for d in deps)
# Add backend dependencies:
mk.create_rule([self._backend_output_list_file]).add_dependencies(
self.backend_input_files)
mk.add_statement('include $(TOPSRCDIR)/config/faster/rules.mk')
for base, install_manifest in self._install_manifests.iteritems():
with self._write_file(
mozpath.join(self.environment.topobjdir, 'faster',
'install_%s' % base.replace('/', '_'))) as fh:
install_manifest.write(fileobj=fh)
with self._write_file(
mozpath.join(self.environment.topobjdir, 'faster',
'Makefile')) as fh:
mk.dump(fh, removal_guard=False)
class TestMetadata(object):
"""Holds information about tests.
This class provides an API to query tests active in the build
configuration.
"""
def __init__(self, all_tests, srcdir, test_defaults=None):
self._tests_by_path = OrderedDefaultDict(list)
self._tests_by_flavor = defaultdict(set)
self._test_dirs = set()
self._objdir = os.path.abspath(os.path.join(all_tests, os.pardir))
self._wpt_loaded = False
self._srcdir = srcdir
with open(all_tests, 'rb') as fh:
test_data = pickle.load(fh)
defaults = None
if test_defaults:
with open(test_defaults, 'rb') as fh:
defaults = pickle.load(fh)
for path, tests in test_data.items():
for metadata in tests:
if defaults:
defaults_manifests = [metadata['manifest']]
ancestor_manifest = metadata.get('ancestor-manifest')
if ancestor_manifest:
# The (ancestor manifest, included manifest) tuple
# contains the defaults of the included manifest, so
# use it instead of [metadata['manifest']].
defaults_manifests[0] = (ancestor_manifest, metadata['manifest'])
defaults_manifests.append(ancestor_manifest)
for manifest in defaults_manifests:
manifest_defaults = defaults.get(manifest)
if manifest_defaults:
metadata = manifestparser.combine_fields(manifest_defaults,
metadata)
self._tests_by_path[path].append(metadata)
self._test_dirs.add(os.path.dirname(path))
flavor = metadata.get('flavor')
self._tests_by_flavor[flavor].add(path)
def tests_with_flavor(self, flavor):
"""Obtain all tests having the specified flavor.
This is a generator of dicts describing each test.
"""
for path in sorted(self._tests_by_flavor.get(flavor, [])):
yield self._tests_by_path[path]
def resolve_tests(self, paths=None, flavor=None, subsuite=None, under_path=None,
tags=None):
"""Resolve tests from an identifier.
This is a generator of dicts describing each test.
``paths`` can be an iterable of values to use to identify tests to run.
If an entry is a known test file, tests associated with that file are
returned (there may be multiple configurations for a single file). If
an entry is a directory, or a prefix of a directory containing tests,
all tests in that directory are returned. If the string appears in a
known test file, that test file is considered. If the path contains
a wildcard pattern, tests matching that pattern are returned.
If ``under_path`` is a string, it will be used to filter out tests that
aren't in the specified path prefix relative to topsrcdir or the
test's installed dir.
If ``flavor`` is a string, it will be used to filter returned tests
to only be the flavor specified. A flavor is something like
``xpcshell``.
If ``subsuite`` is a string, it will be used to filter returned tests
to only be in the subsuite specified.
If ``tags`` are specified, they will be used to filter returned tests
to only those with a matching tag.
"""
if tags:
tags = set(tags)
def fltr(tests):
for test in tests:
if flavor:
if flavor == 'devtools' and test.get('flavor') != 'browser-chrome':
continue
if flavor != 'devtools' and test.get('flavor') != flavor:
continue
if subsuite and test.get('subsuite') != subsuite:
continue
if tags and not (tags & set(test.get('tags', '').split())):
continue
if under_path and not test['file_relpath'].startswith(under_path):
continue
# Make a copy so modifications don't change the source.
yield dict(test)
paths = paths or []
paths = [mozpath.normpath(p) for p in paths]
if not paths:
paths = [None]
candidate_paths = set()
if flavor in (None, 'web-platform-tests') and any(self.is_wpt_path(p) for p in paths):
self.add_wpt_manifest_data()
for path in sorted(paths):
if path is None:
candidate_paths |= set(self._tests_by_path.keys())
continue
if '*' in path:
candidate_paths |= {p for p in self._tests_by_path
if mozpath.match(p, path)}
continue
# If the path is a directory, or the path is a prefix of a directory
# containing tests, pull in all tests in that directory.
if (path in self._test_dirs or
any(p.startswith(path) for p in self._tests_by_path)):
candidate_paths |= {p for p in self._tests_by_path
if p.startswith(path)}
continue
# If it's a test file, add just that file.
candidate_paths |= {p for p in self._tests_by_path if path in p}
for p in sorted(candidate_paths):
tests = self._tests_by_path[p]
for test in fltr(tests):
yield test
def is_wpt_path(self, path):
if path is None:
return True
if mozpath.match(path, "testing/web-platform/tests/**"):
return True
if mozpath.match(path, "testing/web-platform/mozilla/tests/**"):
return True
return False
def add_wpt_manifest_data(self):
if self._wpt_loaded:
return
wpt_path = os.path.join(self._srcdir, "testing", "web-platform")
sys.path = [wpt_path] + sys.path
import manifestupdate
# Set up a logger that will drop all the output
import logging
logger = logging.getLogger("manifestupdate")
logger.propogate = False
manifests = manifestupdate.run(self._srcdir, self._objdir, rebuild=False, download=True,
config_path=None, rewrite_config=True, update=True,
logger=logger)
if not manifests:
print("Loading wpt manifest failed")
return
for manifest, data in manifests.iteritems():
tests_root = data["tests_path"]
for test_type, path, tests in manifest:
full_path = os.path.join(tests_root, path)
src_path = os.path.relpath(full_path, self._srcdir)
if test_type not in ["testharness", "reftest", "wdspec"]:
continue
for test in tests:
self._tests_by_path[src_path].append({
"path": full_path,
"flavor": "web-platform-tests",
"here": os.path.dirname(path),
"manifest": data["manifest_path"],
"name": test.id,
"file_relpath": path,
"head": "",
"support-files": "",
"subsuite": test_type,
"dir_relpath": os.path.dirname(src_path),
"srcdir_relpath": src_path,
})
self._wpt_loaded = True
class TestMetadata(object):
"""Holds information about tests.
This class provides an API to query tests active in the build
configuration.
"""
def __init__(self, all_tests, test_defaults=None):
self._tests_by_path = OrderedDefaultDict(list)
self._tests_by_flavor = defaultdict(set)
self._test_dirs = set()
with open(all_tests, 'rb') as fh:
test_data = pickle.load(fh)
defaults = None
if test_defaults:
with open(test_defaults, 'rb') as fh:
defaults = pickle.load(fh)
for path, tests in test_data.items():
for metadata in tests:
if defaults:
defaults_manifests = [metadata['manifest']]
ancestor_manifest = metadata.get('ancestor-manifest')
if ancestor_manifest:
defaults_manifests.append(ancestor_manifest)
for manifest in defaults_manifests:
manifest_defaults = defaults.get(manifest)
if manifest_defaults:
metadata = manifestparser.combine_fields(
manifest_defaults, metadata)
self._tests_by_path[path].append(metadata)
self._test_dirs.add(os.path.dirname(path))
flavor = metadata.get('flavor')
self._tests_by_flavor[flavor].add(path)
def tests_with_flavor(self, flavor):
"""Obtain all tests having the specified flavor.
This is a generator of dicts describing each test.
"""
for path in sorted(self._tests_by_flavor.get(flavor, [])):
yield self._tests_by_path[path]
def resolve_tests(self,
paths=None,
flavor=None,
subsuite=None,
under_path=None,
tags=None):
"""Resolve tests from an identifier.
This is a generator of dicts describing each test.
``paths`` can be an iterable of values to use to identify tests to run.
If an entry is a known test file, tests associated with that file are
returned (there may be multiple configurations for a single file). If
an entry is a directory, or a prefix of a directory containing tests,
all tests in that directory are returned. If the string appears in a
known test file, that test file is considered. If the path contains
a wildcard pattern, tests matching that pattern are returned.
If ``under_path`` is a string, it will be used to filter out tests that
aren't in the specified path prefix relative to topsrcdir or the
test's installed dir.
If ``flavor`` is a string, it will be used to filter returned tests
to only be the flavor specified. A flavor is something like
``xpcshell``.
If ``subsuite`` is a string, it will be used to filter returned tests
to only be in the subsuite specified.
If ``tags`` are specified, they will be used to filter returned tests
to only those with a matching tag.
"""
if tags:
tags = set(tags)
def fltr(tests):
for test in tests:
if flavor:
if flavor == 'devtools' and test.get(
'flavor') != 'browser-chrome':
continue
if flavor != 'devtools' and test.get('flavor') != flavor:
continue
if subsuite and test.get('subsuite') != subsuite:
continue
if tags and not (tags & set(test.get('tags', '').split())):
continue
if under_path and not test['file_relpath'].startswith(
under_path):
continue
# Make a copy so modifications don't change the source.
yield dict(test)
paths = paths or []
paths = [mozpath.normpath(p) for p in paths]
if not paths:
paths = [None]
candidate_paths = set()
for path in sorted(paths):
if path is None:
candidate_paths |= set(self._tests_by_path.keys())
continue
if '*' in path:
candidate_paths |= {
p
for p in self._tests_by_path if mozpath.match(p, path)
}
continue
# If the path is a directory, or the path is a prefix of a directory
# containing tests, pull in all tests in that directory.
if (path in self._test_dirs
or any(p.startswith(path) for p in self._tests_by_path)):
candidate_paths |= {
p
for p in self._tests_by_path if p.startswith(path)
}
continue
# If it's a test file, add just that file.
candidate_paths |= {p for p in self._tests_by_path if path in p}
for p in sorted(candidate_paths):
tests = self._tests_by_path[p]
for test in fltr(tests):
yield test
class FasterMakeBackend(CommonBackend):
def _init(self):
super(FasterMakeBackend, self)._init()
self._seen_directories = set()
self._defines = dict()
self._jar_manifests = OrderedDict()
self._manifest_entries = OrderedDefaultDict(list)
self._install_manifests = OrderedDefaultDict(InstallManifest)
def _add_preprocess(self, obj, path, dest, **kwargs):
target = mozpath.basename(path)
# This matches what PP_TARGETS do in config/rules.
if target.endswith('.in'):
target = target[:-3]
depfile = mozpath.join(
self.environment.topobjdir, 'faster', '.deps',
mozpath.join(obj.install_target, dest, target).replace('/', '_'))
self._install_manifests[obj.install_target].add_preprocess(
mozpath.join(obj.srcdir, path),
mozpath.join(dest, target),
depfile,
**kwargs)
def consume_object(self, obj):
if not isinstance(obj, Defines) and isinstance(obj, ContextDerived):
defines = self._defines.get(obj.objdir, {})
if defines:
defines = defines.defines
if isinstance(obj, Defines):
self._defines[obj.objdir] = obj
# We're assuming below that Defines come first for a given objdir,
# which is kind of set in stone from the order things are treated
# in emitter.py.
assert obj.objdir not in self._seen_directories
elif isinstance(obj, JARManifest) and \
obj.install_target.startswith('dist/bin'):
defines = self._defines.get(obj.objdir, [])
if defines:
defines = list(defines.get_defines())
self._jar_manifests[obj.path] = (obj.objdir,
obj.install_target,
defines)
elif isinstance(obj, VariablePassthru) and \
obj.install_target.startswith('dist/bin'):
for f in obj.variables.get('EXTRA_COMPONENTS', {}):
path = mozpath.join(obj.install_target, 'components',
mozpath.basename(f))
self._install_manifests[obj.install_target].add_symlink(
mozpath.join(obj.srcdir, f),
mozpath.join('components', mozpath.basename(f))
)
if f.endswith('.manifest'):
manifest = mozpath.join(obj.install_target,
'chrome.manifest')
self._manifest_entries[manifest].append(
'manifest components/%s' % mozpath.basename(f))
for f in obj.variables.get('EXTRA_PP_COMPONENTS', {}):
self._add_preprocess(obj, f, 'components', defines=defines)
if f.endswith('.manifest'):
manifest = mozpath.join(obj.install_target,
'chrome.manifest')
self._manifest_entries[manifest].append(
'manifest components/%s' % mozpath.basename(f))
elif isinstance(obj, JavaScriptModules) and \
obj.install_target.startswith('dist/bin'):
for path, strings in obj.modules.walk():
base = mozpath.join('modules', path)
for f in strings:
if obj.flavor == 'extra':
self._install_manifests[obj.install_target].add_symlink(
mozpath.join(obj.srcdir, f),
mozpath.join(base, mozpath.basename(f))
)
elif obj.flavor == 'extra_pp':
self._add_preprocess(obj, f, base, defines=defines)
elif isinstance(obj, JsPreferenceFile) and \
obj.install_target.startswith('dist/bin'):
# The condition for the directory value in config/rules.mk is:
# ifneq (,$(DIST_SUBDIR)$(XPI_NAME)$(LIBXUL_SDK))
# - LIBXUL_SDK is not supported (it likely doesn't work in the
# recursive backend anyways
# - when XPI_NAME is set, obj.install_target will start with
# dist/xpi-stage
# - when DIST_SUBDIR is set, obj.install_target will start with
# dist/bin/$(DIST_SUBDIR)
# So an equivalent condition that is not cumbersome for us and that
# is enough at least for now is checking if obj.install_target is
# different from dist/bin.
if obj.install_target == 'dist/bin':
pref_dir = 'defaults/pref'
else:
pref_dir = 'defaults/preferences'
dest = mozpath.join(obj.install_target, pref_dir,
mozpath.basename(obj.path))
# We preprocess these, but they don't necessarily have preprocessor
# directives, so tell the preprocessor to not complain about that.
self._add_preprocess(obj, obj.path, pref_dir, defines=defines,
silence_missing_directive_warnings=True)
elif isinstance(obj, Resources) and \
obj.install_target.startswith('dist/bin'):
for path, strings in obj.resources.walk():
base = mozpath.join('res', path)
for f in strings:
flags = strings.flags_for(f)
if flags and flags.preprocess:
self._add_preprocess(obj, f, base, marker='%',
defines=obj.defines)
else:
self._install_manifests[obj.install_target].add_symlink(
mozpath.join(obj.srcdir, f),
mozpath.join(base, mozpath.basename(f))
)
elif isinstance(obj, FinalTargetFiles) and \
obj.install_target.startswith('dist/bin'):
for path, strings in obj.files.walk():
base = mozpath.join(obj.install_target, path)
for f in strings:
self._install_manifests[obj.install_target].add_symlink(
mozpath.join(obj.srcdir, f),
mozpath.join(path, mozpath.basename(f))
)
elif isinstance(obj, DistFiles) and \
obj.install_target.startswith('dist/bin'):
# We preprocess these, but they don't necessarily have preprocessor
# directives, so tell the preprocessor to not complain about that.
for f in obj.files:
self._add_preprocess(obj, f, '', defines=defines,
silence_missing_directive_warnings=True)
else:
# We currently ignore a lot of object types, so just acknowledge
# everything.
return True
self._seen_directories.add(obj.objdir)
return True
def consume_finished(self):
mk = Makefile()
# Add the default rule at the very beginning.
mk.create_rule(['default'])
mk.add_statement('TOPSRCDIR = %s' % self.environment.topsrcdir)
mk.add_statement('TOPOBJDIR = %s' % self.environment.topobjdir)
# Add a few necessary variables inherited from configure
for var in (
'PYTHON',
'ACDEFINES',
'MOZ_CHROME_FILE_FORMAT',
):
mk.add_statement('%s = %s' % (var, self.environment.substs[var]))
# Add all necessary information for jar manifest processing
jar_mn_targets = []
for path, (objdir, install_target, defines) in \
self._jar_manifests.iteritems():
rel_manifest = mozpath.relpath(path, self.environment.topsrcdir)
target = rel_manifest.replace('/', '-')
assert target not in jar_mn_targets
jar_mn_targets.append(target)
target = 'jar-%s' % target
mk.create_rule([target]).add_dependencies([path])
if objdir != mozpath.join(self.environment.topobjdir,
mozpath.dirname(rel_manifest)):
mk.create_rule([target]).add_dependencies(
['objdir = %s' % objdir])
if install_target != 'dist/bin':
mk.create_rule([target]).add_dependencies(
['install_target = %s' % install_target])
if defines:
mk.create_rule([target]).add_dependencies(
['defines = %s' % ' '.join(defines)])
mk.add_statement('JAR_MN_TARGETS = %s' % ' '.join(jar_mn_targets))
# Add information for chrome manifest generation
manifest_targets = []
for target, entries in self._manifest_entries.iteritems():
manifest_targets.append(target)
target = '$(TOPOBJDIR)/%s' % target
mk.create_rule([target]).add_dependencies(
['content = %s' % ' '.join('"%s"' % e for e in entries)])
mk.add_statement('MANIFEST_TARGETS = %s' % ' '.join(manifest_targets))
# Add information for install manifests.
mk.add_statement('INSTALL_MANIFESTS = %s'
% ' '.join(self._install_manifests.keys()))
mk.add_statement('include $(TOPSRCDIR)/config/faster/rules.mk')
for base, install_manifest in self._install_manifests.iteritems():
with self._write_file(
mozpath.join(self.environment.topobjdir, 'faster',
'install_%s' % base.replace('/', '_'))) as fh:
install_manifest.write(fileobj=fh)
with self._write_file(
mozpath.join(self.environment.topobjdir, 'faster',
'Makefile')) as fh:
mk.dump(fh, removal_guard=False)
class FasterMakeBackend(CommonBackend):
def _init(self):
super(FasterMakeBackend, self)._init()
self._seen_directories = set()
self._defines = dict()
self._manifest_entries = OrderedDefaultDict(list)
self._install_manifests = OrderedDefaultDict(OverwriteInstallManifest)
self._dependencies = OrderedDefaultDict(list)
def _add_preprocess(self, obj, path, dest, target=None, **kwargs):
if target is None:
target = mozpath.basename(path)
# This matches what PP_TARGETS do in config/rules.
if target.endswith('.in'):
target = target[:-3]
depfile = mozpath.join(
self.environment.topobjdir, 'faster', '.deps',
mozpath.join(obj.install_target, dest, target).replace('/', '_'))
self._install_manifests[obj.install_target].add_preprocess(
mozpath.join(obj.srcdir, path),
mozpath.join(dest, target),
depfile,
**kwargs)
def consume_object(self, obj):
if not isinstance(obj, Defines) and isinstance(obj, ContextDerived):
defines = self._defines.get(obj.objdir, {})
if defines:
defines = defines.defines
if isinstance(obj, Defines):
self._defines[obj.objdir] = obj
# We're assuming below that Defines come first for a given objdir,
# which is kind of set in stone from the order things are treated
# in emitter.py.
assert obj.objdir not in self._seen_directories
elif isinstance(obj, JARManifest) and \
obj.install_target.startswith('dist/bin'):
self._consume_jar_manifest(obj, defines)
elif isinstance(obj, VariablePassthru) and \
obj.install_target.startswith('dist/bin'):
for f in obj.variables.get('EXTRA_COMPONENTS', {}):
path = mozpath.join(obj.install_target, 'components',
mozpath.basename(f))
self._install_manifests[obj.install_target].add_symlink(
mozpath.join(obj.srcdir, f),
mozpath.join('components', mozpath.basename(f))
)
if f.endswith('.manifest'):
manifest = mozpath.join(obj.install_target,
'chrome.manifest')
self._manifest_entries[manifest].append(
'manifest components/%s' % mozpath.basename(f))
for f in obj.variables.get('EXTRA_PP_COMPONENTS', {}):
self._add_preprocess(obj, f, 'components', defines=defines)
if f.endswith('.manifest'):
manifest = mozpath.join(obj.install_target,
'chrome.manifest')
self._manifest_entries[manifest].append(
'manifest components/%s' % mozpath.basename(f))
elif isinstance(obj, JavaScriptModules) and \
obj.install_target.startswith('dist/bin'):
for path, strings in obj.modules.walk():
base = mozpath.join('modules', path)
for f in strings:
if obj.flavor == 'extra':
self._install_manifests[obj.install_target].add_symlink(
mozpath.join(obj.srcdir, f),
mozpath.join(base, mozpath.basename(f))
)
elif obj.flavor == 'extra_pp':
self._add_preprocess(obj, f, base, defines=defines)
elif isinstance(obj, JsPreferenceFile) and \
obj.install_target.startswith('dist/bin'):
# The condition for the directory value in config/rules.mk is:
# ifneq (,$(DIST_SUBDIR)$(XPI_NAME))
# - when XPI_NAME is set, obj.install_target will start with
# dist/xpi-stage
# - when DIST_SUBDIR is set, obj.install_target will start with
# dist/bin/$(DIST_SUBDIR)
# So an equivalent condition that is not cumbersome for us and that
# is enough at least for now is checking if obj.install_target is
# different from dist/bin.
if obj.install_target == 'dist/bin':
pref_dir = 'defaults/pref'
else:
pref_dir = 'defaults/preferences'
dest = mozpath.join(obj.install_target, pref_dir,
mozpath.basename(obj.path))
# We preprocess these, but they don't necessarily have preprocessor
# directives, so tell the preprocessor to not complain about that.
self._add_preprocess(obj, obj.path, pref_dir, defines=defines,
silence_missing_directive_warnings=True)
elif isinstance(obj, Resources) and \
obj.install_target.startswith('dist/bin'):
for path, strings in obj.resources.walk():
base = mozpath.join('res', path)
for f in strings:
flags = strings.flags_for(f)
if flags and flags.preprocess:
self._add_preprocess(obj, f, base, marker='%',
defines=obj.defines)
else:
self._install_manifests[obj.install_target].add_symlink(
mozpath.join(obj.srcdir, f),
mozpath.join(base, mozpath.basename(f))
)
elif isinstance(obj, FinalTargetFiles) and \
obj.install_target.startswith('dist/bin'):
for path, strings in obj.files.walk():
base = mozpath.join(obj.install_target, path)
for f in strings:
self._install_manifests[obj.install_target].add_symlink(
mozpath.join(obj.srcdir, f),
mozpath.join(path, mozpath.basename(f))
)
elif isinstance(obj, DistFiles) and \
obj.install_target.startswith('dist/bin'):
# We preprocess these, but they don't necessarily have preprocessor
# directives, so tell the preprocessor to not complain about that.
for f in obj.files:
self._add_preprocess(obj, f, '', defines=defines,
silence_missing_directive_warnings=True)
else:
# We currently ignore a lot of object types, so just acknowledge
# everything.
return True
self._seen_directories.add(obj.objdir)
return True
def _consume_jar_manifest(self, obj, defines):
# Ideally, this would all be handled somehow in the emitter, but
# this would require all the magic surrounding l10n and addons in
# the recursive make backend to die, which is not going to happen
# any time soon enough.
# Notably missing:
# - DEFINES from config/config.mk
# - L10n support
# - The equivalent of -e when USE_EXTENSION_MANIFEST is set in
# moz.build, but it doesn't matter in dist/bin.
pp = Preprocessor()
pp.context.update(defines)
pp.context.update(self.environment.defines)
pp.context.update(
AB_CD='en-US',
BUILD_FASTER=1,
)
pp.out = JarManifestParser()
pp.do_include(obj.path)
for jarinfo in pp.out:
install_target = obj.install_target
# Bug 1150417 added some gross hacks, which we don't try to
# support generically. Fortunately, the hacks don't define more
# than chrome manifest entries, so just assume we don't get
# any installation entries.
if jarinfo.name.startswith('../'):
assert not jarinfo.entries
base = mozpath.join('chrome', jarinfo.name)
for e in jarinfo.entries:
if e.is_locale:
src = mozpath.join(
jarinfo.relativesrcdir or mozpath.dirname(obj.path),
'en-US',
e.source)
elif e.source.startswith('/'):
src = mozpath.join(self.environment.topsrcdir,
e.source[1:])
else:
src = mozpath.join(mozpath.dirname(obj.path), e.source)
if '*' in e.source:
if e.preprocess:
raise Exception('%s: Wildcards are not supported with '
'preprocessing' % obj.path)
def _prefix(s):
for p in s.split('/'):
if '*' not in p:
yield p + '/'
prefix = ''.join(_prefix(src))
self._install_manifests[obj.install_target] \
.add_pattern_symlink(
prefix,
src[len(prefix):],
mozpath.join(base, e.output))
continue
if not os.path.exists(src):
if e.is_locale:
raise Exception(
'%s: Cannot find %s' % (obj.path, e.source))
if e.source.startswith('/'):
src = mozpath.join(self.environment.topobjdir,
e.source[1:])
else:
# This actually gets awkward if the jar.mn is not
# in the same directory as the moz.build declaring
# it, but it's how it works in the recursive make,
# not that anything relies on that, but it's simpler.
src = mozpath.join(obj.objdir, e.source)
self._dependencies['install-%s' % obj.install_target] \
.append(mozpath.relpath(
src, self.environment.topobjdir))
if e.preprocess:
kwargs = {}
if src.endswith('.css'):
kwargs['marker'] = '%'
self._add_preprocess(
obj,
src,
mozpath.join(base, mozpath.dirname(e.output)),
mozpath.basename(e.output),
defines=defines,
**kwargs)
else:
self._install_manifests[obj.install_target].add_symlink(
src,
mozpath.join(base, e.output))
manifest = mozpath.normpath(mozpath.join(obj.install_target, base))
manifest += '.manifest'
for m in jarinfo.chrome_manifests:
self._manifest_entries[manifest].append(
m.replace('%', jarinfo.name + '/'))
# ../ special cased for bug 1150417 again.
if not jarinfo.name.startswith('../'):
manifest = mozpath.normpath(mozpath.join(obj.install_target,
'chrome.manifest'))
entry = 'manifest %s.manifest' % base
if entry not in self._manifest_entries[manifest]:
self._manifest_entries[manifest].append(entry)
def consume_finished(self):
mk = Makefile()
# Add the default rule at the very beginning.
mk.create_rule(['default'])
mk.add_statement('TOPSRCDIR = %s' % self.environment.topsrcdir)
mk.add_statement('TOPOBJDIR = %s' % self.environment.topobjdir)
# Add a few necessary variables inherited from configure
for var in (
'PYTHON',
'ACDEFINES',
'MOZ_BUILD_APP',
'MOZ_WIDGET_TOOLKIT',
):
mk.add_statement('%s = %s' % (var, self.environment.substs[var]))
# Add information for chrome manifest generation
manifest_targets = []
for target, entries in self._manifest_entries.iteritems():
manifest_targets.append(target)
target = '$(TOPOBJDIR)/%s' % target
mk.create_rule([target]).add_dependencies(
['content = %s' % ' '.join('"%s"' % e for e in entries)])
mk.add_statement('MANIFEST_TARGETS = %s' % ' '.join(manifest_targets))
# Add information for install manifests.
mk.add_statement('INSTALL_MANIFESTS = %s'
% ' '.join(self._install_manifests.keys()))
# Add dependencies we infered:
for target, deps in self._dependencies.iteritems():
mk.create_rule([target]).add_dependencies(
'$(TOPOBJDIR)/%s' % d for d in deps)
mk.add_statement('include $(TOPSRCDIR)/config/faster/rules.mk')
for base, install_manifest in self._install_manifests.iteritems():
with self._write_file(
mozpath.join(self.environment.topobjdir, 'faster',
'install_%s' % base.replace('/', '_'))) as fh:
install_manifest.write(fileobj=fh)
with self._write_file(
mozpath.join(self.environment.topobjdir, 'faster',
'Makefile')) as fh:
mk.dump(fh, removal_guard=False)
class FasterMakeBackend(CommonBackend, PartialBackend):
def _init(self):
super(FasterMakeBackend, self)._init()
self._manifest_entries = OrderedDefaultDict(set)
self._install_manifests = OrderedDefaultDict(InstallManifest)
self._dependencies = OrderedDefaultDict(list)
self._has_xpidl = False
def _add_preprocess(self, obj, path, dest, target=None, **kwargs):
if target is None:
target = mozpath.basename(path)
# This matches what PP_TARGETS do in config/rules.
if target.endswith('.in'):
target = target[:-3]
if target.endswith('.css'):
kwargs['marker'] = '%'
depfile = mozpath.join(
self.environment.topobjdir, 'faster', '.deps',
mozpath.join(obj.install_target, dest, target).replace('/', '_'))
self._install_manifests[obj.install_target].add_preprocess(
mozpath.join(obj.srcdir, path), mozpath.join(dest, target),
depfile, **kwargs)
def consume_object(self, obj):
if isinstance(obj, JARManifest) and \
obj.install_target.startswith('dist/bin'):
self._consume_jar_manifest(obj)
elif isinstance(obj, (FinalTargetFiles,
FinalTargetPreprocessedFiles)) and \
obj.install_target.startswith('dist/bin'):
defines = obj.defines or {}
if defines:
defines = defines.defines
for path, files in obj.files.walk():
for f in files:
if isinstance(obj, FinalTargetPreprocessedFiles):
self._add_preprocess(obj,
f.full_path,
path,
target=f.target_basename,
defines=defines)
elif '*' in f:
def _prefix(s):
for p in mozpath.split(s):
if '*' not in p:
yield p + '/'
prefix = ''.join(_prefix(f.full_path))
self._install_manifests[obj.install_target] \
.add_pattern_link(
prefix,
f.full_path[len(prefix):],
mozpath.join(path, f.target_basename))
else:
self._install_manifests[obj.install_target].add_link(
f.full_path, mozpath.join(path, f.target_basename))
if isinstance(f, ObjDirPath):
dep_target = 'install-%s' % obj.install_target
self._dependencies[dep_target].append(
mozpath.relpath(f.full_path,
self.environment.topobjdir))
elif isinstance(obj, ChromeManifestEntry) and \
obj.install_target.startswith('dist/bin'):
top_level = mozpath.join(obj.install_target, 'chrome.manifest')
if obj.path != top_level:
entry = 'manifest %s' % mozpath.relpath(
obj.path, obj.install_target)
self._manifest_entries[top_level].add(entry)
self._manifest_entries[obj.path].add(str(obj.entry))
elif isinstance(obj, XPIDLFile):
self._has_xpidl = True
# We're not actually handling XPIDL files.
return False
else:
return False
return True
def consume_finished(self):
mk = Makefile()
# Add the default rule at the very beginning.
mk.create_rule(['default'])
mk.add_statement('TOPSRCDIR = %s' % self.environment.topsrcdir)
mk.add_statement('TOPOBJDIR = %s' % self.environment.topobjdir)
if not self._has_xpidl:
mk.add_statement('NO_XPIDL = 1')
# Add a few necessary variables inherited from configure
for var in (
'PYTHON',
'ACDEFINES',
'MOZ_BUILD_APP',
'MOZ_WIDGET_TOOLKIT',
):
value = self.environment.substs.get(var)
if value is not None:
mk.add_statement('%s = %s' % (var, value))
install_manifests_bases = self._install_manifests.keys()
# Add information for chrome manifest generation
manifest_targets = []
for target, entries in self._manifest_entries.iteritems():
manifest_targets.append(target)
install_target = mozpath.basedir(target, install_manifests_bases)
self._install_manifests[install_target].add_content(
''.join('%s\n' % e for e in sorted(entries)),
mozpath.relpath(target, install_target))
# Add information for install manifests.
mk.add_statement('INSTALL_MANIFESTS = %s' %
' '.join(self._install_manifests.keys()))
# Add dependencies we infered:
for target, deps in self._dependencies.iteritems():
mk.create_rule([target]).add_dependencies('$(TOPOBJDIR)/%s' % d
for d in deps)
mk.add_statement('include $(TOPSRCDIR)/config/faster/rules.mk')
for base, install_manifest in self._install_manifests.iteritems():
with self._write_file(
mozpath.join(self.environment.topobjdir, 'faster',
'install_%s' % base.replace('/', '_'))) as fh:
install_manifest.write(fileobj=fh)
# For artifact builds only, write a single unified manifest for consumption by |mach watch|.
if self.environment.is_artifact_build:
unified_manifest = InstallManifest()
for base, install_manifest in self._install_manifests.iteritems():
# Expect 'dist/bin/**', which includes 'dist/bin' with no trailing slash.
assert base.startswith('dist/bin')
base = base[len('dist/bin'):]
if base and base[0] == '/':
base = base[1:]
unified_manifest.add_entries_from(install_manifest, base=base)
with self._write_file(
mozpath.join(self.environment.topobjdir, 'faster',
'unified_install_dist_bin')) as fh:
unified_manifest.write(fileobj=fh)
with self._write_file(
mozpath.join(self.environment.topobjdir, 'faster',
'Makefile')) as fh:
mk.dump(fh, removal_guard=False)
class FasterMakeBackend(MakeBackend, PartialBackend):
def _init(self):
super(FasterMakeBackend, self)._init()
self._manifest_entries = OrderedDefaultDict(set)
self._install_manifests = OrderedDefaultDict(InstallManifest)
self._dependencies = OrderedDefaultDict(list)
self._l10n_dependencies = OrderedDefaultDict(list)
self._has_xpidl = False
self._generated_files_map = {}
self._generated_files = []
def _add_preprocess(self, obj, path, dest, target=None, **kwargs):
if target is None:
target = mozpath.basename(path)
# This matches what PP_TARGETS do in config/rules.
if target.endswith('.in'):
target = target[:-3]
if target.endswith('.css'):
kwargs['marker'] = '%'
depfile = mozpath.join(
self.environment.topobjdir, 'faster', '.deps',
mozpath.join(obj.install_target, dest, target).replace('/', '_'))
self._install_manifests[obj.install_target].add_preprocess(
mozpath.join(obj.srcdir, path), mozpath.join(dest, target),
depfile, **kwargs)
def consume_object(self, obj):
if isinstance(obj, JARManifest) and \
obj.install_target.startswith('dist/bin'):
self._consume_jar_manifest(obj)
elif isinstance(obj, (FinalTargetFiles,
FinalTargetPreprocessedFiles)) and \
obj.install_target.startswith('dist/bin'):
ab_cd = self.environment.substs['MOZ_UI_LOCALE'][0]
localized = isinstance(
obj, (LocalizedFiles, LocalizedPreprocessedFiles))
defines = obj.defines or {}
if defines:
defines = defines.defines
for path, files in obj.files.walk():
for f in files:
# For localized files we need to find the file from the locale directory.
if (localized and not isinstance(f, ObjDirPath)
and ab_cd != 'en-US'):
src = self.localized_path(obj.relsrcdir, f)
dep_target = 'install-%s' % obj.install_target
if '*' not in src:
merge = mozpath.abspath(
mozpath.join(self.environment.topobjdir,
'l10n_merge', obj.relsrcdir, f))
self._l10n_dependencies[dep_target].append(
(merge, f.full_path, src))
src = merge
else:
src = f.full_path
if isinstance(obj, FinalTargetPreprocessedFiles):
self._add_preprocess(obj,
src,
path,
target=f.target_basename,
defines=defines)
elif '*' in f:
def _prefix(s):
for p in mozpath.split(s):
if '*' not in p:
yield p + '/'
prefix = ''.join(_prefix(src))
if '*' in f.target_basename:
target = path
else:
target = mozpath.join(path, f.target_basename)
mozpath.join(path, f.target_basename)
self._install_manifests[obj.install_target] \
.add_pattern_link(
prefix,
src[len(prefix):],
target)
else:
self._install_manifests[obj.install_target].add_link(
src, mozpath.join(path, f.target_basename))
if isinstance(f, ObjDirPath):
dep_target = 'install-%s' % obj.install_target
dep = mozpath.relpath(f.full_path,
self.environment.topobjdir)
if dep in self._generated_files_map:
# Only the first output file is specified as a
# dependency. If there are multiple output files
# from a single GENERATED_FILES invocation that are
# installed, we only want to run the command once.
dep = self._generated_files_map[dep]
self._dependencies[dep_target].append(dep)
elif isinstance(obj, ChromeManifestEntry) and \
obj.install_target.startswith('dist/bin'):
top_level = mozpath.join(obj.install_target, 'chrome.manifest')
if obj.path != top_level:
entry = 'manifest %s' % mozpath.relpath(
obj.path, obj.install_target)
self._manifest_entries[top_level].add(entry)
self._manifest_entries[obj.path].add(str(obj.entry))
elif isinstance(obj, GeneratedFile):
if obj.outputs:
first_output = mozpath.relpath(
mozpath.join(obj.objdir, obj.outputs[0]),
self.environment.topobjdir)
for o in obj.outputs[1:]:
fullpath = mozpath.join(obj.objdir, o)
self._generated_files_map[mozpath.relpath(
fullpath, self.environment.topobjdir)] = first_output
self._generated_files.append(obj)
return False
elif isinstance(obj, XPIDLModule):
self._has_xpidl = True
# We're not actually handling XPIDL files.
return False
else:
return False
return True
def consume_finished(self):
mk = Makefile()
# Add the default rule at the very beginning.
mk.create_rule(['default'])
mk.add_statement('TOPSRCDIR = %s' % self.environment.topsrcdir)
mk.add_statement('TOPOBJDIR = %s' % self.environment.topobjdir)
mk.add_statement('MDDEPDIR = .deps')
mk.add_statement('TOUCH ?= touch')
mk.add_statement('include $(TOPSRCDIR)/config/makefiles/functions.mk')
mk.add_statement('include $(TOPSRCDIR)/config/AB_rCD.mk')
mk.add_statement('AB_CD = en-US')
if not self._has_xpidl:
mk.add_statement('NO_XPIDL = 1')
# Add a few necessary variables inherited from configure
for var in (
'PYTHON3',
'ACDEFINES',
'MOZ_BUILD_APP',
'MOZ_WIDGET_TOOLKIT',
):
value = self.environment.substs.get(var)
if value is not None:
mk.add_statement('%s = %s' % (var, value))
install_manifests_bases = self._install_manifests.keys()
# Add information for chrome manifest generation
manifest_targets = []
for target, entries in six.iteritems(self._manifest_entries):
manifest_targets.append(target)
install_target = mozpath.basedir(target, install_manifests_bases)
self._install_manifests[install_target].add_content(
''.join('%s\n' % e for e in sorted(entries)),
mozpath.relpath(target, install_target))
# Add information for install manifests.
mk.add_statement('INSTALL_MANIFESTS = %s' %
' '.join(sorted(self._install_manifests.keys())))
# Add dependencies we inferred:
for target, deps in sorted(six.iteritems(self._dependencies)):
mk.create_rule([target]).add_dependencies('$(TOPOBJDIR)/%s' % d
for d in sorted(deps))
# This is not great, but it's better to have some dependencies on these Python files.
python_deps = [
'$(TOPSRCDIR)/python/mozbuild/mozbuild/action/l10n_merge.py',
'$(TOPSRCDIR)/third_party/python/compare-locales/compare_locales/compare.py',
'$(TOPSRCDIR)/third_party/python/compare-locales/compare_locales/paths.py',
]
# Add l10n dependencies we inferred:
for target, deps in sorted(six.iteritems(self._l10n_dependencies)):
mk.create_rule([target]).add_dependencies(
'%s' % d[0] for d in sorted(deps, key=itemgetter(0)))
for (merge, ref_file, l10n_file) in deps:
rule = mk.create_rule([
merge
]).add_dependencies([ref_file, l10n_file] + python_deps)
rule.add_commands([
'$(PYTHON3) -m mozbuild.action.l10n_merge '
'--output {} --ref-file {} --l10n-file {}'.format(
merge, ref_file, l10n_file)
])
# Add a dummy rule for the l10n file since it might not exist.
mk.create_rule([l10n_file])
mk.add_statement('include $(TOPSRCDIR)/config/faster/rules.mk')
for base, install_manifest in six.iteritems(self._install_manifests):
with self._write_file(
mozpath.join(self.environment.topobjdir, 'faster',
'install_%s' % base.replace('/', '_'))) as fh:
install_manifest.write(fileobj=fh)
# For artifact builds only, write a single unified manifest
# for consumption by |mach watch|.
if self.environment.is_artifact_build:
unified_manifest = InstallManifest()
for base, install_manifest in six.iteritems(
self._install_manifests):
# Expect 'dist/bin/**', which includes 'dist/bin' with no trailing slash.
assert base.startswith('dist/bin')
base = base[len('dist/bin'):]
if base and base[0] == '/':
base = base[1:]
unified_manifest.add_entries_from(install_manifest, base=base)
with self._write_file(
mozpath.join(self.environment.topobjdir, 'faster',
'unified_install_dist_bin')) as fh:
unified_manifest.write(fileobj=fh)
for obj in self._generated_files:
for stmt in self._format_statements_for_generated_file(
obj, 'default'):
mk.add_statement(stmt)
with self._write_file(
mozpath.join(self.environment.topobjdir, 'faster',
'Makefile')) as fh:
mk.dump(fh, removal_guard=False)
def _pretty_path(self, path, obj):
if path.startswith(self.environment.topobjdir):
return mozpath.join(
'$(TOPOBJDIR)',
mozpath.relpath(path, self.environment.topobjdir))
elif path.startswith(self.environment.topsrcdir):
return mozpath.join(
'$(TOPSRCDIR)',
mozpath.relpath(path, self.environment.topsrcdir))
else:
return path
def _format_generated_file_input_name(self, path, obj):
return self._pretty_path(path.full_path, obj)
def _format_generated_file_output_name(self, path, obj):
if not isinstance(path, Path):
path = ObjDirPath(obj._context, '!' + path)
return self._pretty_path(path.full_path, obj)
class FasterMakeBackend(CommonBackend):
def _init(self):
super(FasterMakeBackend, self)._init()
self._seen_directories = set()
self._defines = dict()
self._manifest_entries = OrderedDefaultDict(list)
self._install_manifests = OrderedDefaultDict(OverwriteInstallManifest)
self._dependencies = OrderedDefaultDict(list)
def _add_preprocess(self, obj, path, dest, target=None, **kwargs):
if target is None:
target = mozpath.basename(path)
# This matches what PP_TARGETS do in config/rules.
if target.endswith('.in'):
target = target[:-3]
depfile = mozpath.join(
self.environment.topobjdir, 'faster', '.deps',
mozpath.join(obj.install_target, dest, target).replace('/', '_'))
self._install_manifests[obj.install_target].add_preprocess(
mozpath.join(obj.srcdir, path),
mozpath.join(dest, target),
depfile,
**kwargs)
def consume_object(self, obj):
if not isinstance(obj, Defines) and isinstance(obj, ContextDerived):
defines = self._defines.get(obj.objdir, {})
if defines:
defines = defines.defines
if isinstance(obj, Defines):
self._defines[obj.objdir] = obj
# We're assuming below that Defines come first for a given objdir,
# which is kind of set in stone from the order things are treated
# in emitter.py.
assert obj.objdir not in self._seen_directories
elif isinstance(obj, JARManifest) and \
obj.install_target.startswith('dist/bin'):
self._consume_jar_manifest(obj, defines)
elif isinstance(obj, VariablePassthru) and \
obj.install_target.startswith('dist/bin'):
for f in obj.variables.get('EXTRA_COMPONENTS', {}):
path = mozpath.join(obj.install_target, 'components',
mozpath.basename(f))
self._install_manifests[obj.install_target].add_symlink(
mozpath.join(obj.srcdir, f),
mozpath.join('components', mozpath.basename(f))
)
if f.endswith('.manifest'):
manifest = mozpath.join(obj.install_target,
'chrome.manifest')
self._manifest_entries[manifest].append(
'manifest components/%s' % mozpath.basename(f))
for f in obj.variables.get('EXTRA_PP_COMPONENTS', {}):
self._add_preprocess(obj, f, 'components', defines=defines)
if f.endswith('.manifest'):
manifest = mozpath.join(obj.install_target,
'chrome.manifest')
self._manifest_entries[manifest].append(
'manifest components/%s' % mozpath.basename(f))
elif isinstance(obj, JavaScriptModules) and \
obj.install_target.startswith('dist/bin'):
for path, strings in obj.modules.walk():
base = mozpath.join('modules', path)
for f in strings:
if obj.flavor == 'extra':
self._install_manifests[obj.install_target].add_symlink(
mozpath.join(obj.srcdir, f),
mozpath.join(base, mozpath.basename(f))
)
elif obj.flavor == 'extra_pp':
self._add_preprocess(obj, f, base, defines=defines)
elif isinstance(obj, JsPreferenceFile) and \
obj.install_target.startswith('dist/bin'):
# The condition for the directory value in config/rules.mk is:
# ifneq (,$(DIST_SUBDIR)$(XPI_NAME)$(LIBXUL_SDK))
# - LIBXUL_SDK is not supported (it likely doesn't work in the
# recursive backend anyways
# - when XPI_NAME is set, obj.install_target will start with
# dist/xpi-stage
# - when DIST_SUBDIR is set, obj.install_target will start with
# dist/bin/$(DIST_SUBDIR)
# So an equivalent condition that is not cumbersome for us and that
# is enough at least for now is checking if obj.install_target is
# different from dist/bin.
if obj.install_target == 'dist/bin':
pref_dir = 'defaults/pref'
else:
pref_dir = 'defaults/preferences'
dest = mozpath.join(obj.install_target, pref_dir,
mozpath.basename(obj.path))
# We preprocess these, but they don't necessarily have preprocessor
# directives, so tell the preprocessor to not complain about that.
self._add_preprocess(obj, obj.path, pref_dir, defines=defines,
silence_missing_directive_warnings=True)
elif isinstance(obj, Resources) and \
obj.install_target.startswith('dist/bin'):
for path, strings in obj.resources.walk():
base = mozpath.join('res', path)
for f in strings:
flags = strings.flags_for(f)
if flags and flags.preprocess:
self._add_preprocess(obj, f, base, marker='%',
defines=obj.defines)
else:
self._install_manifests[obj.install_target].add_symlink(
mozpath.join(obj.srcdir, f),
mozpath.join(base, mozpath.basename(f))
)
elif isinstance(obj, FinalTargetFiles) and \
obj.install_target.startswith('dist/bin'):
for path, strings in obj.files.walk():
base = mozpath.join(obj.install_target, path)
for f in strings:
self._install_manifests[obj.install_target].add_symlink(
mozpath.join(obj.srcdir, f),
mozpath.join(path, mozpath.basename(f))
)
elif isinstance(obj, DistFiles) and \
obj.install_target.startswith('dist/bin'):
# We preprocess these, but they don't necessarily have preprocessor
# directives, so tell the preprocessor to not complain about that.
for f in obj.files:
self._add_preprocess(obj, f, '', defines=defines,
silence_missing_directive_warnings=True)
else:
# We currently ignore a lot of object types, so just acknowledge
# everything.
return True
self._seen_directories.add(obj.objdir)
return True
def _consume_jar_manifest(self, obj, defines):
# Ideally, this would all be handled somehow in the emitter, but
# this would require all the magic surrounding l10n and addons in
# the recursive make backend to die, which is not going to happen
# any time soon enough.
# Notably missing:
# - DEFINES from config/config.mk
# - L10n support
# - The equivalent of -e when USE_EXTENSION_MANIFEST is set in
# moz.build, but it doesn't matter in dist/bin.
pp = Preprocessor()
pp.context.update(defines)
pp.context.update(self.environment.defines)
pp.context.update(
AB_CD='en-US',
BUILD_FASTER=1,
)
pp.out = JarManifestParser()
pp.do_include(obj.path)
for jarinfo in pp.out:
install_target = obj.install_target
# Bug 1150417 added some gross hacks, which we don't try to
# support generically. Fortunately, the hacks don't define more
# than chrome manifest entries, so just assume we don't get
# any installation entries.
if jarinfo.name.startswith('../'):
assert not jarinfo.entries
base = mozpath.join('chrome', jarinfo.name)
for e in jarinfo.entries:
if e.is_locale:
src = mozpath.join(
jarinfo.relativesrcdir or mozpath.dirname(obj.path),
'en-US',
e.source)
elif e.source.startswith('/'):
src = mozpath.join(self.environment.topsrcdir,
e.source[1:])
else:
src = mozpath.join(mozpath.dirname(obj.path), e.source)
if '*' in e.source:
if e.preprocess:
raise Exception('%s: Wildcards are not supported with '
'preprocessing' % obj.path)
def _prefix(s):
for p in s.split('/'):
if '*' not in p:
yield p + '/'
prefix = ''.join(_prefix(src))
self._install_manifests[obj.install_target] \
.add_pattern_symlink(
prefix,
src[len(prefix):],
mozpath.join(base, e.output))
continue
if not os.path.exists(src):
if e.is_locale:
raise Exception(
'%s: Cannot find %s' % (obj.path, e.source))
if e.source.startswith('/'):
src = mozpath.join(self.environment.topobjdir,
e.source[1:])
else:
# This actually gets awkward if the jar.mn is not
# in the same directory as the moz.build declaring
# it, but it's how it works in the recursive make,
# not that anything relies on that, but it's simpler.
src = mozpath.join(obj.objdir, e.source)
self._dependencies['install-%s' % obj.install_target] \
.append(mozpath.relpath(
src, self.environment.topobjdir))
if e.preprocess:
kwargs = {}
if src.endswith('.css'):
kwargs['marker'] = '%'
self._add_preprocess(
obj,
src,
mozpath.join(base, mozpath.dirname(e.output)),
mozpath.basename(e.output),
defines=defines,
**kwargs)
else:
self._install_manifests[obj.install_target].add_symlink(
src,
mozpath.join(base, e.output))
manifest = mozpath.normpath(mozpath.join(obj.install_target, base))
manifest += '.manifest'
for m in jarinfo.chrome_manifests:
self._manifest_entries[manifest].append(
m.replace('%', jarinfo.name + '/'))
# ../ special cased for bug 1150417 again.
if not jarinfo.name.startswith('../'):
manifest = mozpath.normpath(mozpath.join(obj.install_target,
'chrome.manifest'))
entry = 'manifest %s.manifest' % base
if entry not in self._manifest_entries[manifest]:
self._manifest_entries[manifest].append(entry)
def consume_finished(self):
mk = Makefile()
# Add the default rule at the very beginning.
mk.create_rule(['default'])
mk.add_statement('TOPSRCDIR = %s' % self.environment.topsrcdir)
mk.add_statement('TOPOBJDIR = %s' % self.environment.topobjdir)
# Add a few necessary variables inherited from configure
for var in (
'PYTHON',
'ACDEFINES',
'MOZ_BUILD_APP',
'MOZ_WIDGET_TOOLKIT',
):
mk.add_statement('%s = %s' % (var, self.environment.substs[var]))
# Add information for chrome manifest generation
manifest_targets = []
for target, entries in self._manifest_entries.iteritems():
manifest_targets.append(target)
target = '$(TOPOBJDIR)/%s' % target
mk.create_rule([target]).add_dependencies(
['content = %s' % ' '.join('"%s"' % e for e in entries)])
mk.add_statement('MANIFEST_TARGETS = %s' % ' '.join(manifest_targets))
# Add information for install manifests.
mk.add_statement('INSTALL_MANIFESTS = %s'
% ' '.join(self._install_manifests.keys()))
# Add dependencies we infered:
for target, deps in self._dependencies.iteritems():
mk.create_rule([target]).add_dependencies(
'$(TOPOBJDIR)/%s' % d for d in deps)
mk.add_statement('include $(TOPSRCDIR)/config/faster/rules.mk')
for base, install_manifest in self._install_manifests.iteritems():
with self._write_file(
mozpath.join(self.environment.topobjdir, 'faster',
'install_%s' % base.replace('/', '_'))) as fh:
install_manifest.write(fileobj=fh)
with self._write_file(
mozpath.join(self.environment.topobjdir, 'faster',
'Makefile')) as fh:
mk.dump(fh, removal_guard=False)
class FasterMakeBackend(CommonBackend):
def _init(self):
super(FasterMakeBackend, self)._init()
self._manifest_entries = OrderedDefaultDict(set)
self._install_manifests = OrderedDefaultDict(InstallManifest)
self._dependencies = OrderedDefaultDict(list)
self._has_xpidl = False
def _add_preprocess(self, obj, path, dest, target=None, **kwargs):
if target is None:
target = mozpath.basename(path)
# This matches what PP_TARGETS do in config/rules.
if target.endswith(".in"):
target = target[:-3]
if target.endswith(".css"):
kwargs["marker"] = "%"
depfile = mozpath.join(
self.environment.topobjdir,
"faster",
".deps",
mozpath.join(obj.install_target, dest, target).replace("/", "_"),
)
self._install_manifests[obj.install_target].add_preprocess(
mozpath.join(obj.srcdir, path), mozpath.join(dest, target), depfile, **kwargs
)
def consume_object(self, obj):
if isinstance(obj, JARManifest) and obj.install_target.startswith("dist/bin"):
self._consume_jar_manifest(obj)
elif isinstance(obj, (FinalTargetFiles, FinalTargetPreprocessedFiles)) and obj.install_target.startswith(
"dist/bin"
):
defines = obj.defines or {}
if defines:
defines = defines.defines
for path, files in obj.files.walk():
for f in files:
if isinstance(obj, FinalTargetPreprocessedFiles):
self._add_preprocess(obj, f.full_path, path, target=f.target_basename, defines=defines)
elif "*" in f:
def _prefix(s):
for p in mozpath.split(s):
if "*" not in p:
yield p + "/"
prefix = "".join(_prefix(f.full_path))
self._install_manifests[obj.install_target].add_pattern_symlink(
prefix, f.full_path[len(prefix) :], mozpath.join(path, f.target_basename)
)
else:
self._install_manifests[obj.install_target].add_symlink(
f.full_path, mozpath.join(path, f.target_basename)
)
if isinstance(f, ObjDirPath):
dep_target = "install-%s" % obj.install_target
self._dependencies[dep_target].append(mozpath.relpath(f.full_path, self.environment.topobjdir))
elif isinstance(obj, ChromeManifestEntry) and obj.install_target.startswith("dist/bin"):
top_level = mozpath.join(obj.install_target, "chrome.manifest")
if obj.path != top_level:
entry = "manifest %s" % mozpath.relpath(obj.path, obj.install_target)
self._manifest_entries[top_level].add(entry)
self._manifest_entries[obj.path].add(str(obj.entry))
elif isinstance(obj, XPIDLFile):
self._has_xpidl = True
# We currently ignore a lot of object types, so just acknowledge
# everything.
return True
def consume_finished(self):
mk = Makefile()
# Add the default rule at the very beginning.
mk.create_rule(["default"])
mk.add_statement("TOPSRCDIR = %s" % self.environment.topsrcdir)
mk.add_statement("TOPOBJDIR = %s" % self.environment.topobjdir)
mk.add_statement("BACKEND = %s" % self._backend_output_list_file)
if not self._has_xpidl:
mk.add_statement("NO_XPIDL = 1")
# Add a few necessary variables inherited from configure
for var in ("PYTHON", "ACDEFINES", "MOZ_BUILD_APP", "MOZ_WIDGET_TOOLKIT"):
mk.add_statement("%s = %s" % (var, self.environment.substs[var]))
install_manifests_bases = self._install_manifests.keys()
# Add information for chrome manifest generation
manifest_targets = []
for target, entries in self._manifest_entries.iteritems():
manifest_targets.append(target)
install_target = mozpath.basedir(target, install_manifests_bases)
self._install_manifests[install_target].add_content(
"".join("%s\n" % e for e in sorted(entries)), mozpath.relpath(target, install_target)
)
# Add information for install manifests.
mk.add_statement("INSTALL_MANIFESTS = %s" % " ".join(self._install_manifests.keys()))
# Add dependencies we infered:
for target, deps in self._dependencies.iteritems():
mk.create_rule([target]).add_dependencies("$(TOPOBJDIR)/%s" % d for d in deps)
# Add backend dependencies:
mk.create_rule([self._backend_output_list_file]).add_dependencies(self.backend_input_files)
mk.add_statement("include $(TOPSRCDIR)/config/faster/rules.mk")
for base, install_manifest in self._install_manifests.iteritems():
with self._write_file(
mozpath.join(self.environment.topobjdir, "faster", "install_%s" % base.replace("/", "_"))
) as fh:
install_manifest.write(fileobj=fh)
with self._write_file(mozpath.join(self.environment.topobjdir, "faster", "Makefile")) as fh:
mk.dump(fh, removal_guard=False)
class FasterMakeBackend(CommonBackend, PartialBackend):
def _init(self):
super(FasterMakeBackend, self)._init()
self._manifest_entries = OrderedDefaultDict(set)
self._install_manifests = OrderedDefaultDict(InstallManifest)
self._dependencies = OrderedDefaultDict(list)
self._has_xpidl = False
def _add_preprocess(self, obj, path, dest, target=None, **kwargs):
if target is None:
target = mozpath.basename(path)
# This matches what PP_TARGETS do in config/rules.
if target.endswith('.in'):
target = target[:-3]
if target.endswith('.css'):
kwargs['marker'] = '%'
depfile = mozpath.join(
self.environment.topobjdir, 'faster', '.deps',
mozpath.join(obj.install_target, dest, target).replace('/', '_'))
self._install_manifests[obj.install_target].add_preprocess(
mozpath.join(obj.srcdir, path),
mozpath.join(dest, target),
depfile,
**kwargs)
def consume_object(self, obj):
if isinstance(obj, JARManifest) and \
obj.install_target.startswith('dist/bin'):
self._consume_jar_manifest(obj)
elif isinstance(obj, (FinalTargetFiles,
FinalTargetPreprocessedFiles)) and \
obj.install_target.startswith('dist/bin'):
defines = obj.defines or {}
if defines:
defines = defines.defines
for path, files in obj.files.walk():
for f in files:
if isinstance(obj, FinalTargetPreprocessedFiles):
self._add_preprocess(obj, f.full_path, path,
target=f.target_basename,
defines=defines)
elif '*' in f:
def _prefix(s):
for p in mozpath.split(s):
if '*' not in p:
yield p + '/'
prefix = ''.join(_prefix(f.full_path))
if '*' in f.target_basename:
target = path
else:
target = mozpath.join(path, f.target_basename)
mozpath.join(path, f.target_basename)
self._install_manifests[obj.install_target] \
.add_pattern_link(
prefix,
f.full_path[len(prefix):],
target)
else:
self._install_manifests[obj.install_target].add_link(
f.full_path,
mozpath.join(path, f.target_basename)
)
if isinstance(f, ObjDirPath):
dep_target = 'install-%s' % obj.install_target
self._dependencies[dep_target].append(
mozpath.relpath(f.full_path,
self.environment.topobjdir))
elif isinstance(obj, ChromeManifestEntry) and \
obj.install_target.startswith('dist/bin'):
top_level = mozpath.join(obj.install_target, 'chrome.manifest')
if obj.path != top_level:
entry = 'manifest %s' % mozpath.relpath(obj.path,
obj.install_target)
self._manifest_entries[top_level].add(entry)
self._manifest_entries[obj.path].add(str(obj.entry))
elif isinstance(obj, XPIDLFile):
self._has_xpidl = True
# We're not actually handling XPIDL files.
return False
else:
return False
return True
def consume_finished(self):
mk = Makefile()
# Add the default rule at the very beginning.
mk.create_rule(['default'])
mk.add_statement('TOPSRCDIR = %s' % self.environment.topsrcdir)
mk.add_statement('TOPOBJDIR = %s' % self.environment.topobjdir)
if not self._has_xpidl:
mk.add_statement('NO_XPIDL = 1')
# Add a few necessary variables inherited from configure
for var in (
'PYTHON',
'ACDEFINES',
'MOZ_BUILD_APP',
'MOZ_WIDGET_TOOLKIT',
):
value = self.environment.substs.get(var)
if value is not None:
mk.add_statement('%s = %s' % (var, value))
install_manifests_bases = self._install_manifests.keys()
# Add information for chrome manifest generation
manifest_targets = []
for target, entries in self._manifest_entries.iteritems():
manifest_targets.append(target)
install_target = mozpath.basedir(target, install_manifests_bases)
self._install_manifests[install_target].add_content(
''.join('%s\n' % e for e in sorted(entries)),
mozpath.relpath(target, install_target))
# Add information for install manifests.
mk.add_statement('INSTALL_MANIFESTS = %s'
% ' '.join(self._install_manifests.keys()))
# Add dependencies we infered:
for target, deps in self._dependencies.iteritems():
mk.create_rule([target]).add_dependencies(
'$(TOPOBJDIR)/%s' % d for d in deps)
mk.add_statement('include $(TOPSRCDIR)/config/faster/rules.mk')
for base, install_manifest in self._install_manifests.iteritems():
with self._write_file(
mozpath.join(self.environment.topobjdir, 'faster',
'install_%s' % base.replace('/', '_'))) as fh:
install_manifest.write(fileobj=fh)
# For artifact builds only, write a single unified manifest for consumption by |mach watch|.
if self.environment.is_artifact_build:
unified_manifest = InstallManifest()
for base, install_manifest in self._install_manifests.iteritems():
# Expect 'dist/bin/**', which includes 'dist/bin' with no trailing slash.
assert base.startswith('dist/bin')
base = base[len('dist/bin'):]
if base and base[0] == '/':
base = base[1:]
unified_manifest.add_entries_from(install_manifest, base=base)
with self._write_file(
mozpath.join(self.environment.topobjdir, 'faster',
'unified_install_dist_bin')) as fh:
unified_manifest.write(fileobj=fh)
with self._write_file(
mozpath.join(self.environment.topobjdir, 'faster',
'Makefile')) as fh:
mk.dump(fh, removal_guard=False)
class FasterMakeBackend(CommonBackend):
def _init(self):
super(FasterMakeBackend, self)._init()
self._seen_directories = set()
self._defines = dict()
self._manifest_entries = OrderedDefaultDict(set)
self._install_manifests = OrderedDefaultDict(InstallManifest)
self._dependencies = OrderedDefaultDict(list)
self._has_xpidl = False
def _add_preprocess(self, obj, path, dest, target=None, **kwargs):
if target is None:
target = mozpath.basename(path)
# This matches what PP_TARGETS do in config/rules.
if target.endswith('.in'):
target = target[:-3]
depfile = mozpath.join(
self.environment.topobjdir, 'faster', '.deps',
mozpath.join(obj.install_target, dest, target).replace('/', '_'))
self._install_manifests[obj.install_target].add_preprocess(
mozpath.join(obj.srcdir, path), mozpath.join(dest, target),
depfile, **kwargs)
def consume_object(self, obj):
if not isinstance(obj, Defines) and isinstance(obj, ContextDerived):
defines = self._defines.get(obj.objdir, {})
if defines:
defines = defines.defines
if isinstance(obj, Defines):
self._defines[obj.objdir] = obj
# We're assuming below that Defines come first for a given objdir,
# which is kind of set in stone from the order things are treated
# in emitter.py.
assert obj.objdir not in self._seen_directories
elif isinstance(obj, JARManifest) and \
obj.install_target.startswith('dist/bin'):
self._consume_jar_manifest(obj, defines)
elif isinstance(obj, (FinalTargetFiles,
FinalTargetPreprocessedFiles)) and \
obj.install_target.startswith('dist/bin'):
for path, files in obj.files.walk():
for f in files:
if isinstance(obj, FinalTargetPreprocessedFiles):
self._add_preprocess(obj,
f.full_path,
path,
defines=defines)
else:
self._install_manifests[
obj.install_target].add_symlink(
f.full_path,
mozpath.join(path, mozpath.basename(f)))
elif isinstance(obj, ChromeManifestEntry) and \
obj.install_target.startswith('dist/bin'):
top_level = mozpath.join(obj.install_target, 'chrome.manifest')
if obj.path != top_level:
entry = 'manifest %s' % mozpath.relpath(
obj.path, obj.install_target)
self._manifest_entries[top_level].add(entry)
self._manifest_entries[obj.path].add(str(obj.entry))
elif isinstance(obj, XPIDLFile):
self._has_xpidl = True
# XPIDL are emitted before Defines, which breaks the assert in the
# branch for Defines. OTOH, we don't actually care about the
# XPIDLFile objects just yet, so we can just pretend we didn't see
# an object in the directory yet.
return True
else:
# We currently ignore a lot of object types, so just acknowledge
# everything.
return True
self._seen_directories.add(obj.objdir)
return True
def _consume_jar_manifest(self, obj, defines):
# Ideally, this would all be handled somehow in the emitter, but
# this would require all the magic surrounding l10n and addons in
# the recursive make backend to die, which is not going to happen
# any time soon enough.
# Notably missing:
# - DEFINES from config/config.mk
# - L10n support
# - The equivalent of -e when USE_EXTENSION_MANIFEST is set in
# moz.build, but it doesn't matter in dist/bin.
pp = Preprocessor()
pp.context.update(defines)
pp.context.update(self.environment.defines)
pp.context.update(
AB_CD='en-US',
BUILD_FASTER=1,
)
pp.out = JarManifestParser()
pp.do_include(obj.path)
self.backend_input_files |= pp.includes
for jarinfo in pp.out:
install_target = obj.install_target
if jarinfo.base:
install_target = mozpath.normpath(
mozpath.join(install_target, jarinfo.base))
for e in jarinfo.entries:
if e.is_locale:
if jarinfo.relativesrcdir:
path = mozpath.join(self.environment.topsrcdir,
jarinfo.relativesrcdir)
else:
path = mozpath.dirname(obj.path)
src = mozpath.join(path, 'en-US', e.source)
elif e.source.startswith('/'):
src = mozpath.join(self.environment.topsrcdir,
e.source[1:])
else:
src = mozpath.join(mozpath.dirname(obj.path), e.source)
if '*' in e.source:
if e.preprocess:
raise Exception('%s: Wildcards are not supported with '
'preprocessing' % obj.path)
def _prefix(s):
for p in s.split('/'):
if '*' not in p:
yield p + '/'
prefix = ''.join(_prefix(src))
self._install_manifests[install_target] \
.add_pattern_symlink(
prefix,
src[len(prefix):],
mozpath.join(jarinfo.name, e.output))
continue
if not os.path.exists(src):
if e.is_locale:
raise Exception('%s: Cannot find %s' %
(obj.path, e.source))
if e.source.startswith('/'):
src = mozpath.join(self.environment.topobjdir,
e.source[1:])
else:
# This actually gets awkward if the jar.mn is not
# in the same directory as the moz.build declaring
# it, but it's how it works in the recursive make,
# not that anything relies on that, but it's simpler.
src = mozpath.join(obj.objdir, e.source)
self._dependencies['install-%s' % install_target] \
.append(mozpath.relpath(
src, self.environment.topobjdir))
if e.preprocess:
kwargs = {}
if src.endswith('.css'):
kwargs['marker'] = '%'
self._add_preprocess(obj,
src,
mozpath.join(
jarinfo.name,
mozpath.dirname(e.output)),
mozpath.basename(e.output),
defines=defines,
**kwargs)
else:
self._install_manifests[install_target].add_symlink(
src, mozpath.join(jarinfo.name, e.output))
manifest = mozpath.normpath(
mozpath.join(install_target, jarinfo.name))
manifest += '.manifest'
for m in jarinfo.chrome_manifests:
self._manifest_entries[manifest].add(
m.replace('%',
mozpath.basename(jarinfo.name) + '/'))
if jarinfo.name != 'chrome':
manifest = mozpath.normpath(
mozpath.join(install_target, 'chrome.manifest'))
entry = 'manifest %s.manifest' % jarinfo.name
self._manifest_entries[manifest].add(entry)
def consume_finished(self):
mk = Makefile()
# Add the default rule at the very beginning.
mk.create_rule(['default'])
mk.add_statement('TOPSRCDIR = %s' % self.environment.topsrcdir)
mk.add_statement('TOPOBJDIR = %s' % self.environment.topobjdir)
mk.add_statement('BACKEND = %s' % self._backend_output_list_file)
if not self._has_xpidl:
mk.add_statement('NO_XPIDL = 1')
# Add a few necessary variables inherited from configure
for var in (
'PYTHON',
'ACDEFINES',
'MOZ_BUILD_APP',
'MOZ_WIDGET_TOOLKIT',
):
mk.add_statement('%s = %s' % (var, self.environment.substs[var]))
install_manifests_bases = self._install_manifests.keys()
# Add information for chrome manifest generation
manifest_targets = []
for target, entries in self._manifest_entries.iteritems():
manifest_targets.append(target)
install_target = mozpath.basedir(target, install_manifests_bases)
self._install_manifests[install_target].add_content(
''.join('%s\n' % e for e in sorted(entries)),
mozpath.relpath(target, install_target))
# Add information for install manifests.
mk.add_statement('INSTALL_MANIFESTS = %s' %
' '.join(self._install_manifests.keys()))
# Add dependencies we infered:
for target, deps in self._dependencies.iteritems():
mk.create_rule([target]).add_dependencies('$(TOPOBJDIR)/%s' % d
for d in deps)
# Add backend dependencies:
mk.create_rule([self._backend_output_list_file
]).add_dependencies(self.backend_input_files)
mk.add_statement('include $(TOPSRCDIR)/config/faster/rules.mk')
for base, install_manifest in self._install_manifests.iteritems():
with self._write_file(
mozpath.join(self.environment.topobjdir, 'faster',
'install_%s' % base.replace('/', '_'))) as fh:
install_manifest.write(fileobj=fh)
with self._write_file(
mozpath.join(self.environment.topobjdir, 'faster',
'Makefile')) as fh:
mk.dump(fh, removal_guard=False)
class TestMetadata(object):
"""Holds information about tests.
This class provides an API to query tests active in the build
configuration.
"""
def __init__(self, all_tests, test_defaults=None):
self._tests_by_path = OrderedDefaultDict(list)
self._tests_by_flavor = defaultdict(set)
self._test_dirs = set()
with open(all_tests, 'rb') as fh:
test_data = pickle.load(fh)
defaults = None
if test_defaults:
with open(test_defaults, 'rb') as fh:
defaults = pickle.load(fh)
for path, tests in test_data.items():
for metadata in tests:
if defaults:
defaults_manifests = [metadata['manifest']]
ancestor_manifest = metadata.get('ancestor-manifest')
if ancestor_manifest:
defaults_manifests.append(ancestor_manifest)
for manifest in defaults_manifests:
manifest_defaults = defaults.get(manifest)
if manifest_defaults:
metadata = manifestparser.combine_fields(manifest_defaults,
metadata)
self._tests_by_path[path].append(metadata)
self._test_dirs.add(os.path.dirname(path))
flavor = metadata.get('flavor')
self._tests_by_flavor[flavor].add(path)
def tests_with_flavor(self, flavor):
"""Obtain all tests having the specified flavor.
This is a generator of dicts describing each test.
"""
for path in sorted(self._tests_by_flavor.get(flavor, [])):
yield self._tests_by_path[path]
def resolve_tests(self, paths=None, flavor=None, subsuite=None, under_path=None,
tags=None):
"""Resolve tests from an identifier.
This is a generator of dicts describing each test.
``paths`` can be an iterable of values to use to identify tests to run.
If an entry is a known test file, tests associated with that file are
returned (there may be multiple configurations for a single file). If
an entry is a directory, or a prefix of a directory containing tests,
all tests in that directory are returned. If the string appears in a
known test file, that test file is considered. If the path contains
a wildcard pattern, tests matching that pattern are returned.
If ``under_path`` is a string, it will be used to filter out tests that
aren't in the specified path prefix relative to topsrcdir or the
test's installed dir.
If ``flavor`` is a string, it will be used to filter returned tests
to only be the flavor specified. A flavor is something like
``xpcshell``.
If ``subsuite`` is a string, it will be used to filter returned tests
to only be in the subsuite specified.
If ``tags`` are specified, they will be used to filter returned tests
to only those with a matching tag.
"""
if tags:
tags = set(tags)
def fltr(tests):
for test in tests:
if flavor:
if flavor == 'devtools' and test.get('flavor') != 'browser-chrome':
continue
if flavor != 'devtools' and test.get('flavor') != flavor:
continue
if subsuite and test.get('subsuite') != subsuite:
continue
if tags and not (tags & set(test.get('tags', '').split())):
continue
if under_path and not test['file_relpath'].startswith(under_path):
continue
# Make a copy so modifications don't change the source.
yield dict(test)
paths = paths or []
paths = [mozpath.normpath(p) for p in paths]
if not paths:
paths = [None]
candidate_paths = set()
for path in sorted(paths):
if path is None:
candidate_paths |= set(self._tests_by_path.keys())
continue
if '*' in path:
candidate_paths |= {p for p in self._tests_by_path
if mozpath.match(p, path)}
continue
# If the path is a directory, or the path is a prefix of a directory
# containing tests, pull in all tests in that directory.
if (path in self._test_dirs or
any(p.startswith(path) for p in self._tests_by_path)):
candidate_paths |= {p for p in self._tests_by_path
if p.startswith(path)}
continue
# If it's a test file, add just that file.
candidate_paths |= {p for p in self._tests_by_path if path in p}
for p in sorted(candidate_paths):
tests = self._tests_by_path[p]
for test in fltr(tests):
yield test
class FasterMakeBackend(CommonBackend):
def _init(self):
super(FasterMakeBackend, self)._init()
self._seen_directories = set()
self._defines = dict()
self._jar_manifests = OrderedDict()
self._preprocess_files = OrderedDict()
self._manifest_entries = OrderedDefaultDict(list)
self._install_manifests = OrderedDefaultDict(InstallManifest)
def consume_object(self, obj):
if not isinstance(obj, Defines) and isinstance(obj, ContextDerived):
defines = self._defines.get(obj.objdir, [])
if defines:
defines = list(defines.get_defines())
if isinstance(obj, Defines):
self._defines[obj.objdir] = obj
# We're assuming below that Defines come first for a given objdir,
# which is kind of set in stone from the order things are treated
# in emitter.py.
assert obj.objdir not in self._seen_directories
elif isinstance(obj, JARManifest) and obj.install_target.startswith("dist/bin"):
self._jar_manifests[obj.path] = (obj.objdir, obj.install_target, defines)
elif isinstance(obj, VariablePassthru) and obj.install_target.startswith("dist/bin"):
for f in obj.variables.get("EXTRA_COMPONENTS", {}):
path = mozpath.join(obj.install_target, "components", mozpath.basename(f))
self._install_manifests[obj.install_target].add_symlink(
mozpath.join(obj.srcdir, f), mozpath.join("components", mozpath.basename(f))
)
if f.endswith(".manifest"):
manifest = mozpath.join(obj.install_target, "chrome.manifest")
self._manifest_entries[manifest].append("manifest components/%s" % mozpath.basename(f))
for f in obj.variables.get("EXTRA_PP_COMPONENTS", {}):
path = mozpath.join(obj.install_target, "components", mozpath.basename(f))
self._preprocess_files[path] = (obj.srcdir, f, defines)
if f.endswith(".manifest"):
manifest = mozpath.join(obj.install_target, "chrome.manifest")
self._manifest_entries[manifest].append("manifest components/%s" % mozpath.basename(f))
elif isinstance(obj, JavaScriptModules) and obj.install_target.startswith("dist/bin"):
for path, strings in obj.modules.walk():
base = mozpath.join(obj.install_target, "modules", path)
for f in strings:
if obj.flavor == "extra":
self._install_manifests[obj.install_target].add_symlink(
mozpath.join(obj.srcdir, f), mozpath.join("modules", path, mozpath.basename(f))
)
elif obj.flavor == "extra_pp":
dest = mozpath.join(base, mozpath.basename(f))
self._preprocess_files[dest] = (obj.srcdir, f, defines)
elif isinstance(obj, JsPreferenceFile) and obj.install_target.startswith("dist/bin"):
# The condition for the directory value in config/rules.mk is:
# ifneq (,$(DIST_SUBDIR)$(XPI_NAME)$(LIBXUL_SDK))
# - LIBXUL_SDK is not supported (it likely doesn't work in the
# recursive backend anyways
# - when XPI_NAME is set, obj.install_target will start with
# dist/xpi-stage
# - when DIST_SUBDIR is set, obj.install_target will start with
# dist/bin/$(DIST_SUBDIR)
# So an equivalent condition that is not cumbersome for us and that
# is enough at least for now is checking if obj.install_target is
# different from dist/bin.
if obj.install_target == "dist/bin":
pref_dir = "defaults/pref"
else:
pref_dir = "defaults/preferences"
dest = mozpath.join(obj.install_target, pref_dir, mozpath.basename(obj.path))
# on win32, pref files need CRLF line endings... see bug 206029
if self.environment.substs["OS_ARCH"] == "WINNT":
defines.append("--line-endings=crlf")
# We preprocess these, but they don't necessarily have preprocessor
# directives, so tell the preprocessor to not complain about that.
defines.append("--silence-missing-directive-warnings")
self._preprocess_files[dest] = (obj.srcdir, obj.path, defines)
elif isinstance(obj, Resources) and obj.install_target.startswith("dist/bin"):
for path, strings in obj.resources.walk():
base = mozpath.join(obj.install_target, "res", path)
for f in strings:
flags = strings.flags_for(f)
if flags and flags.preprocess:
dest = mozpath.join(base, mozpath.basename(f))
defines = Defines(obj._context, obj.defines)
defines = list(defines.get_defines())
defines.extend(["--marker", "%"])
self._preprocess_files[dest] = (obj.srcdir, f, defines)
else:
self._install_manifests[obj.install_target].add_symlink(
mozpath.join(obj.srcdir, f), mozpath.join("res", path, mozpath.basename(f))
)
elif isinstance(obj, FinalTargetFiles) and obj.install_target.startswith("dist/bin"):
for path, strings in obj.files.walk():
base = mozpath.join(obj.install_target, path)
for f in strings:
self._install_manifests[obj.install_target].add_symlink(
mozpath.join(obj.srcdir, f), mozpath.join(path, mozpath.basename(f))
)
elif isinstance(obj, DistFiles) and obj.install_target.startswith("dist/bin"):
# We preprocess these, but they don't necessarily have preprocessor
# directives, so tell the preprocessor to not complain about that.
defines.append("--silence-missing-directive-warnings")
for f in obj.files:
dest = mozpath.join(obj.install_target, mozpath.basename(f))
self._preprocess_files[dest] = (obj.srcdir, f, defines)
else:
# We currently ignore a lot of object types, so just acknowledge
# everything.
return True
self._seen_directories.add(obj.objdir)
return True
def consume_finished(self):
mk = Makefile()
# Add the default rule at the very beginning.
mk.create_rule(["default"])
mk.add_statement("TOPSRCDIR = %s" % self.environment.topsrcdir)
mk.add_statement("TOPOBJDIR = %s" % self.environment.topobjdir)
# Add a few necessary variables inherited from configure
for var in ("PYTHON", "ACDEFINES", "MOZ_CHROME_FILE_FORMAT"):
mk.add_statement("%s = %s" % (var, self.environment.substs[var]))
# Add all necessary information for jar manifest processing
jar_mn_targets = []
for path, (objdir, install_target, defines) in self._jar_manifests.iteritems():
rel_manifest = mozpath.relpath(path, self.environment.topsrcdir)
target = rel_manifest.replace("/", "-")
assert target not in jar_mn_targets
jar_mn_targets.append(target)
target = "jar-%s" % target
mk.create_rule([target]).add_dependencies([path])
if objdir != mozpath.join(self.environment.topobjdir, mozpath.dirname(rel_manifest)):
mk.create_rule([target]).add_dependencies(["objdir = %s" % objdir])
if install_target != "dist/bin":
mk.create_rule([target]).add_dependencies(["install_target = %s" % install_target])
if defines:
mk.create_rule([target]).add_dependencies(["defines = %s" % " ".join(defines)])
mk.add_statement("JAR_MN_TARGETS = %s" % " ".join(jar_mn_targets))
# Add information for chrome manifest generation
manifest_targets = []
for target, entries in self._manifest_entries.iteritems():
manifest_targets.append(target)
target = "$(TOPOBJDIR)/%s" % target
mk.create_rule([target]).add_dependencies(["content = %s" % " ".join('"%s"' % e for e in entries)])
mk.add_statement("MANIFEST_TARGETS = %s" % " ".join(manifest_targets))
# Add information for preprocessed files.
preprocess_targets = []
for target, (srcdir, f, defines) in self._preprocess_files.iteritems():
# This matches what PP_TARGETS do in config/rules.
if target.endswith(".in"):
target = target[:-3]
# PP_TARGETS assumes this is true, but doesn't enforce it.
assert target not in self._preprocess_files
preprocess_targets.append(target)
target = "$(TOPOBJDIR)/%s" % target
mk.create_rule([target]).add_dependencies([mozpath.join(srcdir, f)])
if defines:
mk.create_rule([target]).add_dependencies(["defines = %s" % " ".join(defines)])
mk.add_statement("PP_TARGETS = %s" % " ".join(preprocess_targets))
# Add information for install manifests.
mk.add_statement("INSTALL_MANIFESTS = %s" % " ".join(self._install_manifests.keys()))
mk.add_statement("include $(TOPSRCDIR)/config/faster/rules.mk")
for base, install_manifest in self._install_manifests.iteritems():
with self._write_file(
mozpath.join(self.environment.topobjdir, "faster", "install_%s" % base.replace("/", "_"))
) as fh:
install_manifest.write(fileobj=fh)
with self._write_file(mozpath.join(self.environment.topobjdir, "faster", "Makefile")) as fh:
mk.dump(fh, removal_guard=False)
