def _process_reftest_manifest(self, sandbox, flavor, manifest_path):
manifest_path = mozpath.normpath(manifest_path)
manifest_full_path = mozpath.normpath(mozpath.join(sandbox["SRCDIR"], manifest_path))
manifest_reldir = mozpath.dirname(mozpath.relpath(manifest_full_path, sandbox["TOPSRCDIR"]))
manifest = reftest.ReftestManifest()
manifest.load(manifest_full_path)
# reftest manifests don't come from manifest parser. But they are
# similar enough that we can use the same emitted objects. Note
# that we don't perform any installs for reftests.
obj = TestManifest(
sandbox,
manifest_full_path,
manifest,
flavor=flavor,
install_prefix="%s/" % flavor,
relpath=mozpath.join(manifest_reldir, mozpath.basename(manifest_path)),
)
for test in sorted(manifest.files):
obj.tests.append(
{
"path": test,
"here": mozpath.dirname(test),
"manifest": manifest_full_path,
"name": mozpath.basename(test),
"head": "",
"tail": "",
"support-files": "",
"subsuite": "",
}
)
yield obj
def include_file(self, path):
'''Include one file in the sandbox. Users of this class probably want
Note: this will execute all template invocations, as well as @depends
functions that depend on '--help', but nothing else.
'''
if self._paths:
path = mozpath.join(mozpath.dirname(self._paths[-1]), path)
path = mozpath.normpath(path)
if not mozpath.basedir(path, (mozpath.dirname(self._paths[0]),)):
raise ConfigureError(
'Cannot include `%s` because it is not in a subdirectory '
'of `%s`' % (path, mozpath.dirname(self._paths[0])))
else:
path = mozpath.realpath(mozpath.abspath(path))
if path in self._all_paths:
raise ConfigureError(
'Cannot include `%s` because it was included already.' % path)
self._paths.append(path)
self._all_paths.add(path)
source = open(path, 'rb').read()
code = compile(source, path, 'exec')
exec_(code, self)
self._paths.pop(-1)
def _global_dependencies_changed(self):
"""Determine whether the global dependencies have changed."""
current_files = set(iter_modules_in_path(mozpath.dirname(__file__)))
# We need to catch other .py files from /dom/bindings. We assume these
# are in the same directory as the config file.
current_files |= set(iter_modules_in_path(mozpath.dirname(self._config_path)))
current_files.add(self._config_path)
current_hashes = {}
for f in current_files:
# This will fail if the file doesn't exist. If a current global
# dependency doesn't exist, something else is wrong.
with open(f, 'rb') as fh:
current_hashes[f] = hashlib.sha1(fh.read()).hexdigest()
# The set of files has changed.
if current_files ^ set(self._state['global_depends'].keys()):
return True, current_hashes
# Compare hashes.
for f, sha1 in current_hashes.items():
if sha1 != self._state['global_depends'][f]:
return True, current_hashes
return False, current_hashes
def _process_reftest_manifest(self, sandbox, flavor, manifest_path):
manifest_path = mozpath.normpath(manifest_path)
manifest_full_path = mozpath.normpath(mozpath.join(
sandbox['SRCDIR'], manifest_path))
manifest_reldir = mozpath.dirname(mozpath.relpath(manifest_full_path,
sandbox['TOPSRCDIR']))
manifest = reftest.ReftestManifest()
manifest.load(manifest_full_path)
# reftest manifests don't come from manifest parser. But they are
# similar enough that we can use the same emitted objects. Note
# that we don't perform any installs for reftests.
obj = TestManifest(sandbox, manifest_full_path, manifest,
flavor=flavor, install_prefix='%s/' % flavor,
relpath=mozpath.join(manifest_reldir,
mozpath.basename(manifest_path)))
for test in sorted(manifest.files):
obj.tests.append({
'path': test,
'here': mozpath.dirname(test),
'manifest': manifest_full_path,
'name': mozpath.basename(test),
'head': '',
'tail': '',
'support-files': '',
'subsuite': '',
})
yield obj
def install_test_files(topsrcdir, topobjdir, tests_root, test_objs):
"""Installs the requested test files to the objdir. This is invoked by
test runners to avoid installing tens of thousands of test files when
only a few tests need to be run.
"""
flavor_info = {flavor: (root, prefix, install)
for (flavor, root, prefix, install) in TEST_MANIFESTS.values()}
objdir_dest = mozpath.join(topobjdir, tests_root)
converter = SupportFilesConverter()
install_info = TestInstallInfo()
for o in test_objs:
flavor = o['flavor']
if flavor not in flavor_info:
# This is a test flavor that isn't installed by the build system.
continue
root, prefix, install = flavor_info[flavor]
if not install:
# This flavor isn't installed to the objdir.
continue
manifest_path = o['manifest']
manifest_dir = mozpath.dirname(manifest_path)
out_dir = mozpath.join(root, prefix, manifest_dir[len(topsrcdir) + 1:])
file_relpath = o['file_relpath']
source = mozpath.join(topsrcdir, file_relpath)
dest = mozpath.join(root, prefix, file_relpath)
if 'install-to-subdir' in o:
out_dir = mozpath.join(out_dir, o['install-to-subdir'])
manifest_relpath = mozpath.relpath(source, mozpath.dirname(manifest_path))
dest = mozpath.join(out_dir, manifest_relpath)
install_info.installs.append((source, dest))
install_info |= converter.convert_support_files(o, root,
manifest_dir,
out_dir)
manifest = InstallManifest()
for source, dest in set(install_info.installs):
if dest in install_info.external_installs:
continue
manifest.add_symlink(source, dest)
for base, pattern, dest in install_info.pattern_installs:
manifest.add_pattern_symlink(base, pattern, dest)
_resolve_installs(install_info.deferred_installs, topobjdir, manifest)
# Harness files are treated as a monolith and installed each time we run tests.
# Fortunately there are not very many.
manifest |= InstallManifest(mozpath.join(topobjdir,
'_build_manifests',
'install', tests_root))
copier = FileCopier()
manifest.populate_registry(copier)
copier.copy(objdir_dest,
remove_unaccounted=False)
def __init__(self, context, manifest_path, entry):
ContextDerived.__init__(self, context)
assert isinstance(entry, ManifestEntry)
self.path = mozpath.join(self.install_target, manifest_path)
# Ensure the entry is relative to the directory containing the
# manifest path.
entry = entry.rebase(mozpath.dirname(manifest_path))
# Then add the install_target to the entry base directory.
self.entry = entry.move(mozpath.dirname(self.path))
def add_interfaces(self, path, content):
# Interfaces in the same directory are all linked together in an
# interfaces.xpt file.
interfaces_path = mozpath.join(mozpath.dirname(path),
'interfaces.xpt')
if not self.copier.contains(interfaces_path):
FlatFormatter.add_manifest(self, ManifestInterfaces(
mozpath.dirname(path), 'interfaces.xpt'))
self.copier.add(interfaces_path, XPTFile())
self.copier[interfaces_path].add(content)
def close(self, auto_root_manifest=True):
'''
Add possibly missing bits and push all instructions to the formatter.
'''
if auto_root_manifest:
# Simple package manifests don't contain the root manifests, so
# find and add them.
paths = [mozpath.dirname(m) for m in self._manifests]
path = mozpath.dirname(mozpath.commonprefix(paths))
for p, f in self._finder.find(mozpath.join(path,
'chrome.manifest')):
if not p in self._manifests:
self.packager.add(SimpleManifestSink.normalize_path(p), f)
self.packager.close()
def add_manifest(self, entry):
# Store manifest entries in a single manifest per directory, named
# after their parent directory, except for root manifests, all named
# chrome.manifest.
if entry.base:
name = mozpath.basename(entry.base)
else:
name = 'chrome'
path = mozpath.normpath(mozpath.join(entry.base, '%s.manifest' % name))
if not self.copier.contains(path):
# Add a reference to the manifest file in the parent manifest, if
# the manifest file is not a root manifest.
if entry.base:
parent = mozpath.dirname(entry.base)
relbase = mozpath.basename(entry.base)
relpath = mozpath.join(relbase,
mozpath.basename(path))
self.add_manifest(Manifest(parent, relpath))
self.copier.add(path, ManifestFile(entry.base))
if isinstance(entry, ManifestChrome):
data = self._chrome_db.setdefault(entry.name, {})
entries = data.setdefault(entry.type, [])
for e in entries:
# Ideally, we'd actually check whether entry.flags are more
# specific than e.flags, but in practice the following test
# is enough for now.
if not entry.flags or e.flags and entry.flags == e.flags:
errors.fatal('"%s" overrides "%s"' % (entry, e))
entries.append(entry)
self.copier[path].add(entry)
def normalize_path(self, path, filesystem_absolute=False, srcdir=None):
"""Normalizes paths.
If the path is absolute, behavior is governed by filesystem_absolute.
If filesystem_absolute is True, the path is interpreted as absolute on
the actual filesystem. If it is false, the path is treated as absolute
within the current topsrcdir.
If the path is not absolute, it will be treated as relative to the
currently executing file. If there is no currently executing file, it
will be treated as relative to topsrcdir.
"""
if os.path.isabs(path):
if filesystem_absolute:
return path
for root in [self.topsrcdir] + self.external_source_dirs:
# mozpath.join would ignore the self.topsrcdir argument if we
# passed in the absolute path, so omit the leading /
p = mozpath.normpath(mozpath.join(root, path[1:]))
if os.path.exists(p):
return p
# mozpath.join would ignore the self.topsrcdir argument if we passed
# in the absolute path, so omit the leading /
return mozpath.normpath(mozpath.join(self.topsrcdir, path[1:]))
elif srcdir:
return mozpath.normpath(mozpath.join(srcdir, path))
elif len(self._execution_stack):
return mozpath.normpath(mozpath.join(
mozpath.dirname(self._execution_stack[-1]), path))
else:
return mozpath.normpath(mozpath.join(self.topsrcdir, path))
def _write_file(self, path=None, fh=None):
"""Context manager to write a file.
This is a glorified wrapper around FileAvoidWrite with integration to
update the BackendConsumeSummary on this instance.
Example usage:
with self._write_file('foo.txt') as fh:
fh.write('hello world')
"""
if path is not None:
assert fh is None
fh = FileAvoidWrite(path)
else:
assert fh is not None
dirname = mozpath.dirname(fh.name)
try:
os.makedirs(dirname)
except OSError as error:
if error.errno != errno.EEXIST:
raise
yield fh
self._backend_output_files.add(mozpath.relpath(fh.name, self.environment.topobjdir))
existed, updated = fh.close()
if not existed:
self.summary.created_count += 1
elif updated:
self.summary.updated_count += 1
else:
self.summary.unchanged_count += 1
def _get_preprocessor(self, obj):
'''Returns a preprocessor with a few predefined values depending on
the given BaseConfigSubstitution(-like) object, and all the substs
in the current environment.'''
pp = Preprocessor()
srcdir = mozpath.dirname(obj.input_path)
pp.context.update(obj.config.substs)
pp.context.update(
top_srcdir=obj.topsrcdir,
srcdir=srcdir,
relativesrcdir=mozpath.relpath(srcdir, obj.topsrcdir) or '.',
DEPTH=mozpath.relpath(obj.topobjdir, mozpath.dirname(obj.output_path)) or '.',
)
pp.do_filter('attemptSubstitution')
pp.setMarker(None)
with self._write_file(obj.output_path) as fh:
pp.out = fh
yield pp
def _get_preprocessor(self, obj):
"""Returns a preprocessor with a few predefined values depending on
the given BaseConfigSubstitution(-like) object, and all the substs
in the current environment."""
pp = Preprocessor()
srcdir = mozpath.dirname(obj.input_path)
pp.context.update({k: " ".join(v) if isinstance(v, list) else v for k, v in obj.config.substs.iteritems()})
pp.context.update(
top_srcdir=obj.topsrcdir,
topobjdir=obj.topobjdir,
srcdir=srcdir,
relativesrcdir=mozpath.relpath(srcdir, obj.topsrcdir) or ".",
DEPTH=mozpath.relpath(obj.topobjdir, mozpath.dirname(obj.output_path)) or ".",
)
pp.do_filter("attemptSubstitution")
pp.setMarker(None)
with self._write_file(obj.output_path) as fh:
pp.out = fh
yield pp
def _partial_paths(self, path):
'''
Turn "foo/bar/baz/zot" into ["foo/bar/baz", "foo/bar", "foo"].
'''
partial_paths = []
partial_path = path
while partial_path:
partial_path = mozpath.dirname(partial_path)
if partial_path:
partial_paths.append(partial_path)
return partial_paths
def add(self, t, flavor=None):
t = dict(t)
t['flavor'] = flavor
path = mozpath.normpath(t['path'])
assert path.startswith(self.topsrcdir)
key = path[len(self.topsrcdir)+1:]
t['file_relpath'] = key
t['dir_relpath'] = mozpath.dirname(key)
self.tests_by_path[key].append(t)
def consume(self, objs):
"""Consume a stream of TreeMetadata instances.
This is the main method of the interface. This is what takes the
frontend output and does something with it.
Child classes are not expected to implement this method. Instead, the
base class consumes objects and calls methods (possibly) implemented by
child classes.
"""
for obj in objs:
obj_start = time.time()
if not self.consume_object(obj):
raise Exception('Unhandled object of type %s' % type(obj))
self._execution_time += time.time() - obj_start
if isinstance(obj, ContextDerived):
self.backend_input_files |= obj.context_all_paths
# Pull in all loaded Python as dependencies so any Python changes that
# could influence our output result in a rescan.
self.backend_input_files |= set(iter_modules_in_path(
self.environment.topsrcdir, self.environment.topobjdir))
finished_start = time.time()
self.consume_finished()
self._execution_time += time.time() - finished_start
# Purge backend files created in previous run, but not created anymore
delete_files = self._backend_output_list - self._backend_output_files
for path in delete_files:
try:
os.unlink(mozpath.join(self.environment.topobjdir, path))
self._deleted_count += 1
except OSError:
pass
# Remove now empty directories
for dir in set(mozpath.dirname(d) for d in delete_files):
try:
os.removedirs(dir)
except OSError:
pass
# Write out the list of backend files generated, if it changed.
if self._deleted_count or self._created_count or \
not os.path.exists(self._backend_output_list_file):
with open(self._backend_output_list_file, 'w') as fh:
fh.write('\n'.join(sorted(self._backend_output_files)))
else:
# Always update its mtime.
with open(self._backend_output_list_file, 'a'):
pass # can't utime an open file on OS/2, let it get closed first
os.utime(self._backend_output_list_file, None)
def add(self, t, flavor, topsrcdir):
t = dict(t)
t['flavor'] = flavor
path = mozpath.normpath(t['path'])
assert mozpath.basedir(path, [topsrcdir])
key = path[len(topsrcdir)+1:]
t['file_relpath'] = key
t['dir_relpath'] = mozpath.dirname(key)
self.tests_by_path[key].append(t)
def _handle_ipdl_sources(self, ipdl_dir, sorted_ipdl_sources, sorted_nonstatic_ipdl_sources,
sorted_static_ipdl_sources, unified_ipdl_cppsrcs_mapping):
# Preferably we wouldn't have to import ipdl, but we need to parse the
# ast in order to determine the namespaces since they are used in the
# header output paths.
sys.path.append(mozpath.join(self.environment.topsrcdir, 'ipc', 'ipdl'))
import ipdl
backend_file = self._get_backend_file('ipc/ipdl')
outheaderdir = '_ipdlheaders'
srcdir = mozpath.join(self.environment.topsrcdir, 'ipc/ipdl')
cmd = [
'$(PYTHON_PATH)',
'$(PLY_INCLUDE)',
'%s/ipdl.py' % srcdir,
'--sync-msg-list=%s/sync-messages.ini' % srcdir,
'--msg-metadata=%s/message-metadata.ini' % srcdir,
'--outheaders-dir=%s' % outheaderdir,
'--outcpp-dir=.',
]
ipdldirs = sorted(set(mozpath.dirname(p) for p in sorted_ipdl_sources))
cmd.extend(['-I%s' % d for d in ipdldirs])
cmd.extend(sorted_ipdl_sources)
outputs = ['IPCMessageTypeName.cpp', mozpath.join(outheaderdir, 'IPCMessageStart.h'), 'ipdl_lextab.py', 'ipdl_yacctab.py']
for filename in sorted_ipdl_sources:
filepath, ext = os.path.splitext(filename)
dirname, basename = os.path.split(filepath)
dirname = mozpath.relpath(dirname, self.environment.topsrcdir)
extensions = ['']
if ext == '.ipdl':
extensions.extend(['Child', 'Parent'])
with open(filename) as f:
ast = ipdl.parse(f.read(), filename, includedirs=ipdldirs)
self.backend_input_files.add(filename)
headerdir = os.path.join(outheaderdir, *([ns.name for ns in ast.namespaces]))
for extension in extensions:
outputs.append("%s%s.cpp" % (basename, extension))
outputs.append(mozpath.join(headerdir, '%s%s.h' % (basename, extension)))
backend_file.rule(
display='IPDL code generation',
cmd=cmd,
outputs=outputs,
extra_outputs=[self._installed_files],
check_unchanged=True,
)
backend_file.sources['.cpp'].extend(u[0] for u in unified_ipdl_cppsrcs_mapping)
def add(self, path, file):
'''
Add the given BaseFile instance with the given path.
'''
assert not self._closed
if is_manifest(path):
self._add_manifest_file(path, file)
elif path.endswith('.xpt'):
self._queue.append(self.formatter.add_interfaces, path, file)
else:
self._file_queue.append(self.formatter.add, path, file)
if mozpath.basename(path) == 'install.rdf':
self._addons.add(mozpath.dirname(path))
def _fill_with_jar(self, base, jar):
for j in jar:
path = mozpath.join(base, j.filename)
if is_manifest(j.filename):
m = self.files[path] if self.files.contains(path) \
else ManifestFile(mozpath.dirname(path))
for e in parse_manifest(None, path, j):
m.add(e)
if not self.files.contains(path):
self.files.add(path, m)
continue
else:
self.files.add(path, DeflatedFile(j))
def __init__(self, source):
if isinstance(source, BaseFinder):
self._finder = source
else:
self._finder = FileFinder(source)
self.base = self._finder.base
self.files = FileRegistry()
self.kind = 'flat'
self.omnijar = None
self.jarlogs = {}
self.optimizedjars = False
self.compressed = True
jars = set()
for p, f in self._finder.find('*'):
# Skip the precomplete file, which is generated at packaging time.
if p == 'precomplete':
continue
base = mozpath.dirname(p)
# If the file is a zip/jar that is not a .xpi, and contains a
# chrome.manifest, it is an omnijar. All the files it contains
# go in the directory containing the omnijar. Manifests are merged
# if there is a corresponding manifest in the directory.
if not p.endswith('.xpi') and self._maybe_zip(f) and \
(mozpath.basename(p) == self.omnijar or
not self.omnijar):
jar = self._open_jar(p, f)
if 'chrome.manifest' in jar:
self.kind = 'omni'
self.omnijar = mozpath.basename(p)
self._fill_with_jar(base, jar)
continue
# If the file is a manifest, scan its entries for some referencing
# jar: urls. If there are some, the files contained in the jar they
# point to, go under a directory named after the jar.
if is_manifest(p):
m = self.files[p] if self.files.contains(p) \
else ManifestFile(base)
for e in parse_manifest(self.base, p, f.open()):
m.add(self._handle_manifest_entry(e, jars))
if self.files.contains(p):
continue
f = m
# If the file is a packed addon, unpack it under a directory named
# after the xpi.
if p.endswith('.xpi') and self._maybe_zip(f):
self._fill_with_jar(p[:-4], self._open_jar(p, f))
continue
if not p in jars:
self.files.add(p, f)
def __init__(self, sandbox, path, manifest, flavor=None,
install_prefix=None, relpath=None, dupe_manifest=False):
SandboxDerived.__init__(self, sandbox)
self.path = path
self.directory = mozpath.dirname(path)
self.manifest = manifest
self.flavor = flavor
self.install_prefix = install_prefix
self.manifest_relpath = relpath
self.dupe_manifest = dupe_manifest
self.installs = {}
self.pattern_installs = []
self.tests = []
self.external_installs = set()
def exec_file(self, path):
'''Execute one file within the sandbox. Users of this class probably
want to use `run` instead.'''
if self._paths:
path = mozpath.join(mozpath.dirname(self._paths[-1]), path)
if not mozpath.basedir(path, (mozpath.dirname(self._paths[0]),)):
raise ConfigureError(
'Cannot include `%s` because it is not in a subdirectory '
'of `%s`' % (path, mozpath.dirname(self._paths[0])))
else:
path = mozpath.realpath(mozpath.abspath(path))
if path in self._paths:
raise ConfigureError(
'Cannot include `%s` because it was included already.' % path)
self._paths.append(path)
source = open(path, 'rb').read()
code = compile(source, path, 'exec')
exec(code, self)
self._paths.pop(-1)
def test_check_prog_with_path(self):
config, out, status = self.get_result('check_prog("A", ("known-a",), paths=["/some/path"])')
self.assertEqual(status, 1)
self.assertEqual(config, {})
self.assertEqual(out, textwrap.dedent('''\
checking for a... not found
DEBUG: a: Trying known-a
ERROR: Cannot find a
'''))
config, out, status = self.get_result('check_prog("A", ("known-a",), paths=["%s"])' %
os.path.dirname(self.OTHER_A))
self.assertEqual(status, 0)
self.assertEqual(config, {'A': self.OTHER_A})
self.assertEqual(out, textwrap.dedent('''\
checking for a... %s
''' % self.OTHER_A))
dirs = map(mozpath.dirname, (self.OTHER_A, self.KNOWN_A))
config, out, status = self.get_result(textwrap.dedent('''\
check_prog("A", ("known-a",), paths=["%s"])
''' % os.pathsep.join(dirs)))
self.assertEqual(status, 0)
self.assertEqual(config, {'A': self.OTHER_A})
self.assertEqual(out, textwrap.dedent('''\
checking for a... %s
''' % self.OTHER_A))
dirs = map(mozpath.dirname, (self.KNOWN_A, self.KNOWN_B))
config, out, status = self.get_result(textwrap.dedent('''\
check_prog("A", ("known-a",), paths=["%s", "%s"])
''' % (os.pathsep.join(dirs), self.OTHER_A)))
self.assertEqual(status, 0)
self.assertEqual(config, {'A': self.KNOWN_A})
self.assertEqual(out, textwrap.dedent('''\
checking for a... %s
''' % self.KNOWN_A))
config, out, status = self.get_result('check_prog("A", ("known-a",), paths="%s")' %
os.path.dirname(self.OTHER_A))
self.assertEqual(status, 1)
self.assertEqual(config, {})
self.assertEqual(out, textwrap.dedent('''\
checking for a...
DEBUG: a: Trying known-a
ERROR: Paths provided to find_program must be a list of strings, not %r
''' % mozpath.dirname(self.OTHER_A)))
def get_bases(self, addons=True):
'''
Return all paths under which root manifests have been found. Root
manifests are manifests that are included in no other manifest.
`addons` indicates whether to include addon bases as well.
'''
all_bases = set(mozpath.dirname(m)
for m in self._manifests
- set(self._included_manifests))
if not addons:
all_bases -= set(self._addons)
else:
# If for some reason some detected addon doesn't have a
# non-included manifest.
all_bases |= set(self._addons)
return all_bases
def parse_manifest(root, path, fileobj=None):
'''
Parse a manifest file.
'''
base = mozpath.dirname(path)
if root:
path = os.path.normpath(os.path.abspath(os.path.join(root, path)))
if not fileobj:
fileobj = open(path)
linenum = 0
for line in fileobj:
linenum += 1
with errors.context(path, linenum):
e = parse_manifest_line(base, line)
if e:
yield e
def __init__(self, context, path, manifest, flavor=None, install_prefix=None, relpath=None, dupe_manifest=False):
ContextDerived.__init__(self, context)
assert flavor in all_test_flavors()
self.path = path
self.directory = mozpath.dirname(path)
self.manifest = manifest
self.flavor = flavor
self.install_prefix = install_prefix
self.manifest_relpath = relpath
self.manifest_obj_relpath = relpath
self.dupe_manifest = dupe_manifest
self.installs = {}
self.pattern_installs = []
self.tests = []
self.external_installs = set()
def add(self, t, flavor=None, topsrcdir=None):
t = dict(t)
t["flavor"] = flavor
if topsrcdir is None:
topsrcdir = self.topsrcdir
else:
topsrcdir = mozpath.normpath(topsrcdir)
path = mozpath.normpath(t["path"])
assert mozpath.basedir(path, [topsrcdir])
key = path[len(topsrcdir) + 1 :]
t["file_relpath"] = key
t["dir_relpath"] = mozpath.dirname(key)
self.tests_by_path[key].append(t)
def add(self, path, file):
'''
Add the given BaseFile instance with the given path.
'''
assert not self._closed
if is_manifest(path):
self._add_manifest_file(path, file)
elif path.endswith('.xpt'):
self._queue.append(self.formatter.add_interfaces, path, file)
else:
self._file_queue.append(self.formatter.add, path, file)
if mozpath.basename(path) == 'install.rdf':
addon = True
install_rdf = file.open().read()
if self.UNPACK_ADDON_RE.search(install_rdf):
addon = 'unpacked'
self._addons[mozpath.dirname(path)] = addon
def __init__(self, config, gyp_dir_attrs, path, output, executor,
action_overrides, non_unified_sources):
self._path = path
self._config = config
self._output = output
self._non_unified_sources = non_unified_sources
self._gyp_dir_attrs = gyp_dir_attrs
self._action_overrides = action_overrides
self.execution_time = 0.0
self._results = []
# gyp expects plain str instead of unicode. The frontend code gives us
# unicode strings, so convert them.
path = encode(path)
if bool(config.substs['_MSC_VER']):
# This isn't actually used anywhere in this generator, but it's needed
# to override the registry detection of VC++ in gyp.
os.environ['GYP_MSVS_OVERRIDE_PATH'] = 'fake_path'
os.environ['GYP_MSVS_VERSION'] = config.substs['MSVS_VERSION']
params = {
b'parallel': False,
b'generator_flags': {},
b'build_files': [path],
b'root_targets': None,
}
if gyp_dir_attrs.no_chromium:
includes = []
depth = mozpath.dirname(path)
else:
depth = chrome_src
# Files that gyp_chromium always includes
includes = [encode(mozpath.join(script_dir, 'common.gypi'))]
finder = FileFinder(chrome_src)
includes.extend(encode(mozpath.join(chrome_src, name))
for name, _ in finder.find('*/supplement.gypi'))
str_vars = dict((name, encode(value)) for name, value in
gyp_dir_attrs.variables.items())
self._gyp_loader_future = executor.submit(load_gyp, [path], b'mozbuild',
str_vars, includes,
encode(depth), params)
def test_dirname(self):
self.assertEqual(dirname("foo/bar/baz"), "foo/bar")
self.assertEqual(dirname("foo/bar"), "foo")
self.assertEqual(dirname("foo"), "")
self.assertEqual(dirname("foo/bar/"), "foo/bar")
def _process_test_manifest(self, context, info, manifest_path):
flavor, install_root, install_subdir, package_tests = info
manifest_path = mozpath.normpath(manifest_path)
path = mozpath.normpath(mozpath.join(context.srcdir, manifest_path))
manifest_dir = mozpath.dirname(path)
manifest_reldir = mozpath.dirname(mozpath.relpath(path,
context.config.topsrcdir))
install_prefix = mozpath.join(install_root, install_subdir)
try:
m = manifestparser.TestManifest(manifests=[path], strict=True)
defaults = m.manifest_defaults[os.path.normpath(path)]
if not m.tests and not 'support-files' in defaults:
raise SandboxValidationError('Empty test manifest: %s'
% path, context)
obj = TestManifest(context, path, m, flavor=flavor,
install_prefix=install_prefix,
relpath=mozpath.join(manifest_reldir, mozpath.basename(path)),
dupe_manifest='dupe-manifest' in defaults)
filtered = m.tests
# Jetpack add-on tests are expected to be generated during the
# build process so they won't exist here.
if flavor != 'jetpack-addon':
missing = [t['name'] for t in filtered if not os.path.exists(t['path'])]
if missing:
raise SandboxValidationError('Test manifest (%s) lists '
'test that does not exist: %s' % (
path, ', '.join(missing)), context)
out_dir = mozpath.join(install_prefix, manifest_reldir)
if 'install-to-subdir' in defaults:
# This is terrible, but what are you going to do?
out_dir = mozpath.join(out_dir, defaults['install-to-subdir'])
obj.manifest_obj_relpath = mozpath.join(manifest_reldir,
defaults['install-to-subdir'],
mozpath.basename(path))
# "head" and "tail" lists.
# All manifests support support-files.
#
# Keep a set of already seen support file patterns, because
# repeatedly processing the patterns from the default section
# for every test is quite costly (see bug 922517).
extras = (('head', set()),
('tail', set()),
('support-files', set()))
def process_support_files(test):
for thing, seen in extras:
value = test.get(thing, '')
if value in seen:
continue
seen.add(value)
for pattern in value.split():
# We only support globbing on support-files because
# the harness doesn't support * for head and tail.
if '*' in pattern and thing == 'support-files':
obj.pattern_installs.append(
(manifest_dir, pattern, out_dir))
# "absolute" paths identify files that are to be
# placed in the install_root directory (no globs)
elif pattern[0] == '/':
full = mozpath.normpath(mozpath.join(manifest_dir,
mozpath.basename(pattern)))
obj.installs[full] = (mozpath.join(install_root,
pattern[1:]), False)
else:
full = mozpath.normpath(mozpath.join(manifest_dir,
pattern))
dest_path = mozpath.join(out_dir, pattern)
# If the path resolves to a different directory
# tree, we take special behavior depending on the
# entry type.
if not full.startswith(manifest_dir):
# If it's a support file, we install the file
# into the current destination directory.
# This implementation makes installing things
# with custom prefixes impossible. If this is
# needed, we can add support for that via a
# special syntax later.
if thing == 'support-files':
dest_path = mozpath.join(out_dir,
os.path.basename(pattern))
# If it's not a support file, we ignore it.
# This preserves old behavior so things like
# head files doesn't get installed multiple
# times.
else:
continue
obj.installs[full] = (mozpath.normpath(dest_path),
False)
for test in filtered:
obj.tests.append(test)
# Some test files are compiled and should not be copied into the
# test package. They function as identifiers rather than files.
if package_tests:
obj.installs[mozpath.normpath(test['path'])] = \
(mozpath.join(out_dir, test['relpath']), True)
process_support_files(test)
if not filtered:
# If there are no tests, look for support-files under DEFAULT.
process_support_files(defaults)
# We also copy manifests into the output directory,
# including manifests from [include:foo] directives.
for mpath in m.manifests():
mpath = mozpath.normpath(mpath)
out_path = mozpath.join(out_dir, mozpath.basename(mpath))
obj.installs[mpath] = (out_path, False)
# Some manifests reference files that are auto generated as
# part of the build or shouldn't be installed for some
# reason. Here, we prune those files from the install set.
# FUTURE we should be able to detect autogenerated files from
# other build metadata. Once we do that, we can get rid of this.
for f in defaults.get('generated-files', '').split():
# We re-raise otherwise the stack trace isn't informative.
try:
del obj.installs[mozpath.join(manifest_dir, f)]
except KeyError:
raise SandboxValidationError('Error processing test '
'manifest %s: entry in generated-files not present '
'elsewhere in manifest: %s' % (path, f), context)
obj.external_installs.add(mozpath.join(out_dir, f))
yield obj
except (AssertionError, Exception):
raise SandboxValidationError('Error processing test '
'manifest file %s: %s' % (path,
'\n'.join(traceback.format_exception(*sys.exc_info()))),
context)
def consume(self, objs):
"""Consume a stream of TreeMetadata instances.
This is the main method of the interface. This is what takes the
frontend output and does something with it.
Child classes are not expected to implement this method. Instead, the
base class consumes objects and calls methods (possibly) implemented by
child classes.
"""
# Previously generated files.
list_file = mozpath.join(self.environment.topobjdir,
'backend.%s' % self.__class__.__name__)
backend_output_list = set()
if os.path.exists(list_file):
with open(list_file) as fh:
backend_output_list.update(
mozpath.normsep(p) for p in fh.read().splitlines())
for obj in objs:
obj_start = time.time()
if (not self.consume_object(obj)
and not isinstance(self, PartialBackend)):
raise Exception('Unhandled object of type %s' % type(obj))
self._execution_time += time.time() - obj_start
if (isinstance(obj, ContextDerived)
and not isinstance(self, PartialBackend)):
self.backend_input_files |= obj.context_all_paths
# Pull in all loaded Python as dependencies so any Python changes that
# could influence our output result in a rescan.
self.backend_input_files |= set(
iter_modules_in_path(self.environment.topsrcdir,
self.environment.topobjdir))
finished_start = time.time()
self.consume_finished()
self._execution_time += time.time() - finished_start
# Purge backend files created in previous run, but not created anymore
delete_files = backend_output_list - self._backend_output_files
for path in delete_files:
full_path = mozpath.join(self.environment.topobjdir, path)
try:
with open(full_path, 'r') as existing:
old_content = existing.read()
if old_content:
self.file_diffs[full_path] = simple_diff(
full_path, old_content.splitlines(), None)
except IOError:
pass
try:
if not self.dry_run:
os.unlink(full_path)
self._deleted_count += 1
except OSError:
pass
# Remove now empty directories
for dir in set(mozpath.dirname(d) for d in delete_files):
try:
os.removedirs(dir)
except OSError:
pass
# Write out the list of backend files generated, if it changed.
if self._deleted_count or self._created_count or \
not os.path.exists(list_file):
with self._write_file(list_file) as fh:
fh.write('\n'.join(sorted(self._backend_output_files)))
else:
# Always update its mtime.
with open(list_file, 'a'):
os.utime(list_file, None)
# Write out the list of input files for the backend
with self._write_file('%s.in' % list_file) as fh:
fh.write('\n'.join(
sorted(mozpath.normsep(f) for f in self.backend_input_files)))
_verbosity = options.verbosity
headersdir = options.headersdir
cppdir = options.cppdir
includedirs = [ os.path.abspath(incdir) for incdir in options.includedirs ]
if not len(files):
op.error("No IPDL files specified")
ipcmessagestartpath = os.path.join(headersdir, 'IPCMessageStart.h')
ipc_msgtype_name_path = os.path.join(cppdir, 'IPCMessageTypeName.cpp')
# Compiling the IPDL files can take a long time, even on a fast machine.
# Check to see whether we need to do any work.
latestipdlmod = max(os.stat(f).st_mtime
for f in itertools.chain(files,
iter_modules_in_path(mozpath.dirname(__file__))))
def outputModTime(f):
# A non-existant file is newer than everything.
if not os.path.exists(f):
return 0
return os.stat(f).st_mtime
# Because the IPDL headers are placed into directories reflecting their
# namespace, collect a list here so we can easily map output names without
# parsing the actual IPDL files themselves.
headersmap = {}
for (path, dirs, headers) in os.walk(headersdir):
for h in headers:
base = os.path.basename(h)
if base in headersmap:
def consume_finished(self):
CommonBackend.consume_finished(self)
for objdir, backend_file in sorted(self._backend_files.items()):
srcdir = backend_file.srcdir
with self._write_file(fh=backend_file) as bf:
makefile_in = mozpath.join(srcdir, 'Makefile.in')
makefile = mozpath.join(objdir, 'Makefile')
# If Makefile.in exists, use it as a template. Otherwise,
# create a stub.
stub = not os.path.exists(makefile_in)
if not stub:
self.log(logging.DEBUG, 'substitute_makefile',
{'path': makefile}, 'Substituting makefile: {path}')
self.summary.makefile_in_count += 1
for tier, skiplist in self._may_skip.items():
if tier in ('compile', 'binaries'):
continue
if bf.relobjdir in skiplist:
skiplist.remove(bf.relobjdir)
else:
self.log(logging.DEBUG, 'stub_makefile',
{'path': makefile}, 'Creating stub Makefile: {path}')
# Can't skip directories with a jar.mn for the 'libs' tier.
if bf.relobjdir in self._may_skip['libs'] and \
os.path.exists(mozpath.join(srcdir, 'jar.mn')):
self._may_skip['libs'].remove(bf.relobjdir)
obj = self.Substitution()
obj.output_path = makefile
obj.input_path = makefile_in
obj.topsrcdir = bf.environment.topsrcdir
obj.topobjdir = bf.environment.topobjdir
self._create_makefile(obj, stub=stub)
# Write out a master list of all IPDL source files.
ipdl_dir = mozpath.join(self.environment.topobjdir, 'ipc', 'ipdl')
mk = mozmakeutil.Makefile()
sorted_ipdl_sources = list(sorted(self._ipdl_sources))
mk.add_statement('ALL_IPDLSRCS := %s' % ' '.join(sorted_ipdl_sources))
def files_from(ipdl):
base = mozpath.basename(ipdl)
root, ext = mozpath.splitext(base)
# Both .ipdl and .ipdlh become .cpp files
files = ['%s.cpp' % root]
if ext == '.ipdl':
# .ipdl also becomes Child/Parent.cpp files
files.extend(['%sChild.cpp' % root,
'%sParent.cpp' % root])
return files
ipdl_cppsrcs = list(itertools.chain(*[files_from(p) for p in sorted_ipdl_sources]))
self._add_unified_build_rules(mk, ipdl_cppsrcs, ipdl_dir,
unified_prefix='UnifiedProtocols',
unified_files_makefile_variable='CPPSRCS')
mk.add_statement('IPDLDIRS := %s' % ' '.join(sorted(set(mozpath.dirname(p)
for p in self._ipdl_sources))))
with self._write_file(mozpath.join(ipdl_dir, 'ipdlsrcs.mk')) as ipdls:
mk.dump(ipdls, removal_guard=False)
# These contain autogenerated sources that the build config doesn't
# yet know about.
# TODO Emit GENERATED_SOURCES so these special cases are dealt with
# the proper way.
self._may_skip['compile'] -= {'ipc/ipdl'}
self._may_skip['compile'] -= {'dom/bindings', 'dom/bindings/test'}
self._fill_root_mk()
# Write out a dependency file used to determine whether a config.status
# re-run is needed.
inputs = sorted(p.replace(os.sep, '/') for p in self.backend_input_files)
# We need to use $(DEPTH) so the target here matches what's in
# rules.mk. If they are different, the dependencies don't get pulled in
# properly.
with self._write_file('%s.pp' % self._backend_output_list_file) as backend_deps:
backend_deps.write('$(DEPTH)/backend.%s: %s\n' %
(self.__class__.__name__, ' '.join(inputs)))
for path in inputs:
backend_deps.write('%s:\n' % path)
with open(self._backend_output_list_file, 'a'):
pass
os.utime(self._backend_output_list_file, None)
# Make the master test manifest files.
for flavor, t in self._test_manifests.items():
install_prefix, manifests = t
manifest_stem = mozpath.join(install_prefix, '%s.ini' % flavor)
self._write_master_test_manifest(mozpath.join(
self.environment.topobjdir, '_tests', manifest_stem),
manifests)
# Catch duplicate inserts.
try:
self._install_manifests['tests'].add_optional_exists(manifest_stem)
except ValueError:
pass
self._write_manifests('install', self._install_manifests)
ensureParentDir(mozpath.join(self.environment.topobjdir, 'dist', 'foo'))
import logging
import os
import sys
from collections import defaultdict
from mozbuild.base import MozbuildObject
from mozlint.pathutils import findobject
from mozlint.parser import Parser
from mozlint.result import ResultSummary
from mozlog.structuredlog import StructuredLogger
from mozpack import path
import pytest
here = path.abspath(path.dirname(__file__))
build = MozbuildObject.from_environment(cwd=here,
virtualenv_name="python-test")
lintdir = path.dirname(here)
sys.path.insert(0, lintdir)
logger = logging.getLogger("mozlint")
def pytest_generate_tests(metafunc):
"""Finds, loads and returns the config for the linter name specified by the
LINTER global variable in the calling module.
This implies that each test file (that uses this fixture) should only be
used to test a single linter. If no LINTER variable is defined, the test
will fail.
def test_java_tool_checks_linux(self):
def run_configure_java(mock_fs_paths,
mock_java_home=None,
mock_path=None,
args=[]):
script = textwrap.dedent("""\
@depends('--help')
def host(_):
return namespace(os='unknown')
include('%(topsrcdir)s/build/moz.configure/java.configure')
""" % {"topsrcdir": topsrcdir})
def mock_which(exe, path=None):
for mock_fs_path in mock_fs_paths.keys():
(base, filename) = os.path.split(mock_fs_path)
if filename == exe:
if path and os.path.normpath(base) != os.path.normpath(
path):
continue
return mock_fs_path
# Don't let system JAVA_HOME influence the test
original_java_home = os.environ.pop("JAVA_HOME", None)
configure_environ = {}
if mock_java_home:
os.environ["JAVA_HOME"] = mock_java_home
configure_environ["JAVA_HOME"] = mock_java_home
if mock_path:
configure_environ["PATH"] = mock_path
# * Don't attempt to invoke Java, just resolve each mock Java's version as "1.8"
# * Even if the real file sysphabtem has a symlink at the mocked path, don't let
# realpath follow it, as it may influence the test.
# * When finding a binary, check the mock paths rather than the real filesystem.
# Note: Python doesn't allow the different "with" bits to be put in parenthesis,
# because then it thinks it's an un-with-able tuple. Additionally, if this is cleanly
# lined up with "\", black removes them and autoformats them to the block that is
# below.
with patch("mozboot.util._resolve_java_version",
return_value="1.8"), patch(
"os.path.realpath",
side_effect=lambda path: path), patch(
"mozboot.util.which", side_effect=mock_which):
result = self.get_result(
args=args,
command=script,
extra_paths=paths,
environ=configure_environ,
)
if original_java_home:
os.environ["JAVA_HOME"] = original_java_home
return result
java = mozpath.abspath("/usr/bin/java")
javac = mozpath.abspath("/usr/bin/javac")
paths = {java: None, javac: None}
config, out, status = run_configure_java(paths)
self.assertEqual(status, 0)
self.assertEqual(
config,
{
"JAVA": java,
"MOZ_JAVA_CODE_COVERAGE": False,
},
)
self.assertEqual(
out,
textwrap.dedent("""\
checking for java... %s
""" % java),
)
# An alternative valid set of tools referred to by JAVA_HOME.
alt_java = mozpath.abspath("/usr/local/bin/java")
alt_javac = mozpath.abspath("/usr/local/bin/javac")
alt_java_home = mozpath.dirname(mozpath.dirname(alt_java))
paths = {alt_java: None, alt_javac: None, java: None, javac: None}
alt_path = mozpath.dirname(java)
config, out, status = run_configure_java(paths, alt_java_home,
alt_path)
self.assertEqual(status, 0)
self.assertEqual(
config,
{
"JAVA": alt_java,
"MOZ_JAVA_CODE_COVERAGE": False,
},
)
self.assertEqual(
out,
textwrap.dedent("""\
checking for java... %s
""" % alt_java),
)
# We can use --with-java-bin-path instead of JAVA_HOME to similar
# effect.
config, out, status = run_configure_java(
paths,
mock_path=mozpath.dirname(java),
args=["--with-java-bin-path=%s" % mozpath.dirname(alt_java)],
)
self.assertEqual(status, 0)
self.assertEqual(
config,
{
"JAVA": alt_java,
"MOZ_JAVA_CODE_COVERAGE": False,
},
)
self.assertEqual(
out,
textwrap.dedent("""\
checking for java... %s
""" % alt_java),
)
# If --with-java-bin-path and JAVA_HOME are both set,
# --with-java-bin-path takes precedence.
config, out, status = run_configure_java(
paths,
mock_java_home=mozpath.dirname(mozpath.dirname(java)),
mock_path=mozpath.dirname(java),
args=["--with-java-bin-path=%s" % mozpath.dirname(alt_java)],
)
self.assertEqual(status, 0)
self.assertEqual(
config,
{
"JAVA": alt_java,
"MOZ_JAVA_CODE_COVERAGE": False,
},
)
self.assertEqual(
out,
textwrap.dedent("""\
checking for java... %s
""" % alt_java),
)
# --enable-java-coverage should set MOZ_JAVA_CODE_COVERAGE.
alt_java_home = mozpath.dirname(mozpath.dirname(java))
config, out, status = run_configure_java(
paths,
mock_java_home=alt_java_home,
mock_path=mozpath.dirname(java),
args=["--enable-java-coverage"],
)
self.assertEqual(status, 0)
self.assertEqual(
config,
{
"JAVA": java,
"MOZ_JAVA_CODE_COVERAGE": True,
},
)
# Any missing tool is fatal when these checks run.
paths = {}
config, out, status = run_configure_java(
mock_fs_paths={},
mock_path=mozpath.dirname(java),
args=["--enable-java-coverage"],
)
self.assertEqual(status, 1)
self.assertEqual(config, {})
self.assertEqual(
out,
textwrap.dedent("""\
ERROR: Could not find "java" on the $PATH. Please install the Java 1.8 JDK \
and/or set $JAVA_HOME.
"""),
)
CommandProvider,
Command,
)
from mach.registrar import (Registrar)
from shutil import rmtree
from subprocess import Popen
from sys import argv
from sys import exit
from tempfile import mkdtemp
DEFAULT_PORT = 8080
DEFAULT_HOSTNAME = 'localhost'
SRCDIR = mozpath.abspath(mozpath.dirname(__file__))
STORAGE_SERVER_SCRIPT = mozpath.join(SRCDIR, 'run_storage_server.js')
BAGHEERA_SERVER_SCRIPT = mozpath.join(SRCDIR, 'run_bagheera_server.js')
def SyncStorageCommand(func):
"""Decorator that adds shared command arguments to services commands."""
port = CommandArgument('--port',
metavar='PORT',
type=int,
default=DEFAULT_PORT,
help='Port to run server on.')
func = port(func)
def _link_libraries(self, context, obj, variable):
"""Add linkage declarations to a given object."""
assert isinstance(obj, Linkable)
for path in context.get(variable, []):
force_static = path.startswith('static:') and obj.KIND == 'target'
if force_static:
path = path[7:]
name = mozpath.basename(path)
dir = mozpath.dirname(path)
candidates = [l for l in self._libs[name] if l.KIND == obj.KIND]
if dir:
if dir.startswith('/'):
dir = mozpath.normpath(
mozpath.join(obj.topobjdir, dir[1:]))
else:
dir = mozpath.normpath(
mozpath.join(obj.objdir, dir))
dir = mozpath.relpath(dir, obj.topobjdir)
candidates = [l for l in candidates if l.relobjdir == dir]
if not candidates:
# If the given directory is under one of the external
# (third party) paths, use a fake library reference to
# there.
for d in self._external_paths:
if dir.startswith('%s/' % d):
candidates = [self._get_external_library(dir, name,
force_static)]
break
if not candidates:
raise SandboxValidationError(
'%s contains "%s", but there is no "%s" %s in %s.'
% (variable, path, name,
self.LIBRARY_NAME_VAR[obj.KIND], dir), context)
if len(candidates) > 1:
# If there's more than one remaining candidate, it could be
# that there are instances for the same library, in static and
# shared form.
libs = {}
for l in candidates:
key = mozpath.join(l.relobjdir, l.basename)
if force_static:
if isinstance(l, StaticLibrary):
libs[key] = l
else:
if key in libs and isinstance(l, SharedLibrary):
libs[key] = l
if key not in libs:
libs[key] = l
candidates = libs.values()
if force_static and not candidates:
if dir:
raise SandboxValidationError(
'%s contains "static:%s", but there is no static '
'"%s" %s in %s.' % (variable, path, name,
self.LIBRARY_NAME_VAR[obj.KIND], dir), context)
raise SandboxValidationError(
'%s contains "static:%s", but there is no static "%s" '
'%s in the tree' % (variable, name, name,
self.LIBRARY_NAME_VAR[obj.KIND]), context)
if not candidates:
raise SandboxValidationError(
'%s contains "%s", which does not match any %s in the tree.'
% (variable, path, self.LIBRARY_NAME_VAR[obj.KIND]),
context)
elif len(candidates) > 1:
paths = (mozpath.join(l.relativedir, 'moz.build')
for l in candidates)
raise SandboxValidationError(
'%s contains "%s", which matches a %s defined in multiple '
'places:\n %s' % (variable, path,
self.LIBRARY_NAME_VAR[obj.KIND],
'\n '.join(paths)), context)
elif force_static and not isinstance(candidates[0], StaticLibrary):
raise SandboxValidationError(
'%s contains "static:%s", but there is only a shared "%s" '
'in %s. You may want to add FORCE_STATIC_LIB=True in '
'%s/moz.build, or remove "static:".' % (variable, path,
name, candidates[0].relobjdir, candidates[0].relobjdir),
context)
elif isinstance(obj, StaticLibrary) and isinstance(candidates[0],
SharedLibrary):
self._static_linking_shared.add(obj)
obj.link_library(candidates[0])
# Link system libraries from OS_LIBS/HOST_OS_LIBS.
for lib in context.get(variable.replace('USE', 'OS'), []):
obj.link_system_library(lib)
def _handle_ipdl_sources(self, ipdl_dir, sorted_ipdl_sources,
sorted_nonstatic_ipdl_sources,
sorted_static_ipdl_sources,
unified_ipdl_cppsrcs_mapping):
# Preferably we wouldn't have to import ipdl, but we need to parse the
# ast in order to determine the namespaces since they are used in the
# header output paths.
sys.path.append(mozpath.join(self.environment.topsrcdir, 'ipc',
'ipdl'))
import ipdl
backend_file = self._get_backend_file('ipc/ipdl')
outheaderdir = '_ipdlheaders'
srcdir = mozpath.join(self.environment.topsrcdir, 'ipc/ipdl')
cmd = [
'$(PYTHON_PATH)',
'$(PLY_INCLUDE)',
'%s/ipdl.py' % srcdir,
'--sync-msg-list=%s/sync-messages.ini' % srcdir,
'--msg-metadata=%s/message-metadata.ini' % srcdir,
'--outheaders-dir=%s' % outheaderdir,
'--outcpp-dir=.',
]
ipdldirs = sorted(set(mozpath.dirname(p) for p in sorted_ipdl_sources))
cmd.extend(['-I%s' % d for d in ipdldirs])
cmd.extend(sorted_ipdl_sources)
outputs = [
'IPCMessageTypeName.cpp',
mozpath.join(outheaderdir, 'IPCMessageStart.h'), 'ipdl_lextab.py',
'ipdl_yacctab.py'
]
for filename in sorted_ipdl_sources:
filepath, ext = os.path.splitext(filename)
dirname, basename = os.path.split(filepath)
dirname = mozpath.relpath(dirname, self.environment.topsrcdir)
extensions = ['']
if ext == '.ipdl':
extensions.extend(['Child', 'Parent'])
with open(filename) as f:
ast = ipdl.parse(f.read(), filename, includedirs=ipdldirs)
self.backend_input_files.add(filename)
headerdir = os.path.join(outheaderdir,
*([ns.name for ns in ast.namespaces]))
for extension in extensions:
outputs.append("%s%s.cpp" % (basename, extension))
outputs.append(
mozpath.join(headerdir, '%s%s.h' % (basename, extension)))
backend_file.rule(
display='IPDL code generation',
cmd=cmd,
outputs=outputs,
extra_outputs=[self._installed_files],
check_unchanged=True,
)
backend_file.sources['.cpp'].extend(
u[0] for u in unified_ipdl_cppsrcs_mapping)
def _consume_jar_manifest(self, obj):
# Ideally, this would all be handled somehow in the emitter, but
# this would require all the magic surrounding l10n and addons in
# the recursive make backend to die, which is not going to happen
# any time soon enough.
# Notably missing:
# - DEFINES from config/config.mk
# - The equivalent of -e when USE_EXTENSION_MANIFEST is set in
# moz.build, but it doesn't matter in dist/bin.
pp = Preprocessor()
if obj.defines:
pp.context.update(obj.defines.defines)
pp.context.update(self.environment.defines)
ab_cd = obj.config.substs['MOZ_UI_LOCALE'][0]
pp.context.update(AB_CD=ab_cd, )
pp.out = JarManifestParser()
try:
pp.do_include(obj.path.full_path)
except DeprecatedJarManifest as e:
raise DeprecatedJarManifest(
'Parsing error while processing %s: %s' %
(obj.path.full_path, e.message))
self.backend_input_files |= pp.includes
for jarinfo in pp.out:
jar_context = Context(allowed_variables=VARIABLES,
config=obj._context.config)
jar_context.push_source(obj._context.main_path)
jar_context.push_source(obj.path.full_path)
install_target = obj.install_target
if jarinfo.base:
install_target = mozpath.normpath(
mozpath.join(install_target, jarinfo.base))
jar_context['FINAL_TARGET'] = install_target
if obj.defines:
jar_context['DEFINES'] = obj.defines.defines
files = jar_context['FINAL_TARGET_FILES']
files_pp = jar_context['FINAL_TARGET_PP_FILES']
localized_files = jar_context['LOCALIZED_FILES']
localized_files_pp = jar_context['LOCALIZED_PP_FILES']
for e in jarinfo.entries:
if e.is_locale:
if jarinfo.relativesrcdir:
src = '/%s' % jarinfo.relativesrcdir
else:
src = ''
src = mozpath.join(src, 'en-US', e.source)
else:
src = e.source
src = Path(jar_context, src)
if '*' not in e.source and not os.path.exists(src.full_path):
if e.is_locale:
raise Exception('%s: Cannot find %s (tried %s)' %
(obj.path, e.source, src.full_path))
if e.source.startswith('/'):
src = Path(jar_context, '!' + e.source)
else:
# This actually gets awkward if the jar.mn is not
# in the same directory as the moz.build declaring
# it, but it's how it works in the recursive make,
# not that anything relies on that, but it's simpler.
src = Path(obj._context, '!' + e.source)
output_basename = mozpath.basename(e.output)
if output_basename != src.target_basename:
src = RenamedSourcePath(jar_context,
(src, output_basename))
path = mozpath.dirname(mozpath.join(jarinfo.name, e.output))
if e.preprocess:
if '*' in e.source:
raise Exception('%s: Wildcards are not supported with '
'preprocessing' % obj.path)
if e.is_locale:
localized_files_pp[path] += [src]
else:
files_pp[path] += [src]
else:
if e.is_locale:
localized_files[path] += [src]
else:
files[path] += [src]
if files:
self.consume_object(FinalTargetFiles(jar_context, files))
if files_pp:
self.consume_object(
FinalTargetPreprocessedFiles(jar_context, files_pp))
if localized_files:
self.consume_object(
LocalizedFiles(jar_context, localized_files))
if localized_files_pp:
self.consume_object(
LocalizedPreprocessedFiles(jar_context,
localized_files_pp))
for m in jarinfo.chrome_manifests:
entry = parse_manifest_line(
mozpath.dirname(jarinfo.name),
m.replace('%',
mozpath.basename(jarinfo.name) + '/'))
self.consume_object(
ChromeManifestEntry(jar_context,
'%s.manifest' % jarinfo.name, entry))
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
import os
import unittest
from six import StringIO
import mozpack.path as mozpath
from mozbuild.dotproperties import DotProperties
from mozunit import main
test_data_path = mozpath.abspath(mozpath.dirname(__file__))
test_data_path = mozpath.join(test_data_path, "data")
class TestDotProperties(unittest.TestCase):
def test_get(self):
contents = StringIO("""
key=value
""")
p = DotProperties(contents)
self.assertEqual(p.get("missing"), None)
self.assertEqual(p.get("missing", "default"), "default")
self.assertEqual(p.get("key"), "value")
def test_update(self):
contents = StringIO("""
def add_idls(self, idls):
self.idl_files.update(idl.full_path for idl in idls)
self.directories.update(
mozpath.dirname(idl.full_path) for idl in idls)
self._stems.update(
mozpath.splitext(mozpath.basename(idl))[0] for idl in idls)
def test_check_prog_with_path(self):
config, out, status = self.get_result(
'check_prog("A", ("known-a",), paths=["/some/path"])')
self.assertEqual(status, 1)
self.assertEqual(config, {})
self.assertEqual(
out,
textwrap.dedent('''\
checking for a... not found
DEBUG: a: Trying known-a
ERROR: Cannot find a
'''))
config, out, status = self.get_result(
'check_prog("A", ("known-a",), paths=["%s"])' %
os.path.dirname(self.OTHER_A))
self.assertEqual(status, 0)
self.assertEqual(config, {'A': self.OTHER_A})
self.assertEqual(
out,
textwrap.dedent('''\
checking for a... %s
''' % self.OTHER_A))
dirs = map(mozpath.dirname, (self.OTHER_A, self.KNOWN_A))
config, out, status = self.get_result(
textwrap.dedent('''\
check_prog("A", ("known-a",), paths=["%s"])
''' % os.pathsep.join(dirs)))
self.assertEqual(status, 0)
self.assertEqual(config, {'A': self.OTHER_A})
self.assertEqual(
out,
textwrap.dedent('''\
checking for a... %s
''' % self.OTHER_A))
dirs = map(mozpath.dirname, (self.KNOWN_A, self.KNOWN_B))
config, out, status = self.get_result(
textwrap.dedent('''\
check_prog("A", ("known-a",), paths=["%s", "%s"])
''' % (os.pathsep.join(dirs), self.OTHER_A)))
self.assertEqual(status, 0)
self.assertEqual(config, {'A': self.KNOWN_A})
self.assertEqual(
out,
textwrap.dedent('''\
checking for a... %s
''' % self.KNOWN_A))
config, out, status = self.get_result(
'check_prog("A", ("known-a",), paths="%s")' %
os.path.dirname(self.OTHER_A))
self.assertEqual(status, 1)
self.assertEqual(config, {})
self.assertEqual(
out,
textwrap.dedent('''\
checking for a...
DEBUG: a: Trying known-a
ERROR: Paths provided to find_program must be a list of strings, not %r
''' % mozpath.dirname(self.OTHER_A)))
def _read_mozbuild(self, path, config, read_tiers, descend, metadata):
path = mozpath.normpath(path)
log(self._log, logging.DEBUG, 'read_mozbuild', {'path': path},
'Reading file: {path}')
if path in self._read_files:
log(self._log, logging.WARNING, 'read_already', {'path': path},
'File already read. Skipping: {path}')
return
self._read_files.add(path)
time_start = time.time()
topobjdir = config.topobjdir
if not mozpath.basedir(path, [config.topsrcdir]):
external = config.external_source_dir
if external and mozpath.basedir(path, [external]):
config = ConfigEnvironment.from_config_status(
mozpath.join(topobjdir, 'config.status'))
config.topsrcdir = external
config.external_source_dir = None
relpath = mozpath.relpath(path, config.topsrcdir)
reldir = mozpath.dirname(relpath)
if mozpath.dirname(relpath) == 'js/src' and \
not config.substs.get('JS_STANDALONE'):
config = ConfigEnvironment.from_config_status(
mozpath.join(topobjdir, reldir, 'config.status'))
config.topobjdir = topobjdir
config.external_source_dir = None
context = Context(VARIABLES, config)
sandbox = MozbuildSandbox(context, metadata=metadata)
sandbox.exec_file(path)
context.execution_time = time.time() - time_start
if self._sandbox_post_eval_cb:
self._sandbox_post_eval_cb(context)
# We first collect directories populated in variables.
dir_vars = ['DIRS']
if context.config.substs.get('ENABLE_TESTS', False) == '1':
dir_vars.append('TEST_DIRS')
dirs = [(v, context[v]) for v in dir_vars if v in context]
curdir = mozpath.dirname(path)
gyp_contexts = []
for target_dir in context['GYP_DIRS']:
gyp_dir = context['GYP_DIRS'][target_dir]
for v in ('input', 'variables'):
if not getattr(gyp_dir, v):
raise SandboxValidationError(
'Missing value for '
'GYP_DIRS["%s"].%s' % (target_dir, v), context)
# The make backend assumes contexts for sub-directories are
# emitted after their parent, so accumulate the gyp contexts.
# We could emit the parent context before processing gyp
# configuration, but we need to add the gyp objdirs to that context
# first.
from .gyp_reader import read_from_gyp
non_unified_sources = set()
for s in gyp_dir.non_unified_sources:
source = mozpath.normpath(mozpath.join(curdir, s))
if not os.path.exists(source):
raise SandboxValidationError('Cannot find %s.' % source,
context)
non_unified_sources.add(source)
for gyp_context in read_from_gyp(
context.config,
mozpath.join(curdir, gyp_dir.input),
mozpath.join(context.objdir, target_dir),
gyp_dir.variables,
non_unified_sources=non_unified_sources):
gyp_context.update(gyp_dir.sandbox_vars)
gyp_contexts.append(gyp_context)
for gyp_context in gyp_contexts:
if self._sandbox_post_eval_cb:
self._sandbox_post_eval_cb(gyp_context)
context['DIRS'].append(
mozpath.relpath(gyp_context.objdir, context.objdir))
yield context
for gyp_context in gyp_contexts:
yield gyp_context
# Traverse into referenced files.
# It's very tempting to use a set here. Unfortunately, the recursive
# make backend needs order preserved. Once we autogenerate all backend
# files, we should be able to convert this to a set.
recurse_info = OrderedDict()
for var, var_dirs in dirs:
for d in var_dirs:
if d in recurse_info:
raise SandboxValidationError(
'Directory (%s) registered multiple times in %s' %
(mozpath.relpath(d, context.srcdir), var), context)
recurse_info[d] = {}
if 'templates' in sandbox.metadata:
recurse_info[d]['templates'] = dict(
sandbox.metadata['templates'])
if 'exports' in sandbox.metadata:
sandbox.recompute_exports()
recurse_info[d]['exports'] = dict(
sandbox.metadata['exports'])
for path, child_metadata in recurse_info.items():
child_path = path.join('moz.build')
# Ensure we don't break out of the topsrcdir. We don't do realpath
# because it isn't necessary. If there are symlinks in the srcdir,
# that's not our problem. We're not a hosted application: we don't
# need to worry about security too much.
if not is_read_allowed(child_path, context.config):
raise SandboxValidationError(
'Attempting to process file outside of allowed paths: %s' %
child_path, context)
if not descend:
continue
for res in self.read_mozbuild(child_path,
context.config,
read_tiers=False,
metadata=child_metadata):
yield res
self._execution_stack.pop()
def parents(p):
while True:
p = mozpath.dirname(p)
if not p:
break
yield p
def read_mozconfig(self, path=None):
"""Read the contents of a mozconfig into a data structure.
This takes the path to a mozconfig to load. If the given path is
AUTODETECT, will try to find a mozconfig from the environment using
find_mozconfig().
mozconfig files are shell scripts. So, we can't just parse them.
Instead, we run the shell script in a wrapper which allows us to record
state from execution. Thus, the output from a mozconfig is a friendly
static data structure.
"""
if path is self.AUTODETECT:
path = find_mozconfig(self.topsrcdir)
result = {
"path": path,
"topobjdir": None,
"configure_args": None,
"make_flags": None,
"make_extra": None,
"env": None,
"vars": None,
}
if path is None:
if "MOZ_OBJDIR" in os.environ:
result["topobjdir"] = os.environ["MOZ_OBJDIR"]
return result
path = mozpath.normsep(path)
result["configure_args"] = []
result["make_extra"] = []
result["make_flags"] = []
# Since mozconfig_loader is a shell script, running it "normally"
# actually leads to two shell executions on Windows. Avoid this by
# directly calling sh mozconfig_loader.
shell = "sh"
if "MOZILLABUILD" in os.environ:
shell = os.environ["MOZILLABUILD"] + "/msys/bin/sh"
if sys.platform == "win32":
shell = shell + ".exe"
command = [
shell,
mozpath.normsep(self._loader_script),
mozpath.normsep(self.topsrcdir),
path,
sys.executable,
mozpath.join(mozpath.dirname(self._loader_script), "action", "dump_env.py"),
]
try:
env = dict(os.environ)
env["PYTHONIOENCODING"] = "utf-8"
# We need to capture stderr because that's where the shell sends
# errors if execution fails.
output = six.ensure_text(
subprocess.check_output(
command,
stderr=subprocess.STDOUT,
cwd=self.topsrcdir,
env=ensure_subprocess_env(env),
universal_newlines=True,
)
)
except subprocess.CalledProcessError as e:
lines = e.output.splitlines()
# Output before actual execution shouldn't be relevant.
try:
index = lines.index("------END_BEFORE_SOURCE")
lines = lines[index + 1 :]
except ValueError:
pass
raise MozconfigLoadException(path, MOZCONFIG_BAD_EXIT_CODE, lines)
try:
parsed = self._parse_loader_output(output)
except AssertionError:
# _parse_loader_output uses assertions to verify the
# well-formedness of the shell output; when these fail, it
# generally means there was a problem with the output, but we
# include the assertion traceback just to be sure.
print("Assertion failed in _parse_loader_output:")
traceback.print_exc()
raise MozconfigLoadException(
path, MOZCONFIG_BAD_OUTPUT, output.splitlines()
)
def diff_vars(vars_before, vars_after):
set1 = set(vars_before.keys()) - self.IGNORE_SHELL_VARIABLES
set2 = set(vars_after.keys()) - self.IGNORE_SHELL_VARIABLES
added = set2 - set1
removed = set1 - set2
maybe_modified = set1 & set2
changed = {
"added": {},
"removed": {},
"modified": {},
"unmodified": {},
}
for key in added:
changed["added"][key] = vars_after[key]
for key in removed:
changed["removed"][key] = vars_before[key]
for key in maybe_modified:
if vars_before[key] != vars_after[key]:
changed["modified"][key] = (vars_before[key], vars_after[key])
elif key in self.ENVIRONMENT_VARIABLES:
# In order for irrelevant environment variable changes not
# to incur in re-running configure, only a set of
# environment variables are stored when they are
# unmodified. Otherwise, changes such as using a different
# terminal window, or even rebooting, would trigger
# reconfigures.
changed["unmodified"][key] = vars_after[key]
return changed
result["env"] = diff_vars(parsed["env_before"], parsed["env_after"])
# Environment variables also appear as shell variables, but that's
# uninteresting duplication of information. Filter them out.
def filt(x, y):
return {k: v for k, v in x.items() if k not in y}
result["vars"] = diff_vars(
filt(parsed["vars_before"], parsed["env_before"]),
filt(parsed["vars_after"], parsed["env_after"]),
)
result["configure_args"] = [self._expand(o) for o in parsed["ac"]]
if "MOZ_OBJDIR" in parsed["env_before"]:
result["topobjdir"] = parsed["env_before"]["MOZ_OBJDIR"]
mk = [self._expand(o) for o in parsed["mk"]]
for o in mk:
match = self.RE_MAKE_VARIABLE.match(o)
if match is None:
result["make_extra"].append(o)
continue
name, value = match.group("var"), match.group("value")
if name == "MOZ_MAKE_FLAGS":
result["make_flags"] = value.split()
continue
if name == "MOZ_OBJDIR":
result["topobjdir"] = value
if parsed["env_before"].get("MOZ_PROFILE_GENERATE") == "1":
# If MOZ_OBJDIR is specified in the mozconfig, we need to
# make sure that the '/instrumented' directory gets appended
# for the first build to avoid an objdir mismatch when
# running 'mach package' on Windows.
result["topobjdir"] = mozpath.join(
result["topobjdir"], "instrumented"
)
continue
result["make_extra"].append(o)
return result
def add_interfaces(self, path, content):
self.copier.add(path, content)
self.add_manifest(ManifestInterfaces(mozpath.dirname(path),
mozpath.basename(path)))
def use_fetches(config, jobs):
artifact_names = {}
aliases = {}
if config.kind in ("toolchain", "fetch"):
jobs = list(jobs)
for job in jobs:
run = job.get("run", {})
label = job["label"]
get_attribute(artifact_names, label, run, "toolchain-artifact")
value = run.get(f"{config.kind}-alias")
if not value:
value = []
elif isinstance(value, str):
value = [value]
for alias in value:
aliases[f"{config.kind}-{alias}"] = label
for task in config.kind_dependencies_tasks.values():
if task.kind in ("fetch", "toolchain"):
get_attribute(
artifact_names,
task.label,
task.attributes,
f"{task.kind}-artifact",
)
value = task.attributes.get(f"{task.kind}-alias")
if not value:
value = []
elif isinstance(value, str):
value = [value]
for alias in value:
aliases[f"{task.kind}-{alias}"] = task.label
artifact_prefixes = {}
for job in order_tasks(config, jobs):
artifact_prefixes[job["label"]] = get_artifact_prefix(job)
fetches = job.pop("fetches", None)
if not fetches:
yield job
continue
job_fetches = []
name = job.get("name", job.get("label"))
dependencies = job.setdefault("dependencies", {})
worker = job.setdefault("worker", {})
prefix = get_artifact_prefix(job)
has_sccache = False
for kind, artifacts in fetches.items():
if kind in ("fetch", "toolchain"):
for fetch_name in artifacts:
label = f"{kind}-{fetch_name}"
label = aliases.get(label, label)
if label not in artifact_names:
raise Exception(
"Missing fetch job for {kind}-{name}: {fetch}".
format(kind=config.kind,
name=name,
fetch=fetch_name))
path = artifact_names[label]
dependencies[label] = label
job_fetches.append({
"artifact": path,
"task": f"<{label}>",
"extract": True,
})
if kind == "toolchain" and fetch_name.endswith("-sccache"):
has_sccache = True
else:
if kind not in dependencies:
raise Exception(
"{name} can't fetch {kind} artifacts because "
"it has no {kind} dependencies!".format(name=name,
kind=kind))
dep_label = dependencies[kind]
if dep_label in artifact_prefixes:
prefix = artifact_prefixes[dep_label]
else:
if dep_label not in config.kind_dependencies_tasks:
raise Exception(
"{name} can't fetch {kind} artifacts because "
"there are no tasks with label {label} in kind dependencies!"
.format(
name=name,
kind=kind,
label=dependencies[kind],
))
prefix = get_artifact_prefix(
config.kind_dependencies_tasks[dep_label])
for artifact in artifacts:
if isinstance(artifact, str):
path = artifact
dest = None
extract = True
verify_hash = False
else:
path = artifact["artifact"]
dest = artifact.get("dest")
extract = artifact.get("extract", True)
verify_hash = artifact.get("verify-hash", False)
fetch = {
"artifact":
f"{prefix}/{path}"
if not path.startswith("/") else path[1:],
"task":
f"<{kind}>",
"extract":
extract,
}
if dest is not None:
fetch["dest"] = dest
if verify_hash:
fetch["verify-hash"] = verify_hash
job_fetches.append(fetch)
if job.get("use-sccache") and not has_sccache:
raise Exception(
"Must provide an sccache toolchain if using sccache.")
job_artifact_prefixes = {
mozpath.dirname(fetch["artifact"])
for fetch in job_fetches
if not fetch["artifact"].startswith("public/")
}
if job_artifact_prefixes:
# Use taskcluster-proxy and request appropriate scope. For example, add
# 'scopes: [queue:get-artifact:path/to/*]' for 'path/to/artifact.tar.xz'.
worker["taskcluster-proxy"] = True
for prefix in sorted(job_artifact_prefixes):
scope = f"queue:get-artifact:{prefix}/*"
if scope not in job.setdefault("scopes", []):
job["scopes"].append(scope)
artifacts = {}
for f in job_fetches:
_, __, artifact = f["artifact"].rpartition("/")
if "dest" in f:
artifact = f"{f['dest']}/{artifact}"
task = f["task"][1:-1]
if artifact in artifacts:
raise Exception(
f"Task {name} depends on {artifacts[artifact]} and {task} "
f"that both provide {artifact}")
artifacts[artifact] = task
env = worker.setdefault("env", {})
env["MOZ_FETCHES"] = {
"task-reference":
json.dumps(sorted(job_fetches, key=lambda x: sorted(x.items())),
sort_keys=True)
}
# The path is normalized to an absolute path in run-task
env.setdefault("MOZ_FETCHES_DIR", "fetches")
yield job
def read_from_gyp(config, path, output, vars, non_unified_sources=set()):
"""Read a gyp configuration and emits GypContexts for the backend to
process.
config is a ConfigEnvironment, path is the path to a root gyp configuration
file, output is the base path under which the objdir for the various gyp
dependencies will be, and vars a dict of variables to pass to the gyp
processor.
"""
time_start = time.time()
all_sources = set()
# gyp expects plain str instead of unicode. The frontend code gives us
# unicode strings, so convert them.
path = encode(path)
str_vars = dict((name, encode(value)) for name, value in vars.items())
params = {
b'parallel': False,
b'generator_flags': {},
b'build_files': [path],
}
# Files that gyp_chromium always includes
includes = [encode(mozpath.join(script_dir, 'common.gypi'))]
finder = FileFinder(chrome_src, find_executables=False)
includes.extend(
encode(mozpath.join(chrome_src, name))
for name, _ in finder.find('*/supplement.gypi'))
# Read the given gyp file and its dependencies.
generator, flat_list, targets, data = \
gyp.Load([path], format=b'mozbuild',
default_variables=str_vars,
includes=includes,
depth=encode(mozpath.dirname(path)),
params=params)
# Process all targets from the given gyp files and its dependencies.
# The path given to AllTargets needs to use os.sep, while the frontend code
# gives us paths normalized with forward slash separator.
for target in gyp.common.AllTargets(flat_list, targets,
path.replace(b'/', os.sep)):
build_file, target_name, toolset = gyp.common.ParseQualifiedTarget(
target)
# Each target is given its own objdir. The base of that objdir
# is derived from the relative path from the root gyp file path
# to the current build_file, placed under the given output
# directory. Since several targets can be in a given build_file,
# separate them in subdirectories using the build_file basename
# and the target_name.
reldir = mozpath.relpath(mozpath.dirname(build_file),
mozpath.dirname(path))
subdir = '%s_%s' % (
mozpath.splitext(mozpath.basename(build_file))[0],
target_name,
)
# Emit a context for each target.
context = GypContext(
config,
mozpath.relpath(mozpath.join(output, reldir, subdir),
config.topobjdir))
context.add_source(mozpath.abspath(build_file))
# The list of included files returned by gyp are relative to build_file
for f in data[build_file]['included_files']:
context.add_source(
mozpath.abspath(mozpath.join(mozpath.dirname(build_file), f)))
context['IS_GYP_DIR'] = True
spec = targets[target]
# Derive which gyp configuration to use based on MOZ_DEBUG.
c = 'Debug' if config.substs['MOZ_DEBUG'] else 'Release'
if c not in spec['configurations']:
raise RuntimeError('Missing %s gyp configuration for target %s '
'in %s' % (c, target_name, build_file))
target_conf = spec['configurations'][c]
if spec['type'] == 'none':
continue
elif spec['type'] == 'static_library':
# Remove leading 'lib' from the target_name if any, and use as
# library name.
name = spec['target_name']
if name.startswith('lib'):
name = name[3:]
# The context expects an unicode string.
context['LIBRARY_NAME'] = name.decode('utf-8')
# gyp files contain headers and asm sources in sources lists.
sources = set(
mozpath.normpath(mozpath.join(context.srcdir, f))
for f in spec.get('sources', [])
if mozpath.splitext(f)[-1] != '.h')
asm_sources = set(f for f in sources if f.endswith('.S'))
unified_sources = sources - non_unified_sources - asm_sources
sources -= unified_sources
all_sources |= sources
# The context expects alphabetical order when adding sources
context['SOURCES'] = alphabetical_sorted(sources)
context['UNIFIED_SOURCES'] = alphabetical_sorted(unified_sources)
for define in target_conf.get('defines', []):
if '=' in define:
name, value = define.split('=', 1)
context['DEFINES'][name] = value
else:
context['DEFINES'][define] = True
for include in target_conf.get('include_dirs', []):
# moz.build expects all LOCAL_INCLUDES to exist, so ensure they do.
#
# NB: gyp files sometimes have actual absolute paths (e.g.
# /usr/include32) and sometimes paths that moz.build considers
# absolute, i.e. starting from topsrcdir. There's no good way
# to tell them apart here, and the actual absolute paths are
# likely bogus. In any event, actual absolute paths will be
# filtered out by trying to find them in topsrcdir.
if include.startswith('/'):
resolved = mozpath.abspath(
mozpath.join(config.topsrcdir, include[1:]))
else:
resolved = mozpath.abspath(
mozpath.join(mozpath.dirname(build_file), include))
if not os.path.exists(resolved):
continue
context['LOCAL_INCLUDES'] += [include]
context['EXTRA_ASSEMBLER_FLAGS'] = target_conf.get(
'asflags_mozilla', [])
context['EXTRA_COMPILE_FLAGS'] = target_conf.get(
'cflags_mozilla', [])
else:
# Ignore other types than static_library because we don't have
# anything using them, and we're not testing them. They can be
# added when that becomes necessary.
raise NotImplementedError('Unsupported gyp target type: %s' %
spec['type'])
context.execution_time = time.time() - time_start
yield context
time_start = time.time()
def test_check_prog_with_path(self):
config, out, status = self.get_result(
'check_prog("A", ("known-a",), paths=["/some/path"])')
self.assertEqual(status, 1)
self.assertEqual(config, {})
self.assertEqual(
out,
textwrap.dedent("""\
checking for a... not found
DEBUG: a: Looking for known-a
ERROR: Cannot find a
"""),
)
config, out, status = self.get_result(
'check_prog("A", ("known-a",), paths=["%s"])' %
os.path.dirname(self.OTHER_A))
self.assertEqual(status, 0)
self.assertEqual(config, {"A": self.OTHER_A})
self.assertEqual(
out,
textwrap.dedent("""\
checking for a... %s
""" % self.OTHER_A),
)
dirs = map(mozpath.dirname, (self.OTHER_A, self.KNOWN_A))
config, out, status = self.get_result(
textwrap.dedent("""\
check_prog("A", ("known-a",), paths=["%s"])
""" % os.pathsep.join(dirs)))
self.assertEqual(status, 0)
self.assertEqual(config, {"A": self.OTHER_A})
self.assertEqual(
out,
textwrap.dedent("""\
checking for a... %s
""" % self.OTHER_A),
)
dirs = map(mozpath.dirname, (self.KNOWN_A, self.KNOWN_B))
config, out, status = self.get_result(
textwrap.dedent("""\
check_prog("A", ("known-a",), paths=["%s", "%s"])
""" % (os.pathsep.join(dirs), self.OTHER_A)))
self.assertEqual(status, 0)
self.assertEqual(config, {"A": self.KNOWN_A})
self.assertEqual(
out,
textwrap.dedent("""\
checking for a... %s
""" % self.KNOWN_A),
)
config, out, status = self.get_result(
'check_prog("A", ("known-a",), paths="%s")' %
os.path.dirname(self.OTHER_A))
self.assertEqual(status, 1)
self.assertEqual(config, {})
self.assertEqual(
out,
textwrap.dedent("""\
checking for a... """
# noqa # trailing whitespace...
"""
DEBUG: a: Looking for known-a
ERROR: Paths provided to find_program must be a list of strings, not %r
""" % mozpath.dirname(self.OTHER_A)),
)
def _process_final_target_files(self, obj):
target = obj.install_target
if not isinstance(obj, ObjdirFiles):
path = mozpath.basedir(target, (
'dist/bin',
'dist/xpi-stage',
'_tests',
'dist/include',
'dist/sdk',
))
if not path:
raise Exception("Cannot install to " + target)
for path, files in obj.files.walk():
self._add_features(target, path)
for f in files:
output_group = None
if any(mozpath.match(mozpath.basename(f), p)
for p in self._compile_env_files):
output_group = self._installed_files
if not isinstance(f, ObjDirPath):
backend_file = self._get_backend_file(mozpath.join(target, path))
if '*' in f:
if f.startswith('/') or isinstance(f, AbsolutePath):
basepath, wild = os.path.split(f.full_path)
if '*' in basepath:
raise Exception("Wildcards are only supported in the filename part of "
"srcdir-relative or absolute paths.")
# TODO: This is only needed for Windows, so we can
# skip this for now.
pass
else:
def _prefix(s):
for p in mozpath.split(s):
if '*' not in p:
yield p + '/'
prefix = ''.join(_prefix(f.full_path))
self.backend_input_files.add(prefix)
output_dir = ''
# If we have a RenamedSourcePath here, the common backend
# has generated this object from a jar manifest, and we
# can rely on 'path' to be our destination path relative
# to any wildcard match. Otherwise, the output file may
# contribute to our destination directory.
if not isinstance(f, RenamedSourcePath):
output_dir = ''.join(_prefix(mozpath.dirname(f)))
finder = FileFinder(prefix)
for p, _ in finder.find(f.full_path[len(prefix):]):
install_dir = prefix[len(obj.srcdir) + 1:]
output = p
if f.target_basename and '*' not in f.target_basename:
output = mozpath.join(f.target_basename, output)
backend_file.symlink_rule(mozpath.join(prefix, p),
output=mozpath.join(output_dir, output),
output_group=output_group)
else:
backend_file.symlink_rule(f.full_path, output=f.target_basename, output_group=output_group)
else:
if (self.environment.is_artifact_build and
any(mozpath.match(f.target_basename, p) for p in self._compile_env_gen_files)):
# If we have an artifact build we never would have generated this file,
# so do not attempt to install it.
continue
output = mozpath.join('$(MOZ_OBJ_ROOT)', target, path,
f.target_basename)
gen_backend_file = self._get_backend_file(f.context.relobjdir)
if gen_backend_file.requires_delay([f]):
gen_backend_file.delayed_installed_files.append((f.full_path, output, output_group))
else:
gen_backend_file.symlink_rule(f.full_path, output=output,
output_group=output_group)
def read_mozconfig(self, path=None):
"""Read the contents of a mozconfig into a data structure.
This takes the path to a mozconfig to load. If the given path is
AUTODETECT, will try to find a mozconfig from the environment using
find_mozconfig().
mozconfig files are shell scripts. So, we can't just parse them.
Instead, we run the shell script in a wrapper which allows us to record
state from execution. Thus, the output from a mozconfig is a friendly
static data structure.
"""
if path is self.AUTODETECT:
path = self.find_mozconfig()
result = {
'path': path,
'topobjdir': None,
'configure_args': None,
'make_flags': None,
'make_extra': None,
'env': None,
'vars': None,
}
if path is None:
return result
path = mozpath.normsep(path)
result['configure_args'] = []
result['make_extra'] = []
result['make_flags'] = []
env = dict(os.environ)
# Since mozconfig_loader is a shell script, running it "normally"
# actually leads to two shell executions on Windows. Avoid this by
# directly calling sh mozconfig_loader.
shell = 'sh'
if 'MOZILLABUILD' in os.environ:
shell = os.environ['MOZILLABUILD'] + '/msys/bin/sh'
if sys.platform == 'win32':
shell = shell + '.exe'
command = [
shell,
mozpath.normsep(self._loader_script),
mozpath.normsep(self.topsrcdir), path, sys.executable,
mozpath.join(mozpath.dirname(self._loader_script), 'action',
'dump_env.py')
]
try:
# We need to capture stderr because that's where the shell sends
# errors if execution fails.
output = subprocess.check_output(command,
stderr=subprocess.STDOUT,
cwd=self.topsrcdir,
env=env)
except subprocess.CalledProcessError as e:
lines = e.output.splitlines()
# Output before actual execution shouldn't be relevant.
try:
index = lines.index('------END_BEFORE_SOURCE')
lines = lines[index + 1:]
except ValueError:
pass
raise MozconfigLoadException(path, MOZCONFIG_BAD_EXIT_CODE, lines)
try:
parsed = self._parse_loader_output(output)
except AssertionError:
# _parse_loader_output uses assertions to verify the
# well-formedness of the shell output; when these fail, it
# generally means there was a problem with the output, but we
# include the assertion traceback just to be sure.
print('Assertion failed in _parse_loader_output:')
traceback.print_exc()
raise MozconfigLoadException(path, MOZCONFIG_BAD_OUTPUT,
output.splitlines())
def diff_vars(vars_before, vars_after):
set1 = set(vars_before.keys()) - self.IGNORE_SHELL_VARIABLES
set2 = set(vars_after.keys()) - self.IGNORE_SHELL_VARIABLES
added = set2 - set1
removed = set1 - set2
maybe_modified = set1 & set2
changed = {
'added': {},
'removed': {},
'modified': {},
'unmodified': {},
}
for key in added:
changed['added'][key] = vars_after[key]
for key in removed:
changed['removed'][key] = vars_before[key]
for key in maybe_modified:
if vars_before[key] != vars_after[key]:
changed['modified'][key] = (vars_before[key],
vars_after[key])
elif key in self.ENVIRONMENT_VARIABLES:
# In order for irrelevant environment variable changes not
# to incur in re-running configure, only a set of
# environment variables are stored when they are
# unmodified. Otherwise, changes such as using a different
# terminal window, or even rebooting, would trigger
# reconfigures.
changed['unmodified'][key] = vars_after[key]
return changed
result['env'] = diff_vars(parsed['env_before'], parsed['env_after'])
# Environment variables also appear as shell variables, but that's
# uninteresting duplication of information. Filter them out.
def filt(x, y):
return {k: v for k, v in x.items() if k not in y}
result['vars'] = diff_vars(
filt(parsed['vars_before'], parsed['env_before']),
filt(parsed['vars_after'], parsed['env_after']))
result['configure_args'] = [self._expand(o) for o in parsed['ac']]
if 'MOZ_OBJDIR' in parsed['env_before']:
result['topobjdir'] = parsed['env_before']['MOZ_OBJDIR']
mk = [self._expand(o) for o in parsed['mk']]
for o in mk:
match = self.RE_MAKE_VARIABLE.match(o)
if match is None:
result['make_extra'].append(o)
continue
name, value = match.group('var'), match.group('value')
if name == 'MOZ_MAKE_FLAGS':
result['make_flags'] = value.split()
continue
if name == 'MOZ_OBJDIR':
result['topobjdir'] = value
continue
result['make_extra'].append(o)
return result
def use_fetches(config, jobs):
artifact_names = {}
aliases = {}
if config.kind == 'toolchain':
jobs = list(jobs)
for job in jobs:
run = job.get('run', {})
label = 'toolchain-{}'.format(job['name'])
get_attribute(
artifact_names, label, run, 'toolchain-artifact')
value = run.get('toolchain-alias')
if value:
aliases['toolchain-{}'.format(value)] = label
for task in config.kind_dependencies_tasks:
if task.kind in ('fetch', 'toolchain'):
get_attribute(
artifact_names, task.label, task.attributes,
'{kind}-artifact'.format(kind=task.kind),
)
value = task.attributes.get('{}-alias'.format(task.kind))
if value:
aliases['{}-{}'.format(task.kind, value)] = task.label
for job in jobs:
fetches = job.pop('fetches', None)
if not fetches:
yield job
continue
job_fetches = []
name = job.get('name', job.get('label'))
dependencies = job.setdefault('dependencies', {})
worker = job.setdefault('worker', {})
prefix = get_artifact_prefix(job)
for kind, artifacts in fetches.items():
if kind in ('fetch', 'toolchain'):
for fetch_name in artifacts:
label = '{kind}-{name}'.format(kind=kind, name=fetch_name)
label = aliases.get(label, label)
if label not in artifact_names:
raise Exception('Missing fetch job for {kind}-{name}: {fetch}'.format(
kind=config.kind, name=name, fetch=fetch_name))
path = artifact_names[label]
if not path.startswith('public/'):
# Use taskcluster-proxy and request appropriate scope. For example, add
# 'scopes: [queue:get-artifact:path/to/*]' for 'path/to/artifact.tar.xz'.
worker['taskcluster-proxy'] = True
dirname = mozpath.dirname(path)
scope = 'queue:get-artifact:{}/*'.format(dirname)
if scope not in job.setdefault('scopes', []):
job['scopes'].append(scope)
dependencies[label] = label
job_fetches.append({
'artifact': path,
'task': '<{label}>'.format(label=label),
'extract': True,
})
if kind == 'toolchain' and fetch_name.endswith('-sccache'):
job['needs-sccache'] = True
else:
if kind not in dependencies:
raise Exception("{name} can't fetch {kind} artifacts because "
"it has no {kind} dependencies!".format(name=name, kind=kind))
for artifact in artifacts:
if isinstance(artifact, basestring):
path = artifact
dest = None
extract = True
else:
path = artifact['artifact']
dest = artifact.get('dest')
extract = artifact.get('extract', True)
fetch = {
'artifact': '{prefix}/{path}'.format(prefix=prefix, path=path)
if not path.startswith('/') else path[1:],
'task': '<{dep}>'.format(dep=kind),
'extract': extract,
}
if dest is not None:
fetch['dest'] = dest
job_fetches.append(fetch)
env = worker.setdefault('env', {})
env['MOZ_FETCHES'] = {'task-reference': json.dumps(job_fetches, sort_keys=True)}
# The path is normalized to an absolute path in run-task
env.setdefault('MOZ_FETCHES_DIR', 'fetches')
yield job
from mozbuild.util import (
expand_variables, )
# Define this module as gyp.generator.mozbuild so that gyp can use it
# as a generator under the name "mozbuild".
sys.modules['gyp.generator.mozbuild'] = sys.modules[__name__]
# build/gyp_chromium does this:
# script_dir = os.path.dirname(os.path.realpath(__file__))
# chrome_src = os.path.abspath(os.path.join(script_dir, os.pardir))
# sys.path.insert(0, os.path.join(chrome_src, 'tools', 'gyp', 'pylib'))
# We're not importing gyp_chromium, but we want both script_dir and
# chrome_src for the default includes, so go backwards from the pylib
# directory, which is the parent directory of gyp module.
chrome_src = mozpath.abspath(
mozpath.join(mozpath.dirname(gyp.__file__), '../../../../..'))
script_dir = mozpath.join(chrome_src, 'build')
def encode(value):
if isinstance(value, unicode):
return value.encode('utf-8')
return value
# Default variables gyp uses when evaluating gyp files.
generator_default_variables = {}
for dirname in [
b'INTERMEDIATE_DIR', b'SHARED_INTERMEDIATE_DIR', b'PRODUCT_DIR',
b'LIB_DIR', b'SHARED_LIB_DIR'
]:
def _format_statements_for_generated_file(self, obj, tier,
extra_dependencies=''):
"""Return the list of statements to write to the Makefile for this
GeneratedFile.
This function will invoke _format_generated_file_input_name and
_format_generated_file_output_name to munge the input/output filenames
before sending them to the output.
"""
assert isinstance(obj, GeneratedFile)
# Localized generated files can use {AB_CD} and {AB_rCD} in their
# output paths.
if obj.localized:
substs = {'AB_CD': '$(AB_CD)', 'AB_rCD': '$(AB_rCD)'}
else:
substs = {}
outputs = []
needs_AB_rCD = False
for o in obj.outputs:
needs_AB_rCD = needs_AB_rCD or ('AB_rCD' in o)
try:
outputs.append(self._format_generated_file_output_name(
o.format(**substs), obj))
except KeyError as e:
raise ValueError('%s not in %s is not a valid substitution in %s'
% (e.args[0], ', '.join(sorted(substs.keys())), o))
first_output = outputs[0]
dep_file = mozpath.join(mozpath.dirname(first_output), "$(MDDEPDIR)",
"%s.pp" % mozpath.basename(first_output))
# The stub target file needs to go in MDDEPDIR so that it doesn't
# get written into generated Android resource directories, breaking
# Gradle tooling and/or polluting the Android packages.
stub_file = mozpath.join(mozpath.dirname(first_output), "$(MDDEPDIR)",
"%s.stub" % mozpath.basename(first_output))
if obj.inputs:
inputs = [self._format_generated_file_input_name(f, obj)
for f in obj.inputs]
else:
inputs = []
force = ''
if obj.force:
force = ' FORCE'
elif obj.localized:
force = ' $(if $(IS_LANGUAGE_REPACK),FORCE)'
ret = []
if obj.script:
# If we are doing an artifact build, we don't run compiler, so
# we can skip generated files that are needed during compile,
# or let the rule run as the result of something depending on
# it.
if not (obj.required_before_compile or obj.required_during_compile) or \
not self.environment.is_artifact_build:
if tier and not needs_AB_rCD:
# Android localized resources have special Makefile
# handling.
# Double-colon tiers via a variable that the backend adds as a dependency
# later. See https://bugzilla.mozilla.org/show_bug.cgi?id=1645986#c0 as
# to why.
if tier in ('export', 'pre-compile', 'libs', 'misc'):
dep = '%s_TARGETS' % tier.replace('-', '_').upper()
ret.append('%s += %s' % (dep, stub_file))
else:
ret.append('%s: %s' % (tier, stub_file))
for output in outputs:
ret.append('%s: %s ;' % (output, stub_file))
ret.append('EXTRA_MDDEPEND_FILES += %s' % dep_file)
ret.append((
"""{stub}: {script}{inputs}{backend}{force}
\t$(REPORT_BUILD)
\t$(call py_action,file_generate,{locale}{script} """ # wrap for E501
"""{method} {output} {dep_file} {stub}{inputs}{flags})
\t@$(TOUCH) $@
""").format(
stub=stub_file,
output=first_output,
dep_file=dep_file,
inputs=' ' + ' '.join(inputs) if inputs else '',
flags=' ' + ' '.join(shell_quote(f) for f in obj.flags) if obj.flags else '',
backend=' ' + extra_dependencies if extra_dependencies else '',
# Locale repacks repack multiple locales from a single configured objdir,
# so standard mtime dependencies won't work properly when the build is re-run
# with a different locale as input. IS_LANGUAGE_REPACK will reliably be set
# in this situation, so simply force the generation to run in that case.
force=force,
locale='--locale=$(AB_CD) ' if obj.localized else '',
script=obj.script,
method=obj.method
)
)
return ret
def process_gyp_result(gyp_result, gyp_dir_attrs, path, config, output,
non_unified_sources, action_overrides):
flat_list, targets, data = gyp_result
no_chromium = gyp_dir_attrs.no_chromium
no_unified = gyp_dir_attrs.no_unified
# Process all targets from the given gyp files and its dependencies.
# The path given to AllTargets needs to use os.sep, while the frontend code
# gives us paths normalized with forward slash separator.
for target in gyp.common.AllTargets(flat_list, targets,
path.replace(b'/', os.sep)):
build_file, target_name, toolset = gyp.common.ParseQualifiedTarget(
target)
# Each target is given its own objdir. The base of that objdir
# is derived from the relative path from the root gyp file path
# to the current build_file, placed under the given output
# directory. Since several targets can be in a given build_file,
# separate them in subdirectories using the build_file basename
# and the target_name.
reldir = mozpath.relpath(mozpath.dirname(build_file),
mozpath.dirname(path))
subdir = '%s_%s' % (
mozpath.splitext(mozpath.basename(build_file))[0],
target_name,
)
# Emit a context for each target.
context = GypContext(
config,
mozpath.relpath(mozpath.join(output, reldir, subdir),
config.topobjdir))
context.add_source(mozpath.abspath(build_file))
# The list of included files returned by gyp are relative to build_file
for f in data[build_file]['included_files']:
context.add_source(
mozpath.abspath(mozpath.join(mozpath.dirname(build_file), f)))
spec = targets[target]
# Derive which gyp configuration to use based on MOZ_DEBUG.
c = 'Debug' if config.substs.get('MOZ_DEBUG') else 'Release'
if c not in spec['configurations']:
raise RuntimeError('Missing %s gyp configuration for target %s '
'in %s' % (c, target_name, build_file))
target_conf = spec['configurations'][c]
if 'actions' in spec:
handle_actions(spec['actions'], context, action_overrides)
if 'copies' in spec:
handle_copies(spec['copies'], context)
use_libs = []
libs = []
def add_deps(s):
for t in s.get('dependencies', []) + s.get('dependencies_original',
[]):
ty = targets[t]['type']
if ty in ('static_library', 'shared_library'):
use_libs.append(targets[t]['target_name'])
# Manually expand out transitive dependencies--
# gyp won't do this for static libs or none targets.
if ty in ('static_library', 'none'):
add_deps(targets[t])
libs.extend(spec.get('libraries', []))
# XXX: this sucks, but webrtc breaks with this right now because
# it builds a library called 'gtest' and we just get lucky
# that it isn't in USE_LIBS by that name anywhere.
if no_chromium:
add_deps(spec)
os_libs = []
for l in libs:
if l.startswith('-'):
os_libs.append(l)
elif l.endswith('.lib'):
os_libs.append(l[:-4])
elif l:
# For library names passed in from moz.build.
use_libs.append(os.path.basename(l))
if spec['type'] == 'none':
if not ('actions' in spec or 'copies' in spec):
continue
elif spec['type'] in ('static_library', 'shared_library',
'executable'):
# Remove leading 'lib' from the target_name if any, and use as
# library name.
name = spec['target_name']
if spec['type'] in ('static_library', 'shared_library'):
if name.startswith('lib'):
name = name[3:]
# The context expects an unicode string.
context['LIBRARY_NAME'] = name.decode('utf-8')
else:
context['PROGRAM'] = name.decode('utf-8')
if spec['type'] == 'shared_library':
context['FORCE_SHARED_LIB'] = True
elif spec['type'] == 'static_library' and \
spec.get('variables', {}).get('no_expand_libs', '0') == '1':
# PSM links a NSS static library, but our folded libnss
# doesn't actually export everything that all of the
# objects within would need, so that one library
# should be built as a real static library.
context['NO_EXPAND_LIBS'] = True
if use_libs:
context['USE_LIBS'] = sorted(use_libs, key=lambda s: s.lower())
if os_libs:
context['OS_LIBS'] = os_libs
# gyp files contain headers and asm sources in sources lists.
sources = []
unified_sources = []
extensions = set()
use_defines_in_asflags = False
for f in spec.get('sources', []):
ext = mozpath.splitext(f)[-1]
extensions.add(ext)
if f.startswith('$INTERMEDIATE_DIR/'):
s = ObjDirPath(context, f.replace('$INTERMEDIATE_DIR/',
'!'))
else:
s = SourcePath(context, f)
if ext == '.h':
continue
if ext == '.def':
context['SYMBOLS_FILE'] = s
elif ext != '.S' and not no_unified and s not in non_unified_sources:
unified_sources.append(s)
else:
sources.append(s)
# The Mozilla build system doesn't use DEFINES for building
# ASFILES.
if ext == '.s':
use_defines_in_asflags = True
# The context expects alphabetical order when adding sources
context['SOURCES'] = alphabetical_sorted(sources)
context['UNIFIED_SOURCES'] = alphabetical_sorted(unified_sources)
defines = target_conf.get('defines', [])
if config.substs['CC_TYPE'] == 'clang-cl' and no_chromium:
msvs_settings = gyp.msvs_emulation.MsvsSettings(spec, {})
defines.extend(msvs_settings.GetComputedDefines(c))
for define in defines:
if '=' in define:
name, value = define.split('=', 1)
# The NSS gyp file doesn't expose a way to override this
# currently, so we do so here.
if name == 'NSS_ALLOW_SSLKEYLOGFILE' and \
config.substs.get('RELEASE_OR_BETA', False):
continue
context['DEFINES'][name] = value
else:
context['DEFINES'][define] = True
product_dir_dist = '$PRODUCT_DIR/dist/'
for include in target_conf.get('include_dirs', []):
if include.startswith(product_dir_dist):
# special-case includes of <(PRODUCT_DIR)/dist/ to match
# handle_copies above. This is used for NSS' exports.
include = '!/dist/include/' + include[len(product_dir_dist
):]
elif include.startswith(config.topobjdir):
# NSPR_INCLUDE_DIR gets passed into the NSS build this way.
include = '!/' + mozpath.relpath(include, config.topobjdir)
else:
# moz.build expects all LOCAL_INCLUDES to exist, so ensure they do.
#
# NB: gyp files sometimes have actual absolute paths (e.g.
# /usr/include32) and sometimes paths that moz.build considers
# absolute, i.e. starting from topsrcdir. There's no good way
# to tell them apart here, and the actual absolute paths are
# likely bogus. In any event, actual absolute paths will be
# filtered out by trying to find them in topsrcdir.
#
# We do allow !- and %-prefixed paths, assuming they come
# from moz.build and will be handled the same way as if they
# were given to LOCAL_INCLUDES in moz.build.
if include.startswith('/'):
resolved = mozpath.abspath(
mozpath.join(config.topsrcdir, include[1:]))
elif not include.startswith(('!', '%')):
resolved = mozpath.abspath(
mozpath.join(mozpath.dirname(build_file), include))
if not include.startswith(
('!', '%')) and not os.path.exists(resolved):
continue
context['LOCAL_INCLUDES'] += [include]
context['ASFLAGS'] = target_conf.get('asflags_mozilla', [])
if use_defines_in_asflags and defines:
context['ASFLAGS'] += ['-D' + d for d in defines]
if config.substs['OS_TARGET'] == 'SunOS':
context['LDFLAGS'] = target_conf.get('ldflags', [])
flags = target_conf.get('cflags_mozilla', [])
if flags:
suffix_map = {
'.c': 'CFLAGS',
'.cpp': 'CXXFLAGS',
'.cc': 'CXXFLAGS',
'.m': 'CMFLAGS',
'.mm': 'CMMFLAGS',
}
variables = (suffix_map[e] for e in extensions
if e in suffix_map)
for var in variables:
for f in flags:
# We may be getting make variable references out of the
# gyp data, and we don't want those in emitted data, so
# substitute them with their actual value.
f = expand_variables(f, config.substs).split()
if not f:
continue
# the result may be a string or a list.
if isinstance(f, types.StringTypes):
context[var].append(f)
else:
context[var].extend(f)
else:
# Ignore other types because we don't have
# anything using them, and we're not testing them. They can be
# added when that becomes necessary.
raise NotImplementedError('Unsupported gyp target type: %s' %
spec['type'])
if not no_chromium:
# Add some features to all contexts. Put here in case LOCAL_INCLUDES
# order matters.
context['LOCAL_INCLUDES'] += [
'!/ipc/ipdl/_ipdlheaders',
'/ipc/chromium/src',
'/ipc/glue',
]
# These get set via VC project file settings for normal GYP builds.
if config.substs['OS_TARGET'] == 'WINNT':
context['DEFINES']['UNICODE'] = True
context['DEFINES']['_UNICODE'] = True
context['COMPILE_FLAGS']['OS_INCLUDES'] = []
for key, value in gyp_dir_attrs.sandbox_vars.items():
if context.get(key) and isinstance(context[key], list):
# If we have a key from sanbox_vars that's also been
# populated here we use the value from sandbox_vars as our
# basis rather than overriding outright.
context[key] = value + context[key]
elif context.get(key) and isinstance(context[key], dict):
context[key].update(value)
else:
context[key] = value
yield context
def use_fetches(config, jobs):
artifact_names = {}
aliases = {}
if config.kind in ('toolchain', 'fetch'):
jobs = list(jobs)
for job in jobs:
run = job.get('run', {})
label = job['label']
get_attribute(artifact_names, label, run, 'toolchain-artifact')
value = run.get('{}-alias'.format(config.kind))
if value:
aliases['{}-{}'.format(config.kind, value)] = label
for task in config.kind_dependencies_tasks.values():
if task.kind in ('fetch', 'toolchain'):
get_attribute(
artifact_names,
task.label,
task.attributes,
'{kind}-artifact'.format(kind=task.kind),
)
value = task.attributes.get('{}-alias'.format(task.kind))
if value:
aliases['{}-{}'.format(task.kind, value)] = task.label
artifact_prefixes = {}
for job in order_tasks(config, jobs):
artifact_prefixes[job["label"]] = get_artifact_prefix(job)
fetches = job.pop("fetches", None)
if not fetches:
yield job
continue
job_fetches = []
name = job.get('name', job.get('label'))
dependencies = job.setdefault('dependencies', {})
worker = job.setdefault('worker', {})
prefix = get_artifact_prefix(job)
has_sccache = False
for kind, artifacts in fetches.items():
if kind in ('fetch', 'toolchain'):
for fetch_name in artifacts:
label = '{kind}-{name}'.format(kind=kind, name=fetch_name)
label = aliases.get(label, label)
if label not in artifact_names:
raise Exception(
'Missing fetch job for {kind}-{name}: {fetch}'.
format(kind=config.kind,
name=name,
fetch=fetch_name))
path = artifact_names[label]
dependencies[label] = label
job_fetches.append({
'artifact': path,
'task': '<{label}>'.format(label=label),
'extract': True,
})
if kind == 'toolchain' and fetch_name.endswith('-sccache'):
has_sccache = True
else:
if kind not in dependencies:
raise Exception(
"{name} can't fetch {kind} artifacts because "
"it has no {kind} dependencies!".format(name=name,
kind=kind))
dep_label = dependencies[kind]
if dep_label in artifact_prefixes:
prefix = artifact_prefixes[dep_label]
else:
if dep_label not in config.kind_dependencies_tasks:
raise Exception(
"{name} can't fetch {kind} artifacts because "
"there are no tasks with label {label} in kind dependencies!"
.format(
name=name,
kind=kind,
label=dependencies[kind],
))
prefix = get_artifact_prefix(
config.kind_dependencies_tasks[dep_label])
for artifact in artifacts:
if isinstance(artifact, text_type):
path = artifact
dest = None
extract = True
verify_hash = False
else:
path = artifact['artifact']
dest = artifact.get('dest')
extract = artifact.get('extract', True)
verify_hash = artifact.get('verify-hash', False)
fetch = {
'artifact':
'{prefix}/{path}'.format(prefix=prefix, path=path)
if not path.startswith('/') else path[1:],
'task':
'<{dep}>'.format(dep=kind),
'extract':
extract,
}
if dest is not None:
fetch['dest'] = dest
if verify_hash:
fetch['verify-hash'] = verify_hash
job_fetches.append(fetch)
if job.get('use-sccache') and not has_sccache:
raise Exception(
"Must provide an sccache toolchain if using sccache.")
job_artifact_prefixes = {
mozpath.dirname(fetch["artifact"])
for fetch in job_fetches
if not fetch["artifact"].startswith("public/")
}
if job_artifact_prefixes:
# Use taskcluster-proxy and request appropriate scope. For example, add
# 'scopes: [queue:get-artifact:path/to/*]' for 'path/to/artifact.tar.xz'.
worker["taskcluster-proxy"] = True
for prefix in sorted(job_artifact_prefixes):
scope = "queue:get-artifact:{}/*".format(prefix)
if scope not in job.setdefault("scopes", []):
job["scopes"].append(scope)
env = worker.setdefault('env', {})
env['MOZ_FETCHES'] = {
'task-reference':
six.ensure_text(
json.dumps(sorted(job_fetches,
key=lambda x: sorted(x.items())),
sort_keys=True))
}
# The path is normalized to an absolute path in run-task
env.setdefault('MOZ_FETCHES_DIR', 'fetches')
yield job
def test_java_tool_checks(self):
includes = ('util.configure', 'checks.configure', 'java.configure')
def mock_valid_javac(_, args):
if len(args) == 1 and args[0] == '-version':
return 0, '1.7', ''
self.fail("Unexpected arguments to mock_valid_javac: %s" % args)
# A valid set of tools in a standard location.
java = mozpath.abspath('/usr/bin/java')
javah = mozpath.abspath('/usr/bin/javah')
javac = mozpath.abspath('/usr/bin/javac')
jar = mozpath.abspath('/usr/bin/jar')
jarsigner = mozpath.abspath('/usr/bin/jarsigner')
keytool = mozpath.abspath('/usr/bin/keytool')
paths = {
java: None,
javah: None,
javac: mock_valid_javac,
jar: None,
jarsigner: None,
keytool: None,
}
config, out, status = self.get_result(includes=includes,
extra_paths=paths)
self.assertEqual(status, 0)
self.assertEqual(
config, {
'JAVA': java,
'JAVAH': javah,
'JAVAC': javac,
'JAR': jar,
'JARSIGNER': jarsigner,
'KEYTOOL': keytool,
})
self.assertEqual(
out,
textwrap.dedent('''\
checking for java... %s
checking for javah... %s
checking for jar... %s
checking for jarsigner... %s
checking for keytool... %s
checking for javac... %s
checking for javac version... 1.7
''' % (java, javah, jar, jarsigner, keytool, javac)))
# An alternative valid set of tools referred to by JAVA_HOME.
alt_java = mozpath.abspath('/usr/local/bin/java')
alt_javah = mozpath.abspath('/usr/local/bin/javah')
alt_javac = mozpath.abspath('/usr/local/bin/javac')
alt_jar = mozpath.abspath('/usr/local/bin/jar')
alt_jarsigner = mozpath.abspath('/usr/local/bin/jarsigner')
alt_keytool = mozpath.abspath('/usr/local/bin/keytool')
alt_java_home = mozpath.dirname(mozpath.dirname(alt_java))
paths.update({
alt_java: None,
alt_javah: None,
alt_javac: mock_valid_javac,
alt_jar: None,
alt_jarsigner: None,
alt_keytool: None,
})
config, out, status = self.get_result(includes=includes,
extra_paths=paths,
environ={
'JAVA_HOME': alt_java_home,
'PATH': mozpath.dirname(java)
})
self.assertEqual(status, 0)
self.assertEqual(
config, {
'JAVA': alt_java,
'JAVAH': alt_javah,
'JAVAC': alt_javac,
'JAR': alt_jar,
'JARSIGNER': alt_jarsigner,
'KEYTOOL': alt_keytool,
})
self.assertEqual(
out,
textwrap.dedent('''\
checking for java... %s
checking for javah... %s
checking for jar... %s
checking for jarsigner... %s
checking for keytool... %s
checking for javac... %s
checking for javac version... 1.7
''' % (alt_java, alt_javah, alt_jar, alt_jarsigner, alt_keytool,
alt_javac)))
# We can use --with-java-bin-path instead of JAVA_HOME to similar
# effect.
config, out, status = self.get_result(
args=['--with-java-bin-path=%s' % mozpath.dirname(alt_java)],
includes=includes,
extra_paths=paths,
environ={'PATH': mozpath.dirname(java)})
self.assertEqual(status, 0)
self.assertEqual(
config, {
'JAVA': alt_java,
'JAVAH': alt_javah,
'JAVAC': alt_javac,
'JAR': alt_jar,
'JARSIGNER': alt_jarsigner,
'KEYTOOL': alt_keytool,
})
self.assertEqual(
out,
textwrap.dedent('''\
checking for java... %s
checking for javah... %s
checking for jar... %s
checking for jarsigner... %s
checking for keytool... %s
checking for javac... %s
checking for javac version... 1.7
''' % (alt_java, alt_javah, alt_jar, alt_jarsigner, alt_keytool,
alt_javac)))
# If --with-java-bin-path and JAVA_HOME are both set,
# --with-java-bin-path takes precedence.
config, out, status = self.get_result(
args=['--with-java-bin-path=%s' % mozpath.dirname(alt_java)],
includes=includes,
extra_paths=paths,
environ={
'PATH': mozpath.dirname(java),
'JAVA_HOME': mozpath.dirname(mozpath.dirname(java)),
})
self.assertEqual(status, 0)
self.assertEqual(
config, {
'JAVA': alt_java,
'JAVAH': alt_javah,
'JAVAC': alt_javac,
'JAR': alt_jar,
'JARSIGNER': alt_jarsigner,
'KEYTOOL': alt_keytool,
})
self.assertEqual(
out,
textwrap.dedent('''\
checking for java... %s
checking for javah... %s
checking for jar... %s
checking for jarsigner... %s
checking for keytool... %s
checking for javac... %s
checking for javac version... 1.7
''' % (alt_java, alt_javah, alt_jar, alt_jarsigner, alt_keytool,
alt_javac)))
def mock_old_javac(_, args):
if len(args) == 1 and args[0] == '-version':
return 0, '1.6.9', ''
self.fail("Unexpected arguments to mock_old_javac: %s" % args)
# An old javac is fatal.
paths[javac] = mock_old_javac
config, out, status = self.get_result(
includes=includes,
extra_paths=paths,
environ={'PATH': mozpath.dirname(java)})
self.assertEqual(status, 1)
self.assertEqual(
config, {
'JAVA': java,
'JAVAH': javah,
'JAVAC': javac,
'JAR': jar,
'JARSIGNER': jarsigner,
'KEYTOOL': keytool,
})
self.assertEqual(
out,
textwrap.dedent('''\
checking for java... %s
checking for javah... %s
checking for jar... %s
checking for jarsigner... %s
checking for keytool... %s
checking for javac... %s
checking for javac version...
ERROR: javac 1.7 or higher is required (found 1.6.9)
''' % (java, javah, jar, jarsigner, keytool, javac)))
# Any missing tool is fatal when these checks run.
del paths[jarsigner]
config, out, status = self.get_result(
includes=includes,
extra_paths=paths,
environ={'PATH': mozpath.dirname(java)})
self.assertEqual(status, 1)
self.assertEqual(config, {
'JAVA': java,
'JAVAH': javah,
'JAR': jar,
'JARSIGNER': ':',
})
self.assertEqual(
out,
textwrap.dedent('''\
checking for java... %s
checking for javah... %s
checking for jar... %s
checking for jarsigner... not found
ERROR: The program jarsigner was not found. Set $JAVA_HOME to your Java SDK directory or use '--with-java-bin-path={java-bin-dir}'
''' % (java, javah, jar)))
def test_java_tool_checks(self):
includes = ('util.configure', 'checks.configure', 'java.configure')
# A valid set of tools in a standard location.
java = mozpath.abspath('/usr/bin/java')
javah = mozpath.abspath('/usr/bin/javah')
javac = mozpath.abspath('/usr/bin/javac')
jar = mozpath.abspath('/usr/bin/jar')
jarsigner = mozpath.abspath('/usr/bin/jarsigner')
keytool = mozpath.abspath('/usr/bin/keytool')
proguard_jar = mozpath.abspath('/path/to/proguard.jar')
old_proguard_jar = mozpath.abspath('/path/to/old_proguard.jar')
def mock_valid_java(_, args):
# Yield valid proguard.jar output with a version based on the given path.
stdout = \
'ProGuard, version {version}' + \
'Usage: java proguard.ProGuard [options ...]'
args = tuple(args)
if args == ('-jar', proguard_jar):
return 1, stdout.format(version="5.3.3"), ''
elif args == ('-jar', old_proguard_jar):
return 1, stdout.format(version="4.2"), ''
self.fail("Unexpected arguments to mock_valid_java: %s" % args)
def mock_valid_javac(_, args):
if len(args) == 1 and args[0] == '-version':
return 0, '1.8', ''
self.fail("Unexpected arguments to mock_valid_javac: %s" % args)
paths = {
java: mock_valid_java,
javah: None,
javac: mock_valid_javac,
jar: None,
jarsigner: None,
keytool: None,
proguard_jar: mock_valid_java,
}
config, out, status = self.get_result(includes=includes,
extra_paths=paths,
environ={
'PROGUARD_JAR': proguard_jar,
})
self.assertEqual(
out,
textwrap.dedent('''\
checking for java... %s
checking for javah... %s
checking for jar... %s
checking for jarsigner... %s
checking for keytool... %s
checking for javac... %s
checking for javac version... 1.8
checking for proguard.jar version... %s
''' % (java, javah, jar, jarsigner, keytool, javac, proguard_jar)))
self.assertEqual(status, 0)
self.assertEqual(
config, {
'JAVA': java,
'JAVAH': javah,
'JAVAC': javac,
'JAR': jar,
'JARSIGNER': jarsigner,
'KEYTOOL': keytool,
'PROGUARD_JAR': proguard_jar,
})
# An alternative valid set of tools referred to by JAVA_HOME.
alt_java = mozpath.abspath('/usr/local/bin/java')
alt_javah = mozpath.abspath('/usr/local/bin/javah')
alt_javac = mozpath.abspath('/usr/local/bin/javac')
alt_jar = mozpath.abspath('/usr/local/bin/jar')
alt_jarsigner = mozpath.abspath('/usr/local/bin/jarsigner')
alt_keytool = mozpath.abspath('/usr/local/bin/keytool')
alt_java_home = mozpath.dirname(mozpath.dirname(alt_java))
paths.update({
alt_java: mock_valid_java,
alt_javah: None,
alt_javac: mock_valid_javac,
alt_jar: None,
alt_jarsigner: None,
alt_keytool: None,
})
config, out, status = self.get_result(includes=includes,
extra_paths=paths,
environ={
'JAVA_HOME': alt_java_home,
'PATH':
mozpath.dirname(java),
'PROGUARD_JAR': proguard_jar,
})
self.assertEqual(
out,
textwrap.dedent('''\
checking for java... %s
checking for javah... %s
checking for jar... %s
checking for jarsigner... %s
checking for keytool... %s
checking for javac... %s
checking for javac version... 1.8
checking for proguard.jar version... %s
''' % (alt_java, alt_javah, alt_jar, alt_jarsigner, alt_keytool,
alt_javac, proguard_jar)))
self.assertEqual(status, 0)
self.assertEqual(
config, {
'JAVA': alt_java,
'JAVAH': alt_javah,
'JAVAC': alt_javac,
'JAR': alt_jar,
'JARSIGNER': alt_jarsigner,
'KEYTOOL': alt_keytool,
'PROGUARD_JAR': proguard_jar,
})
# We can use --with-java-bin-path instead of JAVA_HOME to similar
# effect.
config, out, status = self.get_result(
args=['--with-java-bin-path=%s' % mozpath.dirname(alt_java)],
includes=includes,
extra_paths=paths,
environ={
'PATH': mozpath.dirname(java),
'PROGUARD_JAR': proguard_jar,
})
self.assertEqual(
out,
textwrap.dedent('''\
checking for java... %s
checking for javah... %s
checking for jar... %s
checking for jarsigner... %s
checking for keytool... %s
checking for javac... %s
checking for javac version... 1.8
checking for proguard.jar version... %s
''' % (alt_java, alt_javah, alt_jar, alt_jarsigner, alt_keytool,
alt_javac, proguard_jar)))
self.assertEqual(status, 0)
self.assertEqual(
config, {
'JAVA': alt_java,
'JAVAH': alt_javah,
'JAVAC': alt_javac,
'JAR': alt_jar,
'JARSIGNER': alt_jarsigner,
'KEYTOOL': alt_keytool,
'PROGUARD_JAR': proguard_jar,
})
# If --with-java-bin-path and JAVA_HOME are both set,
# --with-java-bin-path takes precedence.
config, out, status = self.get_result(
args=['--with-java-bin-path=%s' % mozpath.dirname(alt_java)],
includes=includes,
extra_paths=paths,
environ={
'PATH': mozpath.dirname(java),
'JAVA_HOME': mozpath.dirname(mozpath.dirname(java)),
'PROGUARD_JAR': proguard_jar,
})
self.assertEqual(
out,
textwrap.dedent('''\
checking for java... %s
checking for javah... %s
checking for jar... %s
checking for jarsigner... %s
checking for keytool... %s
checking for javac... %s
checking for javac version... 1.8
checking for proguard.jar version... %s
''' % (alt_java, alt_javah, alt_jar, alt_jarsigner, alt_keytool,
alt_javac, proguard_jar)))
self.assertEqual(status, 0)
self.assertEqual(
config, {
'JAVA': alt_java,
'JAVAH': alt_javah,
'JAVAC': alt_javac,
'JAR': alt_jar,
'JARSIGNER': alt_jarsigner,
'KEYTOOL': alt_keytool,
'PROGUARD_JAR': proguard_jar,
})
def mock_old_javac(_, args):
if len(args) == 1 and args[0] == '-version':
return 0, '1.6.9', ''
self.fail("Unexpected arguments to mock_old_javac: %s" % args)
# An old proguard JAR is fatal.
config, out, status = self.get_result(includes=includes,
extra_paths=paths,
environ={
'PATH':
mozpath.dirname(java),
'PROGUARD_JAR':
old_proguard_jar,
})
self.assertEqual(
out,
textwrap.dedent('''\
checking for java... %s
checking for javah... %s
checking for jar... %s
checking for jarsigner... %s
checking for keytool... %s
checking for javac... %s
checking for javac version... 1.8
checking for proguard.jar version...
ERROR: proguard.jar 5.3.3 or higher is required (looked for %s). Run |mach artifact toolchain --from-build proguard-jar && mv proguard ~/.mozbuild/| or add `export PROGUARD_JAR=/path/to/proguard.jar` to your mozconfig.
''' % (java, javah, jar, jarsigner, keytool, javac, old_proguard_jar)))
self.assertEqual(status, 1)
self.assertEqual(
config, {
'JAVA': java,
'JAVAH': javah,
'JAVAC': javac,
'JAR': jar,
'JARSIGNER': jarsigner,
'KEYTOOL': keytool,
})
# An old javac is fatal.
paths[javac] = mock_old_javac
config, out, status = self.get_result(includes=includes,
extra_paths=paths,
environ={
'PATH':
mozpath.dirname(java),
'PROGUARD_JAR': proguard_jar,
})
self.assertEqual(
out,
textwrap.dedent('''\
checking for java... %s
checking for javah... %s
checking for jar... %s
checking for jarsigner... %s
checking for keytool... %s
checking for javac... %s
checking for javac version...
ERROR: javac 1.8 or higher is required (found 1.6.9). Check the JAVA_HOME environment variable.
''' % (java, javah, jar, jarsigner, keytool, javac)))
self.assertEqual(status, 1)
self.assertEqual(
config, {
'JAVA': java,
'JAVAH': javah,
'JAVAC': javac,
'JAR': jar,
'JARSIGNER': jarsigner,
'KEYTOOL': keytool,
})
# Any missing tool is fatal when these checks run.
del paths[jarsigner]
config, out, status = self.get_result(includes=includes,
extra_paths=paths,
environ={
'PATH':
mozpath.dirname(java),
'PROGUARD_JAR': proguard_jar,
})
self.assertEqual(status, 1)
self.assertEqual(config, {
'JAVA': java,
'JAVAH': javah,
'JAR': jar,
'JARSIGNER': ':',
})
self.assertEqual(
out,
textwrap.dedent('''\
checking for java... %s
checking for javah... %s
checking for jar... %s
checking for jarsigner... not found
ERROR: The program jarsigner was not found. Set $JAVA_HOME to your Java SDK directory or use '--with-java-bin-path={java-bin-dir}'
''' % (java, javah, jar)))
