def _process_reftest_manifest(self, sandbox, flavor, manifest_path):
manifest_path = mozpath.normpath(manifest_path)
manifest_full_path = mozpath.normpath(mozpath.join(
sandbox['SRCDIR'], manifest_path))
manifest_reldir = mozpath.dirname(mozpath.relpath(manifest_full_path,
sandbox['TOPSRCDIR']))
manifest = reftest.ReftestManifest()
manifest.load(manifest_full_path)
# reftest manifests don't come from manifest parser. But they are
# similar enough that we can use the same emitted objects. Note
# that we don't perform any installs for reftests.
obj = TestManifest(sandbox, manifest_full_path, manifest,
flavor=flavor, install_prefix='%s/' % flavor,
relpath=mozpath.join(manifest_reldir,
mozpath.basename(manifest_path)))
for test in sorted(manifest.files):
obj.tests.append({
'path': test,
'here': mozpath.dirname(test),
'manifest': manifest_full_path,
'name': mozpath.basename(test),
'head': '',
'tail': '',
'support-files': '',
'subsuite': '',
})
yield obj
def rewrite_url(self, url):
# This applies one-off rules and returns None for urls that we aren't
# going to be able to resolve to a source file ("about:" urls, for
# instance).
if url in self._final_mapping:
return self._final_mapping[url]
if url.endswith('> eval'):
return None
if url.endswith('> Function'):
return None
if ' -> ' in url:
url = url.split(' -> ')[1].rstrip()
if '?' in url:
url = url.split('?')[0]
url_obj = urlparse.urlparse(url)
if url_obj.scheme == 'jar':
app_name = self.MOZ_APP_NAME
omnijar_name = self.OMNIJAR_NAME
if app_name in url:
if omnijar_name in url:
# e.g. file:///home/worker/workspace/build/application/firefox/omni.ja!/components/MainProcessSingleton.js
parts = url_obj.path.split(omnijar_name + '!', 1)
elif '.xpi!' in url:
# e.g. file:///home/worker/workspace/build/application/firefox/browser/features/[email protected]!/bootstrap.js
parts = url_obj.path.split('.xpi!', 1)
else:
# We don't know how to handle this jar: path, so return it to the
# caller to make it print a warning.
return url_obj.path, None
dir_parts = parts[0].rsplit(app_name + '/', 1)
url = mozpath.normpath(mozpath.join(self.topobjdir, 'dist', 'bin', dir_parts[1].lstrip('/'), parts[1].lstrip('/')))
elif '.xpi!' in url:
# e.g. file:///tmp/tmpMdo5gV.mozrunner/extensions/[email protected]!/bootstrap.js
# This matching mechanism is quite brittle and based on examples seen in the wild.
# There's no rule to match the XPI name to the path in dist/xpi-stage.
parts = url_obj.path.split('.xpi!', 1)
addon_name = os.path.basename(parts[0])
if '*****@*****.**' in addon_name:
addon_name = addon_name[:-len('*****@*****.**')]
elif addon_name.endswith('@mozilla.org'):
addon_name = addon_name[:-len('@mozilla.org')]
url = mozpath.normpath(mozpath.join(self.topobjdir, 'dist', 'xpi-stage', addon_name, parts[1].lstrip('/')))
elif url_obj.scheme == 'file' and os.path.isabs(url_obj.path):
path = url_obj.path
if not os.path.isfile(path):
# This may have been in a profile directory that no
# longer exists.
return None
if not path.startswith(self.topobjdir):
return path, None
url = url_obj.path
elif url_obj.scheme in ('http', 'https', 'javascript', 'data', 'about'):
return None
result = self.find_files(url)
self._final_mapping[url] = result
return result
def is_read_allowed(path, config):
"""Whether we are allowed to load a mozbuild file at the specified path.
This is used as cheap security to ensure the build is isolated to known
source directories.
We are allowed to read from the main source directory and any defined
external source directories. The latter is to allow 3rd party applications
to hook into our build system.
"""
assert os.path.isabs(path)
assert os.path.isabs(config.topsrcdir)
path = mozpath.normpath(path)
topsrcdir = mozpath.normpath(config.topsrcdir)
if mozpath.basedir(path, [topsrcdir]):
return True
if config.external_source_dir:
external_dir = os.path.normcase(config.external_source_dir)
norm_path = os.path.normcase(path)
if mozpath.basedir(norm_path, [external_dir]):
return True
return False
def test_include_failures(self):
with self.assertRaises(ConfigureError) as e:
with self.moz_configure('include("../foo.configure")'):
self.get_config()
self.assertEquals(
e.exception.message,
'Cannot include `%s` because it is not in a subdirectory of `%s`'
% (mozpath.normpath(mozpath.join(test_data_path, '..',
'foo.configure')),
mozpath.normsep(test_data_path))
)
with self.assertRaises(ConfigureError) as e:
with self.moz_configure('''
include('extra.configure')
include('extra.configure')
'''):
self.get_config()
self.assertEquals(
e.exception.message,
'Cannot include `%s` because it was included already.'
% mozpath.normpath(mozpath.join(test_data_path,
'extra.configure'))
)
with self.assertRaises(TypeError) as e:
with self.moz_configure('''
include(42)
'''):
self.get_config()
self.assertEquals(e.exception.message, "Unexpected type: 'int'")
def _process_reftest_manifest(self, sandbox, flavor, manifest_path):
manifest_path = mozpath.normpath(manifest_path)
manifest_full_path = mozpath.normpath(mozpath.join(sandbox["SRCDIR"], manifest_path))
manifest_reldir = mozpath.dirname(mozpath.relpath(manifest_full_path, sandbox["TOPSRCDIR"]))
manifest = reftest.ReftestManifest()
manifest.load(manifest_full_path)
# reftest manifests don't come from manifest parser. But they are
# similar enough that we can use the same emitted objects. Note
# that we don't perform any installs for reftests.
obj = TestManifest(
sandbox,
manifest_full_path,
manifest,
flavor=flavor,
install_prefix="%s/" % flavor,
relpath=mozpath.join(manifest_reldir, mozpath.basename(manifest_path)),
)
for test in sorted(manifest.files):
obj.tests.append(
{
"path": test,
"here": mozpath.dirname(test),
"manifest": manifest_full_path,
"name": mozpath.basename(test),
"head": "",
"tail": "",
"support-files": "",
"subsuite": "",
}
)
yield obj
def handle_manifest_entry(self, entry):
format_strings = {
"content": "chrome://%s/content/",
"resource": "resource://%s/",
"locale": "chrome://%s/locale/",
"skin": "chrome://%s/skin/",
}
if isinstance(entry, (ManifestChrome, ManifestResource)):
if isinstance(entry, ManifestResource):
dest = entry.target
url = urlparse.urlparse(dest)
if not url.scheme:
dest = mozpath.normpath(mozpath.join(entry.base, dest))
if url.scheme == 'file':
dest = mozpath.normpath(url.path)
else:
dest = mozpath.normpath(entry.path)
base_uri = format_strings[entry.type] % entry.name
self.chrome_mapping[base_uri].add(dest)
if isinstance(entry, ManifestOverride):
self.overrides[entry.overloaded] = entry.overload
if isinstance(entry, Manifest):
for e in parse_manifest(None, entry.path):
self.handle_manifest_entry(e)
def normalize_path(self, path, filesystem_absolute=False, srcdir=None):
"""Normalizes paths.
If the path is absolute, behavior is governed by filesystem_absolute.
If filesystem_absolute is True, the path is interpreted as absolute on
the actual filesystem. If it is false, the path is treated as absolute
within the current topsrcdir.
If the path is not absolute, it will be treated as relative to the
currently executing file. If there is no currently executing file, it
will be treated as relative to topsrcdir.
"""
if os.path.isabs(path):
if filesystem_absolute:
return path
for root in [self.topsrcdir] + self.external_source_dirs:
# mozpath.join would ignore the self.topsrcdir argument if we
# passed in the absolute path, so omit the leading /
p = mozpath.normpath(mozpath.join(root, path[1:]))
if os.path.exists(p):
return p
# mozpath.join would ignore the self.topsrcdir argument if we passed
# in the absolute path, so omit the leading /
return mozpath.normpath(mozpath.join(self.topsrcdir, path[1:]))
elif srcdir:
return mozpath.normpath(mozpath.join(srcdir, path))
elif len(self._execution_stack):
return mozpath.normpath(mozpath.join(
mozpath.dirname(self._execution_stack[-1]), path))
else:
return mozpath.normpath(mozpath.join(self.topsrcdir, path))
def is_read_allowed(path, config):
"""Whether we are allowed to load a mozbuild file at the specified path.
This is used as cheap security to ensure the build is isolated to known
source directories.
We are allowed to read from the main source directory and any defined
external source directories. The latter is to allow 3rd party applications
to hook into our build system.
"""
assert os.path.isabs(path)
assert os.path.isabs(config.topsrcdir)
path = mozpath.normpath(path)
topsrcdir = mozpath.normpath(config.topsrcdir)
if path.startswith(topsrcdir):
return True
external_dirs = config.substs.get('EXTERNAL_SOURCE_DIR', '').split()
for external in external_dirs:
if not os.path.isabs(external):
external = mozpath.join(config.topsrcdir, external)
external = mozpath.normpath(external)
if path.startswith(external):
return True
return False
def __init__(self, srcdir, objdir, fix):
self.srcdir = mozpath.normpath(srcdir)
self.objdir = mozpath.normpath(objdir)
self.srcdir_mount = self.getmount(self.srcdir)
self.objdir_mount = self.getmount(self.objdir)
self.path_mounts = [
('srcdir', self.srcdir, self.srcdir_mount),
('objdir', self.objdir, self.objdir_mount)
]
self.fix = fix
self.results = []
def test_config_file_substitution(self):
reader = self.reader("config-file-substitution")
objs = self.read_topsrcdir(reader)
self.assertEqual(len(objs), 2)
self.assertIsInstance(objs[0], ConfigFileSubstitution)
self.assertIsInstance(objs[1], ConfigFileSubstitution)
topobjdir = mozpath.abspath(reader.config.topobjdir)
self.assertEqual(objs[0].relpath, "foo")
self.assertEqual(mozpath.normpath(objs[0].output_path), mozpath.normpath(mozpath.join(topobjdir, "foo")))
self.assertEqual(mozpath.normpath(objs[1].output_path), mozpath.normpath(mozpath.join(topobjdir, "bar")))
def output_to_inputs_tree(self):
'''
Return a dictionary mapping each output path to the set of its
required input paths.
All paths are normalized.
'''
tree = {}
for output, file in self:
output = mozpath.normpath(output)
tree[output] = set(mozpath.normpath(f) for f in file.inputs())
return tree
def input_to_outputs_tree(self):
'''
Return a dictionary mapping each input path to the set of
impacted output paths.
All paths are normalized.
'''
tree = defaultdict(set)
for output, file in self:
output = mozpath.normpath(output)
for input in file.inputs():
input = mozpath.normpath(input)
tree[input].add(output)
return dict(tree)
def test_config_file_substitution(self):
reader = self.reader('config-file-substitution')
objs = self.read_topsrcdir(reader)
self.assertEqual(len(objs), 3)
self.assertIsInstance(objs[0], DirectoryTraversal)
self.assertIsInstance(objs[1], ConfigFileSubstitution)
self.assertIsInstance(objs[2], ConfigFileSubstitution)
topobjdir = mozpath.abspath(reader.config.topobjdir)
self.assertEqual(objs[1].relpath, 'foo')
self.assertEqual(mozpath.normpath(objs[1].output_path),
mozpath.normpath(mozpath.join(topobjdir, 'foo')))
self.assertEqual(mozpath.normpath(objs[2].output_path),
mozpath.normpath(mozpath.join(topobjdir, 'bar')))
def _add_entry(self, dest, entry, normlize_dest=True):
if normlize_dest:
dest = mozpath.normpath(dest)
if dest in self._dests:
raise ValueError("Item already in manifest: %s" % dest)
self._dests[dest] = entry
def __init__(self, config):
self.config = config
self.topsrcdir = mozpath.normpath(config.topsrcdir)
self.tests_by_path = defaultdict(list)
self.installs_by_path = defaultdict(list)
self.deferred_installs = set()
def add_manifest(self, entry):
# Store manifest entries in a single manifest per directory, named
# after their parent directory, except for root manifests, all named
# chrome.manifest.
if entry.base:
name = mozpath.basename(entry.base)
else:
name = 'chrome'
path = mozpath.normpath(mozpath.join(entry.base, '%s.manifest' % name))
if not self.copier.contains(path):
# Add a reference to the manifest file in the parent manifest, if
# the manifest file is not a root manifest.
if entry.base:
parent = mozpath.dirname(entry.base)
relbase = mozpath.basename(entry.base)
relpath = mozpath.join(relbase,
mozpath.basename(path))
self.add_manifest(Manifest(parent, relpath))
self.copier.add(path, ManifestFile(entry.base))
if isinstance(entry, ManifestChrome):
data = self._chrome_db.setdefault(entry.name, {})
entries = data.setdefault(entry.type, [])
for e in entries:
# Ideally, we'd actually check whether entry.flags are more
# specific than e.flags, but in practice the following test
# is enough for now.
if not entry.flags or e.flags and entry.flags == e.flags:
errors.fatal('"%s" overrides "%s"' % (entry, e))
entries.append(entry)
self.copier[path].add(entry)
def _process_files(self, obj, files, target, preprocessor = False, marker='#', target_is_file=False, optional=False):
for f in files:
if optional:
full_dest = f
elif target_is_file:
full_dest = target
else:
full_dest = mozpath.join(target, mozpath.basename(f))
install_manifest, dest = self._get_manifest_from_target(full_dest)
source = None if (obj is None) else mozpath.normpath(mozpath.join(obj.srcdir, f))
if preprocessor:
dep_file = mozpath.join(self.dep_path, target, mozpath.basename(f) +'.pp')
exist_defines = self._paths_to_defines.get(obj.srcdir, {})
xul_defines = dict(exist_defines)
for flag in self.XULPPFLAGS:
if flag.startswith('-D'):
define = flag[2:].split('=')
xul_defines[define[0]] = define[1] if len(define) >= 2 else ''
defines = compute_defines(obj.config, defines = xul_defines)
new_marker = marker
if marker == 'jar':
new_marker = '%' if f.endswith('.css') else '#'
install_manifest.add_preprocess(source, dest, dep_file, marker=new_marker, defines=defines)
elif optional:
install_manifest.add_optional_exists(dest)
else:
install_manifest.add_symlink(source, dest)
def _get_backend_file(self, relobjdir):
objdir = mozpath.normpath(mozpath.join(self.environment.topobjdir, relobjdir))
if objdir not in self._backend_files:
self._backend_files[objdir] = \
BackendTupfile(objdir, self.environment,
self.environment.topsrcdir, self.environment.topobjdir)
return self._backend_files[objdir]
def addDependencies(self, target, deps):
if not target in self.newDeps:
self.newDeps[target] = set()
for p in deps:
p = mozpath.normpath(p)
self.newDeps[target].add(p)
self.sourceFiles.add(p)
def find_reftest_dirs(topsrcdir, manifests):
from reftest import ReftestManifest
dirs = set()
for p in manifests:
m = ReftestManifest()
m.load(os.path.join(topsrcdir, p))
dirs |= m.dirs
dirs = {mozpath.normpath(d[len(topsrcdir):]).lstrip('/') for d in dirs}
# Filter out children captured by parent directories because duplicates
# will confuse things later on.
def parents(p):
while True:
p = mozpath.dirname(p)
if not p:
break
yield p
seen = set()
for d in sorted(dirs, key=len):
if not any(p in seen for p in parents(d)):
seen.add(d)
return sorted(seen)
def include_file(self, path):
'''Include one file in the sandbox. Users of this class probably want
Note: this will execute all template invocations, as well as @depends
functions that depend on '--help', but nothing else.
'''
if self._paths:
path = mozpath.join(mozpath.dirname(self._paths[-1]), path)
path = mozpath.normpath(path)
if not mozpath.basedir(path, (mozpath.dirname(self._paths[0]),)):
raise ConfigureError(
'Cannot include `%s` because it is not in a subdirectory '
'of `%s`' % (path, mozpath.dirname(self._paths[0])))
else:
path = mozpath.realpath(mozpath.abspath(path))
if path in self._all_paths:
raise ConfigureError(
'Cannot include `%s` because it was included already.' % path)
self._paths.append(path)
self._all_paths.add(path)
source = open(path, 'rb').read()
code = compile(source, path, 'exec')
exec_(code, self)
self._paths.pop(-1)
def load_deps_file(self, objdir, fh):
"""Load a single dependency file."""
for rule in read_dep_makefile(fh):
for target in rule.targets():
full_target = mozpath.normpath(mozpath.join(objdir,
target))
normalized_deps = []
for d in rule.dependencies():
full_depend = mozpath.join(objdir, d)
# Resolve symbolic links from $objdir/dist/include and
# the like to their srcdir equivalents. Don't use
# _realpath_cache.get(full_depend, os.path.realpath(...)),
# as the whole point of this cache is to avoid hitting
# the filesystem if we don't have to.
if full_depend in self._realpath_cache:
full_depend = self._realpath_cache[full_depend]
else:
resolved = os.path.realpath(full_depend)
self._realpath_cache[full_depend] = resolved
full_depend = resolved
normalized_deps.append(full_depend)
self.dependencies.setdefault(full_depend, set()).add(full_target)
assert full_target not in self.targets
self.targets[full_target] = normalized_deps
def process_support_files(test):
for thing, seen in extras:
value = test.get(thing, '')
if value in seen:
continue
seen.add(value)
for pattern in value.split():
# We only support globbing on support-files because
# the harness doesn't support * for head and tail.
if '*' in pattern and thing == 'support-files':
obj.pattern_installs.append(
(manifest_dir, pattern, out_dir))
# "absolute" paths identify files that are to be
# placed in the install_root directory (no globs)
elif pattern[0] == '/':
full = mozpath.normpath(mozpath.join(manifest_dir,
mozpath.basename(pattern)))
obj.installs[full] = (mozpath.join(install_root,
pattern[1:]), False)
else:
full = mozpath.normpath(mozpath.join(manifest_dir,
pattern))
dest_path = mozpath.join(out_dir, pattern)
# If the path resolves to a different directory
# tree, we take special behavior depending on the
# entry type.
if not full.startswith(manifest_dir):
# If it's a support file, we install the file
# into the current destination directory.
# This implementation makes installing things
# with custom prefixes impossible. If this is
# needed, we can add support for that via a
# special syntax later.
if thing == 'support-files':
dest_path = mozpath.join(out_dir,
os.path.basename(pattern))
# If it's not a support file, we ignore it.
# This preserves old behavior so things like
# head files doesn't get installed multiple
# times.
else:
continue
obj.installs[full] = (mozpath.normpath(dest_path),
False)
def resolve_tests(self, paths=None, flavor=None, under_path=None):
"""Resolve tests from an identifier.
This is a generator of dicts describing each test.
``paths`` can be an iterable of values to use to identify tests to run.
If an entry is a known test file, tests associated with that file are
returned (there may be multiple configurations for a single file). If
an entry is a directory, all tests in that directory are returned. If
the string appears in a known test file, that test file is considered.
If ``under_path`` is a string, it will be used to filter out tests that
aren't in the specified path prefix relative to topsrcdir or the
test's installed dir.
If ``flavor`` is a string, it will be used to filter returned tests
to only be the flavor specified. A flavor is something like
``xpcshell``.
"""
def fltr(tests):
for test in tests:
if flavor:
if (flavor == 'devtools' and test.get('flavor') != 'browser-chrome') or \
(flavor != 'devtools' and test.get('flavor') != flavor):
continue
if under_path \
and not test['file_relpath'].startswith(under_path):
continue
# Make a copy so modifications don't change the source.
yield dict(test)
paths = paths or []
paths = [mozpath.normpath(p) for p in paths]
if not paths:
paths = [None]
candidate_paths = set()
for path in sorted(paths):
if path is None:
candidate_paths |= set(self._tests_by_path.keys())
continue
# If the path is a directory, pull in all tests in that directory.
if path in self._test_dirs:
candidate_paths |= {p for p in self._tests_by_path
if p.startswith(path)}
continue
# If it's a test file, add just that file.
candidate_paths |= {p for p in self._tests_by_path if path in p}
for p in sorted(candidate_paths):
tests = self._tests_by_path[p]
for test in fltr(tests):
yield test
def test_test_manifest_install_includes(self):
"""Ensure that any [include:foo.ini] are copied to the objdir."""
reader = self.reader('test-manifest-install-includes')
objs = self.read_topsrcdir(reader)
self.assertEqual(len(objs), 1)
o = objs[0]
self.assertEqual(len(o.installs), 3)
self.assertEqual(o.manifest_relpath, "mochitest.ini")
self.assertEqual(o.manifest_obj_relpath, "subdir/mochitest.ini")
expected = [
mozpath.normpath(mozpath.join(o.install_prefix, "subdir/common.ini")),
mozpath.normpath(mozpath.join(o.install_prefix, "subdir/mochitest.ini")),
mozpath.normpath(mozpath.join(o.install_prefix, "subdir/test_foo.html")),
]
paths = sorted([v[0] for v in o.installs.values()])
self.assertEqual(paths, expected)
def add(self, t, flavor=None, topsrcdir=None):
t = dict(t)
t['flavor'] = flavor
if topsrcdir is None:
topsrcdir = self.topsrcdir
else:
topsrcdir = mozpath.normpath(topsrcdir)
path = mozpath.normpath(t['path'])
assert path.startswith(topsrcdir)
key = path[len(topsrcdir)+1:]
t['file_relpath'] = key
t['dir_relpath'] = mozpath.dirname(key)
self.tests_by_path[key].append(t)
def add(self, t, flavor=None, topsrcdir=None):
t = dict(t)
t["flavor"] = flavor
if topsrcdir is None:
topsrcdir = self.topsrcdir
else:
topsrcdir = mozpath.normpath(topsrcdir)
path = mozpath.normpath(t["path"])
assert mozpath.basedir(path, [topsrcdir])
key = path[len(topsrcdir) + 1 :]
t["file_relpath"] = key
t["dir_relpath"] = mozpath.dirname(key)
self.tests_by_path[key].append(t)
def test_test_manifest_install_to_subdir(self):
""" """
reader = self.reader('test-manifest-install-subdir')
objs = self.read_topsrcdir(reader)
self.assertEqual(len(objs), 1)
o = objs[0]
self.assertEqual(len(o.installs), 3)
self.assertEqual(o.manifest_relpath, "subdir.ini")
self.assertEqual(o.manifest_obj_relpath, "subdir/subdir.ini")
expected = [
mozpath.normpath(mozpath.join(o.install_prefix, "subdir/subdir.ini")),
mozpath.normpath(mozpath.join(o.install_prefix, "subdir/support.txt")),
mozpath.normpath(mozpath.join(o.install_prefix, "subdir/test_foo.html")),
]
paths = sorted([v[0] for v in o.installs.values()])
self.assertEqual(paths, expected)
def _process_generated_include(self, obj):
if obj.path.startswith('/'):
path = self.environment.topobjdir.replace('\\', '/') + obj.path
else:
path = mozpath.join(obj.srcdir, obj.path)
path = mozpath.normpath(path).replace('\\', '/')
srcdirConfig = self._get_config(obj.srcdir)
srcdirConfig.setdefault('generated_includes', []).append(path)
def validate_args(self, **kwargs):
from autotry import AutoTry
if not kwargs["paths"] and not kwargs["tests"] and not kwargs["tags"]:
print("Paths, tags, or tests must be specified as an argument to autotry.")
sys.exit(1)
if kwargs["platforms"] is None:
if 'AUTOTRY_PLATFORM_HINT' in os.environ:
kwargs["platforms"] = [os.environ['AUTOTRY_PLATFORM_HINT']]
else:
print("Platforms must be specified as an argument to autotry.")
sys.exit(1)
try:
platforms = self.normalise_list(kwargs["platforms"])
except ValueError as e:
print("Error parsing -p argument:\n%s" % e.message)
sys.exit(1)
try:
tests = (self.normalise_list(kwargs["tests"], allow_subitems=True)
if kwargs["tests"] else {})
except ValueError as e:
print("Error parsing -u argument (%s):\n%s" % (kwargs["tests"], e.message))
sys.exit(1)
try:
talos = (self.normalise_list(kwargs["talos"], allow_subitems=True)
if kwargs["talos"] else [])
except ValueError as e:
print("Error parsing -t argument:\n%s" % e.message)
sys.exit(1)
paths = []
for p in kwargs["paths"]:
p = mozpath.normpath(os.path.abspath(p))
if not (os.path.isdir(p) and p.startswith(self.topsrcdir)):
print('Specified path "%s" is not a directory under the srcdir,'
' unable to specify tests outside of the srcdir' % p)
sys.exit(1)
if len(p) <= len(self.topsrcdir):
print('Specified path "%s" is at the top of the srcdir and would'
' select all tests.' % p)
sys.exit(1)
paths.append(os.path.relpath(p, self.topsrcdir))
try:
tags = self.normalise_list(kwargs["tags"]) if kwargs["tags"] else []
except ValueError as e:
print("Error parsing --tags argument:\n%s" % e.message)
sys.exit(1)
extra_values = {k['dest'] for k in AutoTry.pass_through_arguments.values()}
extra_args = {k: v for k, v in kwargs.items()
if k in extra_values and v}
return kwargs["builds"], platforms, tests, talos, paths, tags, extra_args
from mozwebidlcodegen import (
WebIDLCodegenManager,
WebIDLCodegenManagerState,
)
from mozfile import NamedTemporaryFile
from mozunit import (
MockedOpen,
main,
)
OUR_DIR = mozpath.abspath(mozpath.dirname(__file__))
TOPSRCDIR = mozpath.normpath(mozpath.join(OUR_DIR, '..', '..', '..', '..'))
class TestWebIDLCodegenManager(unittest.TestCase):
TEST_STEMS = {
'Child',
'Parent',
'ExampleBinding',
'TestEvent',
}
@property
def _static_input_paths(self):
s = {mozpath.join(OUR_DIR, p) for p in os.listdir(OUR_DIR)
if p.endswith('.webidl')}
def _read_mozbuild(self, path, config, read_tiers, filesystem_absolute,
descend, metadata):
path = mozpath.normpath(path)
log(self._log, logging.DEBUG, 'read_mozbuild', {'path': path},
'Reading file: {path}')
if path in self._read_files:
log(self._log, logging.WARNING, 'read_already', {'path': path},
'File already read. Skipping: {path}')
return
self._read_files.add(path)
time_start = time.time()
topobjdir = config.topobjdir
if not mozpath.basedir(path, [config.topsrcdir]):
external = config.external_source_dir
if external and mozpath.basedir(path, [external]):
config = ConfigEnvironment.from_config_status(
mozpath.join(topobjdir, 'config.status'))
config.topsrcdir = external
config.external_source_dir = None
relpath = mozpath.relpath(path, config.topsrcdir)
reldir = mozpath.dirname(relpath)
if mozpath.dirname(relpath) == 'js/src' and \
not config.substs.get('JS_STANDALONE'):
config = ConfigEnvironment.from_config_status(
mozpath.join(topobjdir, reldir, 'config.status'))
config.topobjdir = topobjdir
config.external_source_dir = None
context = Context(VARIABLES, config)
sandbox = MozbuildSandbox(context, metadata=metadata)
sandbox.exec_file(path, filesystem_absolute=filesystem_absolute)
context.execution_time = time.time() - time_start
if self._sandbox_post_eval_cb:
self._sandbox_post_eval_cb(context)
# We first collect directories populated in variables.
dir_vars = ['DIRS']
if context.config.substs.get('ENABLE_TESTS', False) == '1':
dir_vars.append('TEST_DIRS')
dirs = [(v, context[v]) for v in dir_vars if v in context]
curdir = mozpath.dirname(path)
gyp_contexts = []
for target_dir in context['GYP_DIRS']:
gyp_dir = context['GYP_DIRS'][target_dir]
for v in ('input', 'variables'):
if not getattr(gyp_dir, v):
raise SandboxValidationError(
'Missing value for '
'GYP_DIRS["%s"].%s' % (target_dir, v), context)
# The make backend assumes contexts for sub-directories are
# emitted after their parent, so accumulate the gyp contexts.
# We could emit the parent context before processing gyp
# configuration, but we need to add the gyp objdirs to that context
# first.
from .gyp_reader import read_from_gyp
non_unified_sources = set()
for s in gyp_dir.non_unified_sources:
source = mozpath.normpath(mozpath.join(curdir, s))
if not os.path.exists(source):
raise SandboxValidationError('Cannot find %s.' % source,
context)
non_unified_sources.add(source)
for gyp_context in read_from_gyp(
context.config,
mozpath.join(curdir, gyp_dir.input),
mozpath.join(context.objdir, target_dir),
gyp_dir.variables,
non_unified_sources=non_unified_sources):
gyp_context.update(gyp_dir.sandbox_vars)
gyp_contexts.append(gyp_context)
for gyp_context in gyp_contexts:
if self._sandbox_post_eval_cb:
self._sandbox_post_eval_cb(gyp_context)
context['DIRS'].append(
mozpath.relpath(gyp_context.objdir, context.objdir))
yield context
for gyp_context in gyp_contexts:
yield gyp_context
# Traverse into referenced files.
# It's very tempting to use a set here. Unfortunately, the recursive
# make backend needs order preserved. Once we autogenerate all backend
# files, we should be able to convert this to a set.
recurse_info = OrderedDict()
for var, var_dirs in dirs:
for d in var_dirs:
if d in recurse_info:
raise SandboxValidationError(
'Directory (%s) registered multiple times in %s' %
(d, var), context)
recurse_info[d] = {}
if 'templates' in sandbox.metadata:
recurse_info[d]['templates'] = dict(
sandbox.metadata['templates'])
if 'exports' in sandbox.metadata:
sandbox.recompute_exports()
recurse_info[d]['exports'] = dict(
sandbox.metadata['exports'])
# We also have tiers whose members are directories.
if 'TIERS' in context:
if not read_tiers:
raise SandboxValidationError(
'TIERS defined but it should not be', context)
for tier, values in context['TIERS'].items():
# We don't descend into external directories because external by
# definition is external to the build system.
for d in values['regular']:
if d in recurse_info:
raise SandboxValidationError(
'Tier directory (%s) registered multiple '
'times in %s' % (d, tier), context)
recurse_info[d] = {'check_external': True}
if 'templates' in sandbox.metadata:
recurse_info[d]['templates'] = dict(
sandbox.metadata['templates'])
for relpath, child_metadata in recurse_info.items():
if 'check_external' in child_metadata:
relpath = '/' + relpath
child_path = sandbox.normalize_path(mozpath.join(
relpath, 'moz.build'),
srcdir=curdir)
# Ensure we don't break out of the topsrcdir. We don't do realpath
# because it isn't necessary. If there are symlinks in the srcdir,
# that's not our problem. We're not a hosted application: we don't
# need to worry about security too much.
if not is_read_allowed(child_path, context.config):
raise SandboxValidationError(
'Attempting to process file outside of allowed paths: %s' %
child_path, context)
if not descend:
continue
for res in self.read_mozbuild(child_path,
context.config,
read_tiers=False,
filesystem_absolute=True,
metadata=child_metadata):
yield res
self._execution_stack.pop()
def _repack(app_finder, l10n_finder, copier, formatter, non_chrome=set()):
app = LocaleManifestFinder(app_finder)
l10n = LocaleManifestFinder(l10n_finder)
# The code further below assumes there's only one locale replaced with
# another one.
if len(app.locales) > 1:
errors.fatal("Multiple app locales aren't supported: " +
",".join(app.locales))
if len(l10n.locales) > 1:
errors.fatal("Multiple l10n locales aren't supported: " +
",".join(l10n.locales))
locale = app.locales[0]
l10n_locale = l10n.locales[0]
# For each base directory, store what path a locale chrome package name
# corresponds to.
# e.g., for the following entry under app/chrome:
# locale foo en-US path/to/files
# keep track that the locale path for foo in app is
# app/chrome/path/to/files.
# As there may be multiple locale entries with the same base, but with
# different flags, that tracking takes the flags into account when there
# are some. Example:
# locale foo en-US path/to/files/win os=Win
# locale foo en-US path/to/files/mac os=Darwin
def key(entry):
if entry.flags:
return '%s %s' % (entry.name, entry.flags)
return entry.name
l10n_paths = {}
for e in l10n.entries:
if isinstance(e, ManifestChrome):
base = mozpath.basedir(e.path, app.bases)
l10n_paths.setdefault(base, {})
l10n_paths[base][key(e)] = e.path
# For chrome and non chrome files or directories, store what langpack path
# corresponds to a package path.
paths = {}
for e in app.entries:
if isinstance(e, ManifestEntryWithRelPath):
base = mozpath.basedir(e.path, app.bases)
if base not in l10n_paths:
errors.fatal("Locale doesn't contain %s/" % base)
# Allow errors to accumulate
continue
if key(e) not in l10n_paths[base]:
errors.fatal("Locale doesn't have a manifest entry for '%s'" %
e.name)
# Allow errors to accumulate
continue
paths[e.path] = l10n_paths[base][key(e)]
for pattern in non_chrome:
for base in app.bases:
path = mozpath.join(base, pattern)
left = set(p for p, f in app_finder.find(path))
right = set(p for p, f in l10n_finder.find(path))
for p in right:
paths[p] = p
for p in left - right:
paths[p] = None
# Create a new package, with non localized bits coming from the original
# package, and localized bits coming from the langpack.
packager = SimplePackager(formatter)
for p, f in app_finder:
if is_manifest(p):
# Remove localized manifest entries.
for e in [e for e in f if e.localized]:
f.remove(e)
# If the path is one that needs a locale replacement, use the
# corresponding file from the langpack.
path = None
if p in paths:
path = paths[p]
if not path:
continue
else:
base = mozpath.basedir(p, paths.keys())
if base:
subpath = mozpath.relpath(p, base)
path = mozpath.normpath(mozpath.join(paths[base], subpath))
if path:
files = [f for p, f in l10n_finder.find(path)]
if not len(files):
if base not in non_chrome:
finderBase = ""
if hasattr(l10n_finder, 'base'):
finderBase = l10n_finder.base
errors.error("Missing file: %s" %
os.path.join(finderBase, path))
else:
packager.add(path, files[0])
else:
packager.add(p, f)
# Add localized manifest entries from the langpack.
l10n_manifests = []
for base in set(e.base for e in l10n.entries):
m = ManifestFile(base, [e for e in l10n.entries if e.base == base])
path = mozpath.join(base, 'chrome.%s.manifest' % l10n_locale)
l10n_manifests.append((path, m))
bases = packager.get_bases()
for path, m in l10n_manifests:
base = mozpath.basedir(path, bases)
packager.add(path, m)
# Add a "manifest $path" entry in the top manifest under that base.
m = ManifestFile(base)
m.add(Manifest(base, mozpath.relpath(path, base)))
packager.add(mozpath.join(base, 'chrome.manifest'), m)
packager.close()
# Add any remaining non chrome files.
for pattern in non_chrome:
for base in bases:
for p, f in l10n_finder.find(mozpath.join(base, pattern)):
if not formatter.contains(p):
formatter.add(p, f)
# Transplant jar preloading information.
for path, log in app_finder.jarlogs.iteritems():
assert isinstance(copier[path], Jarrer)
copier[path].preload([l.replace(locale, l10n_locale) for l in log])
def _consume_jar_manifest(self, obj, defines):
# Ideally, this would all be handled somehow in the emitter, but
# this would require all the magic surrounding l10n and addons in
# the recursive make backend to die, which is not going to happen
# any time soon enough.
# Notably missing:
# - DEFINES from config/config.mk
# - L10n support
# - The equivalent of -e when USE_EXTENSION_MANIFEST is set in
# moz.build, but it doesn't matter in dist/bin.
pp = Preprocessor()
pp.context.update(defines)
pp.context.update(self.environment.defines)
pp.context.update(
AB_CD='en-US',
BUILD_FASTER=1,
)
pp.out = JarManifestParser()
pp.do_include(obj.path)
self.backend_input_files |= pp.includes
for jarinfo in pp.out:
install_target = obj.install_target
if jarinfo.base:
install_target = mozpath.join(install_target, jarinfo.base)
for e in jarinfo.entries:
if e.is_locale:
if jarinfo.relativesrcdir:
path = mozpath.join(self.environment.topsrcdir,
jarinfo.relativesrcdir)
else:
path = mozpath.dirname(obj.path)
src = mozpath.join(path, 'en-US', e.source)
elif e.source.startswith('/'):
src = mozpath.join(self.environment.topsrcdir,
e.source[1:])
else:
src = mozpath.join(mozpath.dirname(obj.path), e.source)
if '*' in e.source:
if e.preprocess:
raise Exception('%s: Wildcards are not supported with '
'preprocessing' % obj.path)
def _prefix(s):
for p in s.split('/'):
if '*' not in p:
yield p + '/'
prefix = ''.join(_prefix(src))
self._install_manifests[install_target] \
.add_pattern_symlink(
prefix,
src[len(prefix):],
mozpath.join(jarinfo.name, e.output))
continue
if not os.path.exists(src):
if e.is_locale:
raise Exception('%s: Cannot find %s' %
(obj.path, e.source))
if e.source.startswith('/'):
src = mozpath.join(self.environment.topobjdir,
e.source[1:])
else:
# This actually gets awkward if the jar.mn is not
# in the same directory as the moz.build declaring
# it, but it's how it works in the recursive make,
# not that anything relies on that, but it's simpler.
src = mozpath.join(obj.objdir, e.source)
self._dependencies['install-%s' % install_target] \
.append(mozpath.relpath(
src, self.environment.topobjdir))
if e.preprocess:
kwargs = {}
if src.endswith('.css'):
kwargs['marker'] = '%'
self._add_preprocess(obj,
src,
mozpath.join(
jarinfo.name,
mozpath.dirname(e.output)),
mozpath.basename(e.output),
defines=defines,
**kwargs)
else:
self._install_manifests[install_target].add_symlink(
src, mozpath.join(jarinfo.name, e.output))
manifest = mozpath.normpath(
mozpath.join(install_target, jarinfo.name))
manifest += '.manifest'
for m in jarinfo.chrome_manifests:
self._manifest_entries[manifest].append(
m.replace('%',
mozpath.basename(jarinfo.name) + '/'))
if jarinfo.name != 'chrome':
manifest = mozpath.normpath(
mozpath.join(install_target, 'chrome.manifest'))
entry = 'manifest %s.manifest' % jarinfo.name
if entry not in self._manifest_entries[manifest]:
self._manifest_entries[manifest].append(entry)
def _read_mozbuild(self, path, config, descend, metadata):
path = mozpath.normpath(path)
log(self._log, logging.DEBUG, 'read_mozbuild', {'path': path},
'Reading file: {path}')
if path in self._read_files:
log(self._log, logging.WARNING, 'read_already', {'path': path},
'File already read. Skipping: {path}')
return
self._read_files.add(path)
time_start = time.time()
topobjdir = config.topobjdir
if not mozpath.basedir(path, [config.topsrcdir]):
external = config.external_source_dir
if external and mozpath.basedir(path, [external]):
config = ConfigEnvironment.from_config_status(
mozpath.join(topobjdir, 'config.status'))
config.topsrcdir = external
config.external_source_dir = None
relpath = mozpath.relpath(path, config.topsrcdir)
reldir = mozpath.dirname(relpath)
if mozpath.dirname(relpath) == 'js/src' and \
not config.substs.get('JS_STANDALONE'):
config = ConfigEnvironment.from_config_status(
mozpath.join(topobjdir, reldir, 'config.status'))
config.topobjdir = topobjdir
config.external_source_dir = None
context = Context(VARIABLES, config)
sandbox = MozbuildSandbox(context, metadata=metadata)
sandbox.exec_file(path)
context.execution_time = time.time() - time_start
# Yield main context before doing any processing. This gives immediate
# consumers an opportunity to change state before our remaining
# processing is performed.
yield context
# We first collect directories populated in variables.
dir_vars = ['DIRS']
if context.config.substs.get('ENABLE_TESTS', False) == '1':
dir_vars.append('TEST_DIRS')
dirs = [(v, context[v]) for v in dir_vars if v in context]
curdir = mozpath.dirname(path)
gyp_contexts = []
for target_dir in context.get('GYP_DIRS', []):
gyp_dir = context['GYP_DIRS'][target_dir]
for v in ('input', 'variables'):
if not getattr(gyp_dir, v):
raise SandboxValidationError(
'Missing value for '
'GYP_DIRS["%s"].%s' % (target_dir, v), context)
# The make backend assumes contexts for sub-directories are
# emitted after their parent, so accumulate the gyp contexts.
# We could emit the parent context before processing gyp
# configuration, but we need to add the gyp objdirs to that context
# first.
from .gyp_reader import read_from_gyp
non_unified_sources = set()
for s in gyp_dir.non_unified_sources:
source = SourcePath(context, s)
if not os.path.exists(source.full_path):
raise SandboxValidationError('Cannot find %s.' % source,
context)
non_unified_sources.add(source)
for gyp_context in read_from_gyp(
context.config,
mozpath.join(curdir, gyp_dir.input),
mozpath.join(context.objdir, target_dir),
gyp_dir.variables,
non_unified_sources=non_unified_sources):
gyp_context.update(gyp_dir.sandbox_vars)
gyp_contexts.append(gyp_context)
for gyp_context in gyp_contexts:
context['DIRS'].append(
mozpath.relpath(gyp_context.objdir, context.objdir))
sandbox.subcontexts.append(gyp_context)
for subcontext in sandbox.subcontexts:
yield subcontext
# Traverse into referenced files.
# It's very tempting to use a set here. Unfortunately, the recursive
# make backend needs order preserved. Once we autogenerate all backend
# files, we should be able to convert this to a set.
recurse_info = OrderedDict()
for var, var_dirs in dirs:
for d in var_dirs:
if d in recurse_info:
raise SandboxValidationError(
'Directory (%s) registered multiple times in %s' %
(mozpath.relpath(d.full_path, context.srcdir), var),
context)
recurse_info[d] = {}
for key in sandbox.metadata:
if key == 'exports':
sandbox.recompute_exports()
recurse_info[d][key] = dict(sandbox.metadata[key])
for path, child_metadata in recurse_info.items():
child_path = path.join('moz.build').full_path
# Ensure we don't break out of the topsrcdir. We don't do realpath
# because it isn't necessary. If there are symlinks in the srcdir,
# that's not our problem. We're not a hosted application: we don't
# need to worry about security too much.
if not is_read_allowed(child_path, context.config):
raise SandboxValidationError(
'Attempting to process file outside of allowed paths: %s' %
child_path, context)
if not descend:
continue
for res in self.read_mozbuild(child_path,
context.config,
metadata=child_metadata):
yield res
self._execution_stack.pop()
def read_from_gyp(config, path, output, vars, non_unified_sources=set()):
"""Read a gyp configuration and emits GypContexts for the backend to
process.
config is a ConfigEnvironment, path is the path to a root gyp configuration
file, output is the base path under which the objdir for the various gyp
dependencies will be, and vars a dict of variables to pass to the gyp
processor.
"""
time_start = time.time()
all_sources = set()
# gyp expects plain str instead of unicode. The frontend code gives us
# unicode strings, so convert them.
path = encode(path)
str_vars = dict((name, encode(value)) for name, value in vars.items())
params = {
b'parallel': False,
b'generator_flags': {},
b'build_files': [path],
}
# Files that gyp_chromium always includes
includes = [encode(mozpath.join(script_dir, 'common.gypi'))]
finder = FileFinder(chrome_src, find_executables=False)
includes.extend(
encode(mozpath.join(chrome_src, name))
for name, _ in finder.find('*/supplement.gypi'))
# Read the given gyp file and its dependencies.
generator, flat_list, targets, data = \
gyp.Load([path], format=b'mozbuild',
default_variables=str_vars,
includes=includes,
depth=encode(mozpath.dirname(path)),
params=params)
# Process all targets from the given gyp files and its dependencies.
# The path given to AllTargets needs to use os.sep, while the frontend code
# gives us paths normalized with forward slash separator.
for target in gyp.common.AllTargets(flat_list, targets,
path.replace(b'/', os.sep)):
build_file, target_name, toolset = gyp.common.ParseQualifiedTarget(
target)
# Each target is given its own objdir. The base of that objdir
# is derived from the relative path from the root gyp file path
# to the current build_file, placed under the given output
# directory. Since several targets can be in a given build_file,
# separate them in subdirectories using the build_file basename
# and the target_name.
reldir = mozpath.relpath(mozpath.dirname(build_file),
mozpath.dirname(path))
subdir = '%s_%s' % (
mozpath.splitext(mozpath.basename(build_file))[0],
target_name,
)
# Emit a context for each target.
context = GypContext(
config,
mozpath.relpath(mozpath.join(output, reldir, subdir),
config.topobjdir))
context.add_source(mozpath.abspath(build_file))
# The list of included files returned by gyp are relative to build_file
for f in data[build_file]['included_files']:
context.add_source(
mozpath.abspath(mozpath.join(mozpath.dirname(build_file), f)))
context['IS_GYP_DIR'] = True
spec = targets[target]
# Derive which gyp configuration to use based on MOZ_DEBUG.
c = 'Debug' if config.substs['MOZ_DEBUG'] else 'Release'
if c not in spec['configurations']:
raise RuntimeError('Missing %s gyp configuration for target %s '
'in %s' % (c, target_name, build_file))
target_conf = spec['configurations'][c]
if spec['type'] == 'none':
continue
elif spec['type'] == 'static_library':
# Remove leading 'lib' from the target_name if any, and use as
# library name.
name = spec['target_name']
if name.startswith('lib'):
name = name[3:]
# The context expects an unicode string.
context['LIBRARY_NAME'] = name.decode('utf-8')
# gyp files contain headers and asm sources in sources lists.
sources = set(
mozpath.normpath(mozpath.join(context.srcdir, f))
for f in spec.get('sources', [])
if mozpath.splitext(f)[-1] != '.h')
asm_sources = set(f for f in sources if f.endswith('.S'))
unified_sources = sources - non_unified_sources - asm_sources
sources -= unified_sources
all_sources |= sources
# The context expects alphabetical order when adding sources
context['SOURCES'] = alphabetical_sorted(sources)
context['UNIFIED_SOURCES'] = alphabetical_sorted(unified_sources)
for define in target_conf.get('defines', []):
if '=' in define:
name, value = define.split('=', 1)
context['DEFINES'][name] = value
else:
context['DEFINES'][define] = True
for include in target_conf.get('include_dirs', []):
# moz.build expects all LOCAL_INCLUDES to exist, so ensure they do.
#
# NB: gyp files sometimes have actual absolute paths (e.g.
# /usr/include32) and sometimes paths that moz.build considers
# absolute, i.e. starting from topsrcdir. There's no good way
# to tell them apart here, and the actual absolute paths are
# likely bogus. In any event, actual absolute paths will be
# filtered out by trying to find them in topsrcdir.
if include.startswith('/'):
resolved = mozpath.abspath(
mozpath.join(config.topsrcdir, include[1:]))
else:
resolved = mozpath.abspath(
mozpath.join(mozpath.dirname(build_file), include))
if not os.path.exists(resolved):
continue
context['LOCAL_INCLUDES'] += [include]
context['EXTRA_ASSEMBLER_FLAGS'] = target_conf.get(
'asflags_mozilla', [])
context['EXTRA_COMPILE_FLAGS'] = target_conf.get(
'cflags_mozilla', [])
else:
# Ignore other types than static_library because we don't have
# anything using them, and we're not testing them. They can be
# added when that becomes necessary.
raise NotImplementedError('Unsupported gyp target type: %s' %
spec['type'])
# Add some features to all contexts. Put here in case LOCAL_INCLUDES
# order matters.
context['LOCAL_INCLUDES'] += [
'/ipc/chromium/src',
'/ipc/glue',
]
context['GENERATED_INCLUDES'] += ['/ipc/ipdl/_ipdlheaders']
# These get set via VC project file settings for normal GYP builds.
if config.substs['OS_TARGET'] == 'WINNT':
context['DEFINES']['UNICODE'] = True
context['DEFINES']['_UNICODE'] = True
context['DISABLE_STL_WRAPPING'] = True
context.execution_time = time.time() - time_start
yield context
time_start = time.time()
def post_dispatch_handler(context, handler, instance, result, start_time,
end_time, depth, args):
"""Perform global operations after command dispatch.
For now, we will use this to handle build system telemetry.
"""
# Don't write telemetry data if this mach command was invoked as part of another
# mach command.
if depth != 1 or os.environ.get('MACH_MAIN_PID') != str(os.getpid()):
return
# Don't write telemetry data for 'mach' when 'DISABLE_TELEMETRY' is set.
if os.environ.get('DISABLE_TELEMETRY') == '1':
return
# We have not opted-in to telemetry
if not context.settings.build.telemetry:
return
from mozbuild.telemetry import gather_telemetry
from mozbuild.base import MozbuildObject
import mozpack.path as mozpath
if not isinstance(instance, MozbuildObject):
instance = MozbuildObject.from_environment()
try:
substs = instance.substs
except Exception:
substs = {}
command_attrs = getattr(context, 'command_attrs', {})
# We gather telemetry for every operation.
paths = {
instance.topsrcdir: '$topsrcdir/',
instance.topobjdir: '$topobjdir/',
mozpath.normpath(os.path.expanduser('~')): '$HOME/',
}
# This might override one of the existing entries, that's OK.
# We don't use a sigil here because we treat all arguments as potentially relative
# paths, so we'd like to get them back as they were specified.
paths[mozpath.normpath(os.getcwd())] = ''
data = gather_telemetry(command=handler.name,
success=(result == 0),
start_time=start_time,
end_time=end_time,
mach_context=context,
substs=substs,
command_attrs=command_attrs,
paths=paths)
if data:
telemetry_dir = os.path.join(get_state_dir(), 'telemetry')
try:
os.mkdir(telemetry_dir)
except OSError as e:
if e.errno != errno.EEXIST:
raise
outgoing_dir = os.path.join(telemetry_dir, 'outgoing')
try:
os.mkdir(outgoing_dir)
except OSError as e:
if e.errno != errno.EEXIST:
raise
with open(os.path.join(outgoing_dir,
str(uuid.uuid4()) + '.json'), 'w') as f:
json.dump(data, f, sort_keys=True)
if should_skip_telemetry_submission(handler):
return True
state_dir = get_state_dir()
machpath = os.path.join(instance.topsrcdir, 'mach')
with open(os.devnull, 'wb') as devnull:
subprocess.Popen([
sys.executable, machpath, 'python', '--no-virtualenv',
os.path.join(topsrcdir, 'build', 'submit_telemetry_data.py'),
state_dir
],
stdout=devnull,
stderr=devnull)
def test_normpath(self):
self.assertEqual(
normpath(os.path.join('foo', 'bar', 'baz', '..', 'qux')),
'foo/bar/qux')
def normalize(path):
path = mozpath.normpath(path)
if os.path.isabs(path):
return path
return mozpath.join(root, path)
def config_status(config, execute=True):
# Sanitize config data to feed config.status
# Ideally, all the backend and frontend code would handle the booleans, but
# there are so many things involved, that it's easier to keep config.status
# untouched for now.
def sanitize_config(v):
if v is True:
return '1'
if v is False:
return ''
# Serialize types that look like lists and tuples as lists.
if not isinstance(v, (bytes, six.text_type, dict)) and isinstance(
v, Iterable):
return list(v)
return v
sanitized_config = {}
sanitized_config['substs'] = {
k: sanitize_config(v)
for k, v in six.iteritems(config)
if k not in ('DEFINES', 'TOPSRCDIR', 'TOPOBJDIR', 'CONFIG_STATUS_DEPS',
'OLD_CONFIGURE_SUBSTS', 'OLD_CONFIGURE_DEFINES')
}
for k, v in config['OLD_CONFIGURE_SUBSTS']:
sanitized_config['substs'][k] = sanitize_config(v)
sanitized_config['defines'] = {
k: sanitize_config(v)
for k, v in six.iteritems(config['DEFINES'])
}
for k, v in config['OLD_CONFIGURE_DEFINES']:
sanitized_config['defines'][k] = sanitize_config(v)
sanitized_config['topsrcdir'] = config['TOPSRCDIR']
sanitized_config['topobjdir'] = config['TOPOBJDIR']
sanitized_config['mozconfig'] = config.get('MOZCONFIG')
if not check_unicode(sanitized_config):
print("Configuration should be all unicode.", file=sys.stderr)
print("Please file a bug for the above.", file=sys.stderr)
sys.exit(1)
# Some values in sanitized_config also have more complex types, such as
# EnumString, which using when calling config_status would currently
# break the build, as well as making it inconsistent with re-running
# config.status, for which they are normalized to plain strings via
# indented_repr. Likewise for non-dict non-string iterables being
# converted to lists.
def normalize(obj):
if isinstance(obj, dict):
return {k: normalize(v) for k, v in six.iteritems(obj)}
if isinstance(obj, six.text_type):
return six.text_type(obj)
if isinstance(obj, Iterable):
return [normalize(o) for o in obj]
return obj
sanitized_config = normalize(sanitized_config)
# Create config.status. Eventually, we'll want to just do the work it does
# here, when we're able to skip configure tests/use cached results/not rely
# on autoconf.
with codecs.open('config.status', 'w', 'utf-8') as fh:
fh.write(
textwrap.dedent('''\
#!%(python)s
# coding=utf-8
from __future__ import unicode_literals
''') % {'python': config['PYTHON3']})
for k, v in sorted(six.iteritems(sanitized_config)):
fh.write('%s = ' % k)
write_indented_repr(fh, v)
fh.write("__all__ = ['topobjdir', 'topsrcdir', 'defines', "
"'substs', 'mozconfig']")
if execute:
fh.write(
textwrap.dedent('''
if __name__ == '__main__':
from mozbuild.util import patch_main
patch_main()
from mozbuild.config_status import config_status
args = dict([(name, globals()[name]) for name in __all__])
config_status(**args)
'''))
partial_config = PartialConfigEnvironment(config['TOPOBJDIR'])
partial_config.write_vars(sanitized_config)
# Write out a file so the build backend knows to re-run configure when
# relevant Python changes.
with io.open('config_status_deps.in', 'w', encoding='utf-8',
newline='\n') as fh:
for f in sorted(
itertools.chain(
config['CONFIG_STATUS_DEPS'],
iter_modules_in_path(config['TOPOBJDIR'],
config['TOPSRCDIR']))):
fh.write('%s\n' % mozpath.normpath(f))
# Other things than us are going to run this file, so we need to give it
# executable permissions.
os.chmod('config.status', 0o755)
if execute:
from mozbuild.config_status import config_status
return config_status(args=[], **sanitized_config)
return 0
def convert_support_files(self, test, install_root, manifest_dir, out_dir):
# Arguments:
# test - The test object to process.
# install_root - The directory under $objdir/_tests that will contain
# the tests for this harness (examples are "testing/mochitest",
# "xpcshell").
# manifest_dir - Absoulute path to the (srcdir) directory containing the
# manifest that included this test
# out_dir - The path relative to $objdir/_tests used as the destination for the
# test, based on the relative path to the manifest in the srcdir,
# the install_root, and 'install-to-subdir', if present in the manifest.
info = TestInstallInfo()
for thing, seen in self._fields:
value = test.get(thing, '')
# We need to memoize on the basis of both the path and the output
# directory for the benefit of tests specifying 'install-to-subdir'.
if (value, out_dir) in seen:
continue
seen.add((value, out_dir))
for pattern in value.split():
if thing == 'generated-files':
info.external_installs.add(
mozpath.normpath(mozpath.join(out_dir, pattern)))
# '!' indicates our syntax for inter-directory support file
# dependencies. These receive special handling in the backend.
elif pattern[0] == '!':
info.deferred_installs.add(pattern)
# We only support globbing on support-files because
# the harness doesn't support * for head and tail.
elif '*' in pattern and thing == 'support-files':
info.pattern_installs.append(
(manifest_dir, pattern, out_dir))
# "absolute" paths identify files that are to be
# placed in the install_root directory (no globs)
elif pattern[0] == '/':
full = mozpath.normpath(
mozpath.join(manifest_dir, mozpath.basename(pattern)))
info.installs.append(
(full, mozpath.join(install_root, pattern[1:])))
else:
full = mozpath.normpath(mozpath.join(
manifest_dir, pattern))
dest_path = mozpath.join(out_dir, pattern)
# If the path resolves to a different directory
# tree, we take special behavior depending on the
# entry type.
if not full.startswith(manifest_dir):
# If it's a support file, we install the file
# into the current destination directory.
# This implementation makes installing things
# with custom prefixes impossible. If this is
# needed, we can add support for that via a
# special syntax later.
if thing == 'support-files':
dest_path = mozpath.join(out_dir,
os.path.basename(pattern))
# If it's not a support file, we ignore it.
# This preserves old behavior so things like
# head files doesn't get installed multiple
# times.
else:
continue
info.installs.append((full, mozpath.normpath(dest_path)))
return info
def resolve_tests(self, paths=None, flavor=None, subsuite=None, under_path=None,
tags=None):
"""Resolve tests from an identifier.
This is a generator of dicts describing each test.
``paths`` can be an iterable of values to use to identify tests to run.
If an entry is a known test file, tests associated with that file are
returned (there may be multiple configurations for a single file). If
an entry is a directory, or a prefix of a directory containing tests,
all tests in that directory are returned. If the string appears in a
known test file, that test file is considered. If the path contains
a wildcard pattern, tests matching that pattern are returned.
If ``under_path`` is a string, it will be used to filter out tests that
aren't in the specified path prefix relative to topsrcdir or the
test's installed dir.
If ``flavor`` is a string, it will be used to filter returned tests
to only be the flavor specified. A flavor is something like
``xpcshell``.
If ``subsuite`` is a string, it will be used to filter returned tests
to only be in the subsuite specified.
If ``tags`` are specified, they will be used to filter returned tests
to only those with a matching tag.
"""
if tags:
tags = set(tags)
def fltr(tests):
for test in tests:
if flavor:
if flavor == 'devtools' and test.get('flavor') != 'browser-chrome':
continue
if flavor != 'devtools' and test.get('flavor') != flavor:
continue
if subsuite and test.get('subsuite') != subsuite:
continue
if tags and not (tags & set(test.get('tags', '').split())):
continue
if under_path and not test['file_relpath'].startswith(under_path):
continue
# Make a copy so modifications don't change the source.
yield dict(test)
paths = paths or []
paths = [mozpath.normpath(p) for p in paths]
if not paths:
paths = [None]
candidate_paths = set()
if flavor in (None, 'puppeteer') and any(self.is_puppeteer_path(p) for p in paths):
self.add_puppeteer_manifest_data()
if flavor in (None, 'web-platform-tests') and any(self.is_wpt_path(p) for p in paths):
self.add_wpt_manifest_data()
for path in sorted(paths):
if path is None:
candidate_paths |= set(self._tests_by_path.keys())
continue
if '*' in path:
candidate_paths |= {p for p in self._tests_by_path
if mozpath.match(p, path)}
continue
# If the path is a directory, or the path is a prefix of a directory
# containing tests, pull in all tests in that directory.
if (path in self._test_dirs or
any(p.startswith(path) for p in self._tests_by_path)):
candidate_paths |= {p for p in self._tests_by_path
if p.startswith(path)}
continue
# If it's a test file, add just that file.
candidate_paths |= {p for p in self._tests_by_path if path in p}
for p in sorted(candidate_paths):
tests = self._tests_by_path[p]
for test in fltr(tests):
yield test
def resolve_tests(self, cwd=None, **kwargs):
"""Resolve tests from an identifier.
This is a generator of dicts describing each test. All arguments are
optional.
Paths in returned tests are automatically translated to the paths in
the _tests directory under the object directory.
Args:
cwd (str):
If specified, we will limit our results to tests under this
directory. The directory should be defined as an absolute path
under topsrcdir or topobjdir.
paths (list):
An iterable of values to use to identify tests to run. If an
entry is a known test file, tests associated with that file are
returned (there may be multiple configurations for a single
file). If an entry is a directory, or a prefix of a directory
containing tests, all tests in that directory are returned. If
the string appears in a known test file, that test file is
considered. If the path contains a wildcard pattern, tests
matching that pattern are returned.
under_path (str):
If specified, will be used to filter out tests that aren't in
the specified path prefix relative to topsrcdir or the test's
installed dir.
flavor (str):
If specified, will be used to filter returned tests to only be
the flavor specified. A flavor is something like ``xpcshell``.
subsuite (str):
If specified will be used to filter returned tests to only be
in the subsuite specified. To filter only tests that *don't*
have any subsuite, pass the string 'undefined'.
tags (list):
If specified, will be used to filter out tests that don't contain
a matching tag.
"""
if cwd:
norm_cwd = mozpath.normpath(cwd)
norm_srcdir = mozpath.normpath(self.topsrcdir)
norm_objdir = mozpath.normpath(self.topobjdir)
reldir = None
if norm_cwd.startswith(norm_objdir):
reldir = norm_cwd[len(norm_objdir) + 1:]
elif norm_cwd.startswith(norm_srcdir):
reldir = norm_cwd[len(norm_srcdir) + 1:]
kwargs["under_path"] = reldir
rewrite_base = None
for test in self._resolve(**kwargs):
rewrite_base = self.test_rewrites.get(test["flavor"], None)
if rewrite_base:
rewrite_base = os.path.join(self.topobjdir,
os.path.normpath(rewrite_base))
yield rewrite_test_base(test, rewrite_base)
else:
yield test
def _resolve(self,
paths=None,
flavor="",
subsuite=None,
under_path=None,
tags=None):
"""Given parameters, resolve them to produce an appropriate list of tests.
Args:
paths (list):
By default, set to None. If provided as a list of paths, then
this method will attempt to load the appropriate set of tests
that live in this path.
flavor (string):
By default, an empty string. If provided as a string, then this
method will attempt to load tests that belong to this flavor.
Additional filtering also takes the flavor into consideration.
subsuite (string):
By default, set to None. If provided as a string, then this value
is used to perform filtering of a candidate set of tests.
"""
if tags:
tags = set(tags)
def fltr(tests):
"""Filters tests based on several criteria.
Args:
tests (list):
List of tests that belong to the same candidate path.
Returns:
test (dict):
If the test survived the filtering process, it is returned
as a valid test.
"""
for test in tests:
if flavor:
if flavor == "devtools" and test.get(
"flavor") != "browser-chrome":
continue
if flavor != "devtools" and test.get("flavor") != flavor:
continue
if subsuite and test.get("subsuite", "undefined") != subsuite:
continue
if tags and not (tags & set(test.get("tags", "").split())):
continue
if under_path and not test["file_relpath"].startswith(
under_path):
continue
# Make a copy so modifications don't change the source.
yield dict(test)
paths = paths or []
paths = [mozpath.normpath(p) for p in paths]
if not paths:
paths = [None]
if flavor in ("", "puppeteer",
None) and (any(self.is_puppeteer_path(p)
for p in paths) or paths == [None]):
self.add_puppeteer_manifest_data()
if flavor in ("", "web-platform-tests",
None) and (any(self.is_wpt_path(p)
for p in paths) or paths == [None]):
self.add_wpt_manifest_data()
candidate_paths = set()
for path in sorted(paths):
if path is None:
candidate_paths |= set(self.tests_by_path.keys())
continue
if "*" in path:
candidate_paths |= {
p
for p in self.tests_by_path if mozpath.match(p, path)
}
continue
# If the path is a directory, or the path is a prefix of a directory
# containing tests, pull in all tests in that directory.
if path in self.test_dirs or any(
p.startswith(path) for p in self.tests_by_path):
candidate_paths |= {
p
for p in self.tests_by_path if p.startswith(path)
}
continue
# If the path is a manifest, add all tests defined in that manifest.
if any(path.endswith(e) for e in (".ini", ".list")):
key = "manifest" if os.path.isabs(path) else "manifest_relpath"
candidate_paths |= {
t["file_relpath"]
for t in self.tests if mozpath.normpath(t[key]) == path
}
continue
# If it's a test file, add just that file.
candidate_paths |= {p for p in self.tests_by_path if path in p}
for p in sorted(candidate_paths):
tests = self.tests_by_path[p]
for test in fltr(tests):
yield test
def __init__(self, config):
self.config = config
self.topsrcdir = mozpath.normpath(config.topsrcdir)
self.tests_by_path = defaultdict(list)
from __future__ import absolute_import, unicode_literals
import json
import os
import mozunit
import mozpack.path as mozpath
import pytest
from mozlint.result import Issue, ResultSummary
from mozlint import formatters
NORMALISED_PATHS = {
'abc': os.path.normpath('a/b/c.txt'),
'def': os.path.normpath('d/e/f.txt'),
'cwd': mozpath.normpath(os.getcwd()),
}
EXPECTED = {
'compact': {
'kwargs': {},
'format': """
a/b/c.txt: line 1, Error - oh no foo (foo)
a/b/c.txt: line 4, col 10, Error - oh no baz (baz)
d/e/f.txt: line 4, col 2, Warning - oh no bar (bar-not-allowed)
3 problems
""".strip(),
},
'stylish': {
'kwargs': {
def test_normpath(self):
self.assertEqual(
normpath(self.SEP.join(("foo", "bar", "baz", "..", "qux"))),
"foo/bar/qux")
def test_migration(cmd, obj_dir, to_test, references):
'''Test the given recipe.
This creates a workdir by l10n-merging gecko-strings and the m-c source,
to mimmic gecko-strings after the patch to test landed.
It then runs the recipe with a gecko-strings clone as localization, both
dry and wet.
It inspects the generated commits, and shows a diff between the merged
reference and the generated content.
The diff is intended to be visually inspected. Some changes might be
expected, in particular when formatting of the en-US strings is different.
'''
rv = 0
migration_name = os.path.splitext(os.path.split(to_test)[1])[0]
work_dir = mozpath.join(obj_dir, migration_name)
if os.path.exists(work_dir):
shutil.rmtree(work_dir)
os.makedirs(mozpath.join(work_dir, 'reference'))
l10n_toml = mozpath.join(cmd.topsrcdir, 'browser', 'locales', 'l10n.toml')
pc = TOMLParser().parse(l10n_toml, env={'l10n_base': work_dir})
pc.set_locales(['reference'])
files = ProjectFiles('reference', [pc])
for ref in references:
if ref != mozpath.normpath(ref):
cmd.log(
logging.ERROR, 'fluent-migration-test', {
'file': to_test,
'ref': ref,
}, 'Reference path "{ref}" needs to be normalized for {file}')
rv = 1
continue
full_ref = mozpath.join(work_dir, 'reference', ref)
m = files.match(full_ref)
if m is None:
raise ValueError("Bad reference path: " + ref)
m_c_path = m[1]
g_s_path = mozpath.join(work_dir, 'gecko-strings', ref)
resources = [
b'' if not os.path.exists(f) else open(f, 'rb').read()
for f in (g_s_path, m_c_path)
]
ref_dir = os.path.dirname(full_ref)
if not os.path.exists(ref_dir):
os.makedirs(ref_dir)
open(full_ref, 'wb').write(merge_channels(ref, resources))
client = hglib.clone(source=mozpath.join(get_state_dir(), 'gecko-strings'),
dest=mozpath.join(work_dir, 'en-US'))
client.open()
old_tip = client.tip().node
run_migration = [
cmd._virtualenv_manager.python_path, '-m', 'fluent.migrate.tool',
'--lang', 'en-US', '--reference-dir',
mozpath.join(work_dir, 'reference'), '--localization-dir',
mozpath.join(work_dir, 'en-US'), '--dry-run',
'fluent_migrations.' + migration_name
]
cmd.run_process(
run_migration,
cwd=work_dir,
line_handler=print,
)
# drop --dry-run
run_migration.pop(-2)
cmd.run_process(
run_migration,
cwd=work_dir,
line_handler=print,
)
tip = client.tip().node
if old_tip == tip:
cmd.log(logging.WARN, 'fluent-migration-test', {
'file': to_test,
}, 'No migration applied for {file}')
return rv
for ref in references:
cmd.run_process([
'diff',
'-u',
'-B',
mozpath.join(work_dir, 'reference', ref),
mozpath.join(work_dir, 'en-US', ref),
],
ensure_exit_code=False,
line_handler=print)
messages = [l.desc for l in client.log('::{} - ::{}'.format(tip, old_tip))]
bug = re.search('[0-9]{5,}', migration_name).group()
# Just check first message for bug number, they're all following the same pattern
if bug not in messages[0]:
rv = 1
cmd.log(logging.ERROR, 'fluent-migration-test', {
'file': to_test,
}, 'Missing or wrong bug number for {file}')
if any('part {}'.format(n + 1) not in msg
for n, msg in enumerate(messages)):
rv = 1
cmd.log(logging.ERROR, 'fluent-migration-test', {
'file': to_test,
}, 'Commit messages should have "part {{index}}" for {file}')
return rv
def path(self):
return mozpath.normpath(mozpath.join(self.base,
self.relpath))
def read_reftest_manifest(context, manifest_path):
path = mozpath.normpath(mozpath.join(context.srcdir, manifest_path))
manifest = reftest.ReftestManifest(finder=context._finder)
manifest.load(path)
return manifest
def repackage_mar(topsrcdir,
package,
mar,
output,
arch=None,
mar_channel_id=None):
if not zipfile.is_zipfile(package) and not tarfile.is_tarfile(package):
raise Exception("Package file %s is not a valid .zip or .tar file." %
package)
if arch and arch not in _BCJ_OPTIONS:
raise Exception(
"Unknown architecture {}, available architectures: {}".format(
arch, list(_BCJ_OPTIONS.keys())))
ensureParentDir(output)
tmpdir = tempfile.mkdtemp()
try:
if tarfile.is_tarfile(package):
z = tarfile.open(package)
z.extractall(tmpdir)
filelist = z.getnames()
z.close()
else:
z = zipfile.ZipFile(package)
z.extractall(tmpdir)
filelist = z.namelist()
z.close()
toplevel_dirs = set([mozpath.split(f)[0] for f in filelist])
excluded_stuff = set(
[" ", ".background", ".DS_Store", ".VolumeIcon.icns"])
toplevel_dirs = toplevel_dirs - excluded_stuff
# Make sure the .zip file just contains a directory like 'firefox/' at
# the top, and find out what it is called.
if len(toplevel_dirs) != 1:
raise Exception(
"Package file is expected to have a single top-level directory"
"(eg: 'firefox'), not: %s" % toplevel_dirs)
ffxdir = mozpath.join(tmpdir, toplevel_dirs.pop())
make_full_update = mozpath.join(
topsrcdir, "tools/update-packaging/make_full_update.sh")
env = os.environ.copy()
env["MOZ_PRODUCT_VERSION"] = get_application_ini_value(
tmpdir, "App", "Version")
env["MAR"] = mozpath.normpath(mar)
if arch:
env["BCJ_OPTIONS"] = " ".join(_BCJ_OPTIONS[arch])
if mar_channel_id:
env["MAR_CHANNEL_ID"] = mar_channel_id
# The Windows build systems have xz installed but it isn't in the path
# like it is on Linux and Mac OS X so just use the XZ env var so the mar
# generation scripts can find it.
xz_path = mozpath.join(topsrcdir, "xz/xz.exe")
if os.path.exists(xz_path):
env["XZ"] = mozpath.normpath(xz_path)
cmd = [make_full_update, output, ffxdir]
if sys.platform == "win32":
# make_full_update.sh is a bash script, and Windows needs to
# explicitly call out the shell to execute the script from Python.
cmd.insert(0, env["MOZILLABUILD"] + "/msys/bin/bash.exe")
subprocess.check_call(cmd, env=ensure_subprocess_env(env))
finally:
shutil.rmtree(tmpdir)
def prepare(srcdir, objdir, shell, args):
parser = argparse.ArgumentParser()
parser.add_argument('--target', type=str)
parser.add_argument('--host', type=str)
parser.add_argument('--build', type=str)
parser.add_argument('--cache-file', type=str)
# The --srcdir argument is simply ignored. It's a useless autoconf feature
# that we don't support well anyways. This makes it stripped from `others`
# and allows to skip setting it when calling the subconfigure (configure
# will take it from the configure path anyways).
parser.add_argument('--srcdir', type=str)
data_file = os.path.join(objdir, CONFIGURE_DATA)
previous_args = None
if os.path.exists(data_file):
with open(data_file, 'rb') as f:
data = pickle.load(f)
previous_args = data['args']
# Msys likes to break environment variables and command line arguments,
# so read those from stdin, as they are passed from the configure script
# when necessary (on windows).
# However, for some reason, $PATH is not handled like other environment
# variables, and msys remangles it even when giving it is already a msys
# $PATH. Fortunately, the mangling/demangling is just find for $PATH, so
# we can just take the value from the environment. Msys will convert it
# back properly when calling subconfigure.
input = sys.stdin.read()
if input:
data = {a: b for [a, b] in eval(input)}
environ = {a: b for a, b in data['env']}
environ['PATH'] = os.environ['PATH']
args = data['args']
else:
environ = os.environ
args, others = parser.parse_known_args(args)
data = {
'target': args.target,
'host': args.host,
'build': args.build,
'args': others,
'shell': shell,
'srcdir': srcdir,
'env': environ,
}
if args.cache_file:
data['cache-file'] = mozpath.normpath(
mozpath.join(os.getcwd(), args.cache_file))
else:
data['cache-file'] = mozpath.join(objdir, 'config.cache')
if previous_args is not None:
data['previous-args'] = previous_args
try:
os.makedirs(objdir)
except OSError as e:
if e.errno != errno.EEXIST:
raise
with open(data_file, 'wb') as f:
pickle.dump(data, f)
def __init__(self,
topsrcdir,
topobjdir,
defines=None,
non_global_defines=None,
substs=None,
source=None,
mozconfig=None):
if not source:
source = mozpath.join(topobjdir, 'config.status')
self.source = source
self.defines = ReadOnlyDict(defines or {})
self.non_global_defines = non_global_defines or []
self.substs = dict(substs or {})
self.topsrcdir = mozpath.abspath(topsrcdir)
self.topobjdir = mozpath.abspath(topobjdir)
self.mozconfig = mozpath.abspath(mozconfig) if mozconfig else None
self.lib_prefix = self.substs.get('LIB_PREFIX', '')
self.rust_lib_prefix = self.substs.get('RUST_LIB_PREFIX', '')
if 'LIB_SUFFIX' in self.substs:
self.lib_suffix = '.%s' % self.substs['LIB_SUFFIX']
if 'RUST_LIB_SUFFIX' in self.substs:
self.rust_lib_suffix = '.%s' % self.substs['RUST_LIB_SUFFIX']
self.dll_prefix = self.substs.get('DLL_PREFIX', '')
self.dll_suffix = self.substs.get('DLL_SUFFIX', '')
if self.substs.get('IMPORT_LIB_SUFFIX'):
self.import_prefix = self.lib_prefix
self.import_suffix = '.%s' % self.substs['IMPORT_LIB_SUFFIX']
else:
self.import_prefix = self.dll_prefix
self.import_suffix = self.dll_suffix
self.bin_suffix = self.substs.get('BIN_SUFFIX', '')
global_defines = [
name for name in self.defines
if not name in self.non_global_defines
]
self.substs['ACDEFINES'] = ' '.join([
'-D%s=%s' %
(name, shell_quote(self.defines[name]).replace('$', '$$'))
for name in sorted(global_defines)
])
def serialize(name, obj):
if isinstance(obj, StringTypes):
return obj
if isinstance(obj, Iterable):
return ' '.join(obj)
raise Exception('Unhandled type %s for %s', type(obj), str(name))
self.substs['ALLSUBSTS'] = '\n'.join(
sorted([
'%s = %s' % (name, serialize(name, self.substs[name]))
for name in self.substs if self.substs[name]
]))
self.substs['ALLEMPTYSUBSTS'] = '\n'.join(
sorted([
'%s =' % name for name in self.substs if not self.substs[name]
]))
self.substs = ReadOnlyDict(self.substs)
self.external_source_dir = None
external = self.substs.get('EXTERNAL_SOURCE_DIR', '')
if external:
external = mozpath.normpath(external)
if not os.path.isabs(external):
external = mozpath.join(self.topsrcdir, external)
self.external_source_dir = mozpath.normpath(external)
# Populate a Unicode version of substs. This is an optimization to make
# moz.build reading faster, since each sandbox needs a Unicode version
# of these variables and doing it over a thousand times is a hotspot
# during sandbox execution!
# Bug 844509 tracks moving everything to Unicode.
self.substs_unicode = {}
def decode(v):
if not isinstance(v, text_type):
try:
return v.decode('utf-8')
except UnicodeDecodeError:
return v.decode('utf-8', 'replace')
for k, v in self.substs.items():
if not isinstance(v, StringTypes):
if isinstance(v, Iterable):
type(v)(decode(i) for i in v)
elif not isinstance(v, text_type):
v = decode(v)
self.substs_unicode[k] = v
self.substs_unicode = ReadOnlyDict(self.substs_unicode)
def read_relevant_mozbuilds(self, paths):
"""Read and process moz.build files relevant for a set of paths.
For an iterable of relative-to-root filesystem paths ``paths``,
find all moz.build files that may apply to them based on filesystem
hierarchy and read those moz.build files.
The return value is a 2-tuple. The first item is a dict mapping each
input filesystem path to a list of Context instances that are relevant
to that path. The second item is a list of all Context instances. Each
Context instance is in both data structures.
"""
relevants = self._find_relevant_mozbuilds(paths)
topsrcdir = self.config.topsrcdir
# Source moz.build file to directories to traverse.
dirs = defaultdict(set)
# Relevant path to absolute paths of relevant contexts.
path_mozbuilds = {}
# There is room to improve this code (and the code in
# _find_relevant_mozbuilds) to better handle multiple files in the same
# directory. Bug 1136966 tracks.
for path, mbpaths in relevants.items():
path_mozbuilds[path] = [
mozpath.join(topsrcdir, p) for p in mbpaths
]
for i, mbpath in enumerate(mbpaths[0:-1]):
source_dir = mozpath.dirname(mbpath)
target_dir = mozpath.dirname(mbpaths[i + 1])
d = mozpath.normpath(mozpath.join(topsrcdir, mbpath))
dirs[d].add(mozpath.relpath(target_dir, source_dir))
# Exporting doesn't work reliably in tree traversal mode. Override
# the function to no-op.
functions = dict(FUNCTIONS)
def export(sandbox):
return lambda varname: None
functions['export'] = tuple([export] + list(FUNCTIONS['export'][1:]))
metadata = {
'functions': functions,
}
contexts = defaultdict(list)
all_contexts = []
for context in self.read_mozbuild(mozpath.join(topsrcdir, 'moz.build'),
self.config,
metadata=metadata):
# Explicitly set directory traversal variables to override default
# traversal rules.
if not isinstance(context, SubContext):
for v in ('DIRS', 'GYP_DIRS', 'TEST_DIRS'):
context[v][:] = []
context['DIRS'] = sorted(dirs[context.main_path])
contexts[context.main_path].append(context)
all_contexts.append(context)
result = {}
for path, paths in path_mozbuilds.items():
result[path] = reduce(lambda x, y: x + y,
(contexts[p] for p in paths), [])
return result, all_contexts
def validate_args(self, **kwargs):
tests_selected = kwargs["tests"] or kwargs["paths"] or kwargs["tags"]
if kwargs["platforms"] is None and (kwargs["jobs"] is None
or tests_selected):
if 'AUTOTRY_PLATFORM_HINT' in os.environ:
kwargs["platforms"] = [os.environ['AUTOTRY_PLATFORM_HINT']]
elif tests_selected:
print("Must specify platform when selecting tests.")
sys.exit(1)
else:
print(
"Either platforms or jobs must be specified as an argument to autotry."
)
sys.exit(1)
try:
platforms = (self.normalise_list(kwargs["platforms"])
if kwargs["platforms"] else {})
except ValueError as e:
print("Error parsing -p argument:\n%s" % e.message)
sys.exit(1)
try:
tests = (self.normalise_list(kwargs["tests"], allow_subitems=True)
if kwargs["tests"] else {})
except ValueError as e:
print("Error parsing -u argument (%s):\n%s" %
(kwargs["tests"], e.message))
sys.exit(1)
try:
talos = (self.normalise_list(kwargs["talos"], allow_subitems=True)
if kwargs["talos"] else [])
except ValueError as e:
print("Error parsing -t argument:\n%s" % e.message)
sys.exit(1)
try:
jobs = (self.normalise_list(kwargs["jobs"])
if kwargs["jobs"] else {})
except ValueError as e:
print("Error parsing -j argument:\n%s" % e.message)
sys.exit(1)
paths = []
for p in kwargs["paths"]:
p = mozpath.normpath(os.path.abspath(p))
if not (os.path.isdir(p) and p.startswith(self.topsrcdir)):
print(
'Specified path "%s" is not a directory under the srcdir,'
' unable to specify tests outside of the srcdir' % p)
sys.exit(1)
if len(p) <= len(self.topsrcdir):
print(
'Specified path "%s" is at the top of the srcdir and would'
' select all tests.' % p)
sys.exit(1)
paths.append(os.path.relpath(p, self.topsrcdir))
try:
tags = self.normalise_list(
kwargs["tags"]) if kwargs["tags"] else []
except ValueError as e:
print("Error parsing --tags argument:\n%s" % e.message)
sys.exit(1)
extra_values = {
k['dest']
for k in SyntaxParser.pass_through_arguments.values()
}
extra_args = {
k: v
for k, v in kwargs.items() if k in extra_values and v
}
return kwargs[
"builds"], platforms, tests, talos, jobs, paths, tags, extra_args
def _process_test_manifest(self, sandbox, info, manifest_path):
flavor, install_prefix, filter_inactive = info
manifest_path = os.path.normpath(manifest_path)
path = mozpath.normpath(mozpath.join(sandbox['SRCDIR'], manifest_path))
manifest_dir = mozpath.dirname(path)
manifest_reldir = mozpath.dirname(
mozpath.relpath(path, sandbox['TOPSRCDIR']))
try:
m = manifestparser.TestManifest(manifests=[path], strict=True)
if not m.tests:
raise SandboxValidationError('Empty test manifest: %s' % path)
obj = TestManifest(sandbox,
path,
m,
flavor=flavor,
install_prefix=install_prefix,
relpath=mozpath.join(manifest_reldir,
mozpath.basename(path)),
dupe_manifest='dupe-manifest' in m.tests[0])
filtered = m.tests
if filter_inactive:
filtered = m.active_tests(disabled=False, **self.mozinfo)
out_dir = mozpath.join(install_prefix, manifest_reldir)
finder = FileFinder(base=manifest_dir, find_executables=False)
# "head" and "tail" lists.
# All manifests support support-files.
#
# Keep a set of already seen support file patterns, because
# repeatedly processing the patterns from the default section
# for every test is quite costly (see bug 922517).
extras = (('head', set()), ('tail', set()), ('support-files',
set()))
for test in filtered:
obj.tests.append(test)
obj.installs[mozpath.normpath(test['path'])] = \
mozpath.join(out_dir, test['relpath'])
for thing, seen in extras:
value = test.get(thing, '')
if value in seen:
continue
seen.add(value)
for pattern in value.split():
# We only support globbing on support-files because
# the harness doesn't support * for head and tail.
#
# While we could feed everything through the finder, we
# don't because we want explicitly listed files that
# no longer exist to raise an error. The finder is also
# slower than simple lookup.
if '*' in pattern and thing == 'support-files':
paths = [f[0] for f in finder.find(pattern)]
if not paths:
raise SandboxValidationError(
'%s support-files '
'wildcard in %s returns no results.' %
(pattern, path))
for f in paths:
full = mozpath.normpath(
mozpath.join(manifest_dir, f))
obj.installs[full] = mozpath.join(out_dir, f)
else:
full = mozpath.normpath(
mozpath.join(manifest_dir, pattern))
# Only install paths in our directory. This
# rule is somewhat arbitrary and could be lifted.
if not full.startswith(manifest_dir):
continue
obj.installs[full] = mozpath.join(out_dir, pattern)
# We also copy the manifest into the output directory.
out_path = mozpath.join(out_dir, os.path.basename(manifest_path))
obj.installs[path] = out_path
# Some manifests reference files that are auto generated as
# part of the build or shouldn't be installed for some
# reason. Here, we prune those files from the install set.
# FUTURE we should be able to detect autogenerated files from
# other build metadata. Once we do that, we can get rid of this.
for f in m.tests[0].get('generated-files', '').split():
# We re-raise otherwise the stack trace isn't informative.
try:
del obj.installs[mozpath.join(manifest_dir, f)]
except KeyError:
raise SandboxValidationError(
'Error processing test '
'manifest %s: entry in generated-files not present '
'elsewhere in manifest: %s' % (path, f))
obj.external_installs.add(mozpath.join(out_dir, f))
yield obj
except (AssertionError, Exception):
raise SandboxValidationError(
'Error processing test '
'manifest file %s: %s' %
(path, '\n'.join(traceback.format_exception(*sys.exc_info()))))
def _process_test_manifest(self, sandbox, info, manifest_path):
flavor, install_root, install_subdir, filter_inactive = info
manifest_path = mozpath.normpath(manifest_path)
path = mozpath.normpath(mozpath.join(sandbox['SRCDIR'], manifest_path))
manifest_dir = mozpath.dirname(path)
manifest_reldir = mozpath.dirname(
mozpath.relpath(path, sandbox['TOPSRCDIR']))
install_prefix = mozpath.join(install_root, install_subdir)
try:
m = manifestparser.TestManifest(manifests=[path], strict=True)
defaults = m.manifest_defaults[os.path.normpath(path)]
if not m.tests and not 'support-files' in defaults:
raise SandboxValidationError('Empty test manifest: %s' % path)
obj = TestManifest(sandbox,
path,
m,
flavor=flavor,
install_prefix=install_prefix,
relpath=mozpath.join(manifest_reldir,
mozpath.basename(path)),
dupe_manifest='dupe-manifest' in defaults)
filtered = m.tests
if filter_inactive:
filtered = m.active_tests(disabled=False, **self.mozinfo)
out_dir = mozpath.join(install_prefix, manifest_reldir)
# "head" and "tail" lists.
# All manifests support support-files.
#
# Keep a set of already seen support file patterns, because
# repeatedly processing the patterns from the default section
# for every test is quite costly (see bug 922517).
extras = (('head', set()), ('tail', set()), ('support-files',
set()))
def process_support_files(test):
for thing, seen in extras:
value = test.get(thing, '')
if value in seen:
continue
seen.add(value)
for pattern in value.split():
# We only support globbing on support-files because
# the harness doesn't support * for head and tail.
if '*' in pattern and thing == 'support-files':
obj.pattern_installs.append(
(manifest_dir, pattern, out_dir))
# "absolute" paths identify files that are to be
# placed in the install_root directory (no globs)
elif pattern[0] == '/':
full = mozpath.normpath(
mozpath.join(manifest_dir,
mozpath.basename(pattern)))
obj.installs[full] = (mozpath.join(
install_root, pattern[1:]), False)
else:
full = mozpath.normpath(
mozpath.join(manifest_dir, pattern))
dest_path = mozpath.join(out_dir, pattern)
# If the path resolves to a different directory
# tree, we take special behavior depending on the
# entry type.
if not full.startswith(manifest_dir):
# If it's a support file, we install the file
# into the current destination directory.
# This implementation makes installing things
# with custom prefixes impossible. If this is
# needed, we can add support for that via a
# special syntax later.
if thing == 'support-files':
dest_path = mozpath.join(
out_dir, os.path.basename(pattern))
# If it's not a support file, we ignore it.
# This preserves old behavior so things like
# head files doesn't get installed multiple
# times.
else:
continue
obj.installs[full] = (mozpath.normpath(dest_path),
False)
for test in filtered:
obj.tests.append(test)
obj.installs[mozpath.normpath(test['path'])] = \
(mozpath.join(out_dir, test['relpath']), True)
process_support_files(test)
if not filtered:
# If there are no tests, look for support-files under DEFAULT.
process_support_files(defaults)
# We also copy the manifest into the output directory.
out_path = mozpath.join(out_dir, mozpath.basename(manifest_path))
obj.installs[path] = (out_path, False)
# Some manifests reference files that are auto generated as
# part of the build or shouldn't be installed for some
# reason. Here, we prune those files from the install set.
# FUTURE we should be able to detect autogenerated files from
# other build metadata. Once we do that, we can get rid of this.
for f in defaults.get('generated-files', '').split():
# We re-raise otherwise the stack trace isn't informative.
try:
del obj.installs[mozpath.join(manifest_dir, f)]
except KeyError:
raise SandboxValidationError(
'Error processing test '
'manifest %s: entry in generated-files not present '
'elsewhere in manifest: %s' % (path, f))
obj.external_installs.add(mozpath.join(out_dir, f))
yield obj
except (AssertionError, Exception):
raise SandboxValidationError(
'Error processing test '
'manifest file %s: %s' %
(path, '\n'.join(traceback.format_exception(*sys.exc_info()))))
def read_manifestparser_manifest(context, manifest_path):
path = mozpath.normpath(mozpath.join(context.srcdir, manifest_path))
return manifestparser.TestManifest(manifests=[path],
strict=True,
rootdir=context.config.topsrcdir,
finder=context._finder)
def convert_support_files(self, test, install_root, manifest_dir, out_dir):
# Arguments:
# test - The test object to process.
# install_root - The directory under $objdir/_tests that will contain
# the tests for this harness (examples are "testing/mochitest",
# "xpcshell").
# manifest_dir - Absoulute path to the (srcdir) directory containing the
# manifest that included this test
# out_dir - The path relative to $objdir/_tests used as the destination for the
# test, based on the relative path to the manifest in the srcdir and
# the install_root.
info = TestInstallInfo()
for field, seen in self._fields:
value = test.get(field, '')
for pattern in value.split():
# We track uniqueness locally (per test) where duplicates are forbidden,
# and globally, where they are permitted. If a support file appears multiple
# times for a single test, there are unnecessary entries in the manifest. But
# many entries will be shared across tests that share defaults.
key = field, pattern, out_dir
if key in info.seen:
raise ValueError(
"%s appears multiple times in a test manifest under a %s field,"
" please omit the duplicate entry." % (pattern, field))
info.seen.add(key)
if key in seen:
continue
seen.add(key)
if field == 'generated-files':
info.external_installs.add(
mozpath.normpath(mozpath.join(out_dir, pattern)))
# '!' indicates our syntax for inter-directory support file
# dependencies. These receive special handling in the backend.
elif pattern[0] == '!':
info.deferred_installs.add(pattern)
# We only support globbing on support-files because
# the harness doesn't support * for head.
elif '*' in pattern and field == 'support-files':
info.pattern_installs.append(
(manifest_dir, pattern, out_dir))
# "absolute" paths identify files that are to be
# placed in the install_root directory (no globs)
elif pattern[0] == '/':
full = mozpath.normpath(
mozpath.join(manifest_dir, mozpath.basename(pattern)))
info.installs.append(
(full, mozpath.join(install_root, pattern[1:])))
else:
full = mozpath.normpath(mozpath.join(
manifest_dir, pattern))
dest_path = mozpath.join(out_dir, pattern)
# If the path resolves to a different directory
# tree, we take special behavior depending on the
# entry type.
if not full.startswith(manifest_dir):
# If it's a support file, we install the file
# into the current destination directory.
# This implementation makes installing things
# with custom prefixes impossible. If this is
# needed, we can add support for that via a
# special syntax later.
if field == 'support-files':
dest_path = mozpath.join(out_dir,
os.path.basename(pattern))
# If it's not a support file, we ignore it.
# This preserves old behavior so things like
# head files doesn't get installed multiple
# times.
else:
continue
info.installs.append((full, mozpath.normpath(dest_path)))
return info
def _consume_jar_manifest(self, obj):
# Ideally, this would all be handled somehow in the emitter, but
# this would require all the magic surrounding l10n and addons in
# the recursive make backend to die, which is not going to happen
# any time soon enough.
# Notably missing:
# - DEFINES from config/config.mk
# - L10n support
# - The equivalent of -e when USE_EXTENSION_MANIFEST is set in
# moz.build, but it doesn't matter in dist/bin.
pp = Preprocessor()
if obj.defines:
pp.context.update(obj.defines.defines)
pp.context.update(self.environment.defines)
pp.context.update(
AB_CD='en-US',
BUILD_FASTER=1,
)
pp.out = JarManifestParser()
try:
pp.do_include(obj.path.full_path)
except DeprecatedJarManifest as e:
raise DeprecatedJarManifest(
'Parsing error while processing %s: %s' %
(obj.path.full_path, e.message))
self.backend_input_files |= pp.includes
for jarinfo in pp.out:
jar_context = Context(allowed_variables=VARIABLES,
config=obj._context.config)
jar_context.push_source(obj._context.main_path)
jar_context.push_source(obj.path.full_path)
install_target = obj.install_target
if jarinfo.base:
install_target = mozpath.normpath(
mozpath.join(install_target, jarinfo.base))
jar_context['FINAL_TARGET'] = install_target
if obj.defines:
jar_context['DEFINES'] = obj.defines.defines
files = jar_context['FINAL_TARGET_FILES']
files_pp = jar_context['FINAL_TARGET_PP_FILES']
for e in jarinfo.entries:
if e.is_locale:
if jarinfo.relativesrcdir:
src = '/%s' % jarinfo.relativesrcdir
else:
src = ''
src = mozpath.join(src, 'en-US', e.source)
else:
src = e.source
src = Path(jar_context, src)
if '*' not in e.source and not os.path.exists(src.full_path):
if e.is_locale:
raise Exception('%s: Cannot find %s' %
(obj.path, e.source))
if e.source.startswith('/'):
src = Path(jar_context, '!' + e.source)
else:
# This actually gets awkward if the jar.mn is not
# in the same directory as the moz.build declaring
# it, but it's how it works in the recursive make,
# not that anything relies on that, but it's simpler.
src = Path(obj._context, '!' + e.source)
output_basename = mozpath.basename(e.output)
if output_basename != src.target_basename:
src = RenamedSourcePath(jar_context,
(src, output_basename))
path = mozpath.dirname(mozpath.join(jarinfo.name, e.output))
if e.preprocess:
if '*' in e.source:
raise Exception('%s: Wildcards are not supported with '
'preprocessing' % obj.path)
files_pp[path] += [src]
else:
files[path] += [src]
if files:
self.consume_object(FinalTargetFiles(jar_context, files))
if files_pp:
self.consume_object(
FinalTargetPreprocessedFiles(jar_context, files_pp))
for m in jarinfo.chrome_manifests:
entry = parse_manifest_line(
mozpath.dirname(jarinfo.name),
m.replace('%',
mozpath.basename(jarinfo.name) + '/'))
self.consume_object(
ChromeManifestEntry(jar_context,
'%s.manifest' % jarinfo.name, entry))
