def is_read_allowed(path, config):
"""Whether we are allowed to load a mozbuild file at the specified path.
This is used as cheap security to ensure the build is isolated to known
source directories.
We are allowed to read from the main source directory and any defined
external source directories. The latter is to allow 3rd party applications
to hook into our build system.
"""
assert os.path.isabs(path)
assert os.path.isabs(config.topsrcdir)
path = mozpath.normpath(path)
topsrcdir = mozpath.normpath(config.topsrcdir)
if mozpath.basedir(path, [topsrcdir]):
return True
if config.external_source_dir:
external_dir = os.path.normcase(config.external_source_dir)
norm_path = os.path.normcase(path)
if mozpath.basedir(norm_path, [external_dir]):
return True
return False
def is_read_allowed(path, config):
"""Whether we are allowed to load a mozbuild file at the specified path.
This is used as cheap security to ensure the build is isolated to known
source directories.
We are allowed to read from the main source directory and any defined
external source directories. The latter is to allow 3rd party applications
to hook into our build system.
"""
assert os.path.isabs(path)
assert os.path.isabs(config.topsrcdir)
path = mozpath.normpath(path)
topsrcdir = mozpath.normpath(config.topsrcdir)
if mozpath.basedir(path, [topsrcdir]):
return True
if config.external_source_dir:
external_dir = os.path.normcase(config.external_source_dir)
norm_path = os.path.normcase(path)
if mozpath.basedir(norm_path, [external_dir]):
return True
return False
def test_basedir(self):
foobarbaz = os.path.join("foo", "bar", "baz")
self.assertEqual(basedir(foobarbaz, ["foo", "bar", "baz"]), "foo")
self.assertEqual(basedir(foobarbaz, ["foo", "foo/bar", "baz"]), "foo/bar")
self.assertEqual(basedir(foobarbaz, ["foo/bar", "foo", "baz"]), "foo/bar")
self.assertEqual(basedir(foobarbaz, ["foo", "bar", ""]), "foo")
self.assertEqual(basedir(foobarbaz, ["bar", "baz", ""]), "")
def test_basedir(self):
foobarbaz = os.path.join('foo', 'bar', 'baz')
self.assertEqual(basedir(foobarbaz, ['foo', 'bar', 'baz']), 'foo')
self.assertEqual(basedir(foobarbaz, ['foo', 'foo/bar', 'baz']),
'foo/bar')
self.assertEqual(basedir(foobarbaz, ['foo/bar', 'foo', 'baz']),
'foo/bar')
self.assertEqual(basedir(foobarbaz, ['foo', 'bar', '']), 'foo')
self.assertEqual(basedir(foobarbaz, ['bar', 'baz', '']), '')
def test_basedir(self):
foobarbaz = os.path.join('foo', 'bar', 'baz')
self.assertEqual(basedir(foobarbaz, ['foo', 'bar', 'baz']), 'foo')
self.assertEqual(basedir(foobarbaz, ['foo', 'foo/bar', 'baz']),
'foo/bar')
self.assertEqual(basedir(foobarbaz, ['foo/bar', 'foo', 'baz']),
'foo/bar')
self.assertEqual(basedir(foobarbaz, ['foo', 'bar', '']), 'foo')
self.assertEqual(basedir(foobarbaz, ['bar', 'baz', '']), '')
def test_basedir(self):
foobarbaz = self.SEP.join(('foo', 'bar', 'baz'))
self.assertEqual(basedir(foobarbaz, ['foo', 'bar', 'baz']), 'foo')
self.assertEqual(basedir(foobarbaz, ['foo', 'foo/bar', 'baz']),
'foo/bar')
self.assertEqual(basedir(foobarbaz, ['foo/bar', 'foo', 'baz']),
'foo/bar')
self.assertEqual(basedir(foobarbaz, ['foo', 'bar', '']), 'foo')
self.assertEqual(basedir(foobarbaz, ['bar', 'baz', '']), '')
def test_basedir(self):
foobarbaz = self.SEP.join(("foo", "bar", "baz"))
self.assertEqual(basedir(foobarbaz, ["foo", "bar", "baz"]), "foo")
self.assertEqual(basedir(foobarbaz, ["foo", "foo/bar", "baz"]),
"foo/bar")
self.assertEqual(basedir(foobarbaz, ["foo/bar", "foo", "baz"]),
"foo/bar")
self.assertEqual(basedir(foobarbaz, ["foo", "bar", ""]), "foo")
self.assertEqual(basedir(foobarbaz, ["bar", "baz", ""]), "")
def _add_addon(self, path, addon_type):
'''
Add the given BaseFile to the collection of addons if a parent
directory is not already in the collection.
'''
if mozpath.basedir(path, self._addons) is not None:
return
for dir in self._addons:
if mozpath.basedir(dir, [path]) is not None:
del self._addons[dir]
break
self._addons[path] = addon_type
def _add_base(self, base, addon=False):
if addon:
# Because add_base is always called with parents before children,
# all the possible ancestry of `base` is already present in
# `_sub_formatter`.
parent_base = mozpath.basedir(base, self._sub_formatter.keys())
rel_base = mozpath.relpath(base, parent_base)
# If the addon is under a resource directory, package it in the
# omnijar.
parent_sub_formatter = self._sub_formatter[parent_base]
if parent_sub_formatter.is_resource(rel_base):
omnijar_sub_formatter = \
parent_sub_formatter._sub_formatter[self._omnijar_name]
self._sub_formatter[base] = FlatSubFormatter(
FileRegistrySubtree(rel_base,
omnijar_sub_formatter.copier))
return
JarFormatter._add_base(self, base, addon)
else:
# Initialize a chrome.manifest next to the omnijar file so that
# there's always a chrome.manifest file, even an empty one.
path = mozpath.normpath(mozpath.join(base, 'chrome.manifest'))
if not self.copier.contains(path):
self.copier.add(path, ManifestFile(''))
self._sub_formatter[base] = OmniJarSubFormatter(
FileRegistrySubtree(base, self.copier), self._omnijar_name,
self._compress, self._non_resources)
def _get_files_info(self, paths, rev=None):
reader = self.mozbuild_reader(config_mode='empty', vcs_revision=rev)
# Normalize to relative from topsrcdir.
relpaths = []
for p in paths:
a = mozpath.abspath(p)
if not mozpath.basedir(a, [self.topsrcdir]):
raise InvalidPathException('path is outside topsrcdir: %s' % p)
relpaths.append(mozpath.relpath(a, self.topsrcdir))
# Expand wildcards.
# One variable is for ordering. The other for membership tests.
# (Membership testing on a list can be slow.)
allpaths = []
all_paths_set = set()
for p in relpaths:
if '*' not in p:
if p not in all_paths_set:
all_paths_set.add(p)
allpaths.append(p)
continue
if rev:
raise InvalidPathException(
'cannot use wildcard in version control mode')
for path, f in reader.finder.find(p):
if path not in all_paths_set:
all_paths_set.add(path)
allpaths.append(path)
return reader.files_info(allpaths)
def _add_base(self, base, addon=False):
if addon:
# Because add_base is always called with parents before children,
# all the possible ancestry of `base` is already present in
# `_sub_formatter`.
parent_base = mozpath.basedir(base, self._sub_formatter.keys())
rel_base = mozpath.relpath(base, parent_base)
# If the addon is under a resource directory, package it in the
# omnijar.
parent_sub_formatter = self._sub_formatter[parent_base]
if parent_sub_formatter.is_resource(rel_base):
omnijar_sub_formatter = parent_sub_formatter._sub_formatter[
self._omnijar_name]
self._sub_formatter[base] = FlatSubFormatter(
FileRegistrySubtree(rel_base,
omnijar_sub_formatter.copier))
return
JarFormatter._add_base(self, base, addon)
else:
self._sub_formatter[base] = OmniJarSubFormatter(
FileRegistrySubtree(base, self.copier),
self._omnijar_name,
self._compress,
self._non_resources,
)
def normalize_path(path):
"""
Remove any bin/ prefix.
"""
if mozpath.basedir(path, ["bin"]) == "bin":
return mozpath.relpath(path, "bin")
return path
def _get_files_info(self, paths):
from mozpack.files import FileFinder
# Normalize to relative from topsrcdir.
relpaths = []
for p in paths:
a = mozpath.abspath(p)
if not mozpath.basedir(a, [self.topsrcdir]):
raise InvalidPathException('path is outside topsrcdir: %s' % p)
relpaths.append(mozpath.relpath(a, self.topsrcdir))
finder = FileFinder(self.topsrcdir, find_executables=False)
# Expand wildcards.
allpaths = []
for p in relpaths:
if '*' not in p:
if p not in allpaths:
allpaths.append(p)
continue
for path, f in finder.find(p):
if path not in allpaths:
allpaths.append(path)
reader = self._get_reader()
return reader.files_info(allpaths)
def include_file(self, path):
'''Include one file in the sandbox. Users of this class probably want
to use `run` instead.
Note: this will execute all template invocations, as well as @depends
functions that depend on '--help', but nothing else.
'''
if self._paths:
path = mozpath.join(mozpath.dirname(self._paths[-1]), path)
path = mozpath.normpath(path)
if not mozpath.basedir(path, (mozpath.dirname(self._paths[0]),)):
raise ConfigureError(
'Cannot include `%s` because it is not in a subdirectory '
'of `%s`' % (path, mozpath.dirname(self._paths[0])))
else:
path = mozpath.realpath(mozpath.abspath(path))
if path in self._all_paths:
raise ConfigureError(
'Cannot include `%s` because it was included already.' % path)
self._paths.append(path)
self._all_paths.add(path)
source = open(path, 'rb').read()
code = compile(source, path, 'exec')
exec_(code, self)
self._paths.pop(-1)
def _get_files_info(self, paths):
from mozpack.files import FileFinder
# Normalize to relative from topsrcdir.
relpaths = []
for p in paths:
a = mozpath.abspath(p)
if not mozpath.basedir(a, [self.topsrcdir]):
raise InvalidPathException('path is outside topsrcdir: %s' % p)
relpaths.append(mozpath.relpath(a, self.topsrcdir))
finder = FileFinder(self.topsrcdir, find_executables=False)
# Expand wildcards.
allpaths = []
for p in relpaths:
if '*' not in p:
if p not in allpaths:
allpaths.append(p)
continue
for path, f in finder.find(p):
if path not in allpaths:
allpaths.append(path)
reader = self._get_reader()
return reader.files_info(allpaths)
def include_file(self, path):
'''Include one file in the sandbox. Users of this class probably want
Note: this will execute all template invocations, as well as @depends
functions that depend on '--help', but nothing else.
'''
if self._paths:
path = mozpath.join(mozpath.dirname(self._paths[-1]), path)
path = mozpath.normpath(path)
if not mozpath.basedir(path, (mozpath.dirname(self._paths[0]),)):
raise ConfigureError(
'Cannot include `%s` because it is not in a subdirectory '
'of `%s`' % (path, mozpath.dirname(self._paths[0])))
else:
path = mozpath.realpath(mozpath.abspath(path))
if path in self._all_paths:
raise ConfigureError(
'Cannot include `%s` because it was included already.' % path)
self._paths.append(path)
self._all_paths.add(path)
source = open(path, 'rb').read()
code = compile(source, path, 'exec')
exec_(code, self)
self._paths.pop(-1)
def _chromepath(self, path):
'''
Return the chrome base directory under which the given path is. Used to
detect under which .jar (if any) the path should go.
'''
self._frozen_chrome = True
return mozpath.basedir(path, self._chrome)
def filter_path(p):
p = mozpath.abspath(p)
base = mozpath.basedir(p, paths)
if base:
return mozpath.relpath(p, base)
# Best-effort.
return '<path omitted>'
def _add_manifest_file(self, path, file):
'''
Add the given BaseFile with manifest file contents with the given path.
'''
self._manifests.add(path)
base = ''
if hasattr(file, 'path'):
# Find the directory the given path is relative to.
b = mozpath.normsep(file.path)
if b.endswith('/' + path) or b == path:
base = os.path.normpath(b[:-len(path)])
for e in parse_manifest(base, path, file.open()):
# ManifestResources need to be given after ManifestChrome, so just
# put all ManifestChrome in a separate queue to make them first.
if isinstance(e, ManifestChrome):
# e.move(e.base) just returns a clone of the entry.
self._chrome_queue.append(self.formatter.add_manifest,
e.move(e.base))
elif not isinstance(e, (Manifest, ManifestInterfaces)):
self._queue.append(self.formatter.add_manifest, e.move(e.base))
# If a binary component is added to an addon, prevent the addon
# from being packed.
if isinstance(e, ManifestBinaryComponent):
addon = mozpath.basedir(e.base, self._addons)
if addon:
self._addons[addon] = 'unpacked'
if isinstance(e, Manifest):
if e.flags:
errors.fatal('Flags are not supported on ' +
'"manifest" entries')
self._included_manifests[e.path] = path
def filter_path(p):
p = mozpath.abspath(p)
base = mozpath.basedir(p, paths.keys())
if base:
return paths[base] + mozpath.relpath(p, base)
# Best-effort.
return "<path omitted>"
def add(self, t, flavor, topsrcdir):
t = dict(t)
t['flavor'] = flavor
path = mozpath.normpath(t['path'])
manifest = mozpath.normpath(t['manifest'])
assert mozpath.basedir(path, [topsrcdir])
assert mozpath.basedir(manifest, [topsrcdir])
key = path[len(topsrcdir) + 1:]
t['file_relpath'] = key
t['dir_relpath'] = mozpath.dirname(key)
t['srcdir_relpath'] = key
t['manifest_relpath'] = manifest[len(topsrcdir) + 1:]
self.tests_by_path[key].append(t)
def _chromepath(self, path):
'''
Return the chrome base directory under which the given path is. Used to
detect under which .jar (if any) the path should go.
'''
self._frozen_chrome = True
return mozpath.basedir(path, self._chrome)
def isfile(self, path):
path = mozpath.abspath(path)
if path in self._paths:
return True
if mozpath.basedir(path, [topsrcdir, topobjdir]):
return os.path.isfile(path)
return False
def normalize_path(path):
'''
Remove any bin/ prefix.
'''
if mozpath.basedir(path, ['bin']) == 'bin':
return mozpath.relpath(path, 'bin')
return path
def normalize_path(path):
'''
Remove any bin/ prefix.
'''
if mozpath.basedir(path, ['bin']) == 'bin':
return mozpath.relpath(path, 'bin')
return path
def isfile(self, path):
path = mozpath.abspath(path)
if path in self._paths:
return True
if mozpath.basedir(path, [topsrcdir, topobjdir]):
return os.path.isfile(path)
return False
def _add_manifest_file(self, path, file):
'''
Add the given BaseFile with manifest file contents with the given path.
'''
self._manifests.add(path)
base = ''
if hasattr(file, 'path'):
# Find the directory the given path is relative to.
b = mozpath.normsep(file.path)
if b.endswith('/' + path) or b == path:
base = os.path.normpath(b[:-len(path)])
for e in parse_manifest(base, path, file.open()):
# ManifestResources need to be given after ManifestChrome, so just
# put all ManifestChrome in a separate queue to make them first.
if isinstance(e, ManifestChrome):
# e.move(e.base) just returns a clone of the entry.
self._chrome_queue.append(self.formatter.add_manifest,
e.move(e.base))
elif not isinstance(e, (Manifest, ManifestInterfaces)):
self._queue.append(self.formatter.add_manifest, e.move(e.base))
# If a binary component is added to an addon, prevent the addon
# from being packed.
if isinstance(e, ManifestBinaryComponent):
addon = mozpath.basedir(e.base, self._addons)
if addon:
self._addons[addon] = 'unpacked'
if isinstance(e, Manifest):
if e.flags:
errors.fatal('Flags are not supported on ' +
'"manifest" entries')
self._included_manifests[e.path] = path
def _get_base(self, path):
'''
Return the deepest base directory containing the given path.
'''
self._frozen_bases = True
base = mozpath.basedir(path, self._sub_formatter.keys())
relpath = mozpath.relpath(path, base) if base else path
return base, relpath
def _get_base(self, path):
'''
Return the deepest base directory containing the given path.
'''
self._frozen_bases = True
base = mozpath.basedir(path, self._sub_formatter.keys())
relpath = mozpath.relpath(path, base) if base else path
return base, relpath
def isdir(self, path):
path = mozpath.abspath(path)
if any(
mozpath.basedir(mozpath.dirname(p), [path])
for p in self._paths):
return True
if self._real_file(path):
return os.path.isdir(path)
return False
def consume_finished(self):
mk = Makefile()
# Add the default rule at the very beginning.
mk.create_rule(['default'])
mk.add_statement('TOPSRCDIR = %s' % self.environment.topsrcdir)
mk.add_statement('TOPOBJDIR = %s' % self.environment.topobjdir)
mk.add_statement('BACKEND = %s' % self._backend_output_list_file)
if not self._has_xpidl:
mk.add_statement('NO_XPIDL = 1')
# Add a few necessary variables inherited from configure
for var in (
'PYTHON',
'ACDEFINES',
'MOZ_BUILD_APP',
'MOZ_WIDGET_TOOLKIT',
):
mk.add_statement('%s = %s' % (var, self.environment.substs[var]))
install_manifests_bases = self._install_manifests.keys()
# Add information for chrome manifest generation
manifest_targets = []
for target, entries in self._manifest_entries.iteritems():
manifest_targets.append(target)
install_target = mozpath.basedir(target, install_manifests_bases)
self._install_manifests[install_target].add_content(
''.join('%s\n' % e for e in sorted(entries)),
mozpath.relpath(target, install_target))
# Add information for install manifests.
mk.add_statement('INSTALL_MANIFESTS = %s' %
' '.join(self._install_manifests.keys()))
# Add dependencies we infered:
for target, deps in self._dependencies.iteritems():
mk.create_rule([target]).add_dependencies('$(TOPOBJDIR)/%s' % d
for d in deps)
# Add backend dependencies:
mk.create_rule([self._backend_output_list_file
]).add_dependencies(self.backend_input_files)
mk.add_statement('include $(TOPSRCDIR)/config/faster/rules.mk')
for base, install_manifest in self._install_manifests.iteritems():
with self._write_file(
mozpath.join(self.environment.topobjdir, 'faster',
'install_%s' % base.replace('/', '_'))) as fh:
install_manifest.write(fileobj=fh)
with self._write_file(
mozpath.join(self.environment.topobjdir, 'faster',
'Makefile')) as fh:
mk.dump(fh, removal_guard=False)
def consume_finished(self):
mk = Makefile()
# Add the default rule at the very beginning.
mk.create_rule(['default'])
mk.add_statement('TOPSRCDIR = %s' % self.environment.topsrcdir)
mk.add_statement('TOPOBJDIR = %s' % self.environment.topobjdir)
mk.add_statement('BACKEND = %s' % self._backend_output_list_file)
if not self._has_xpidl:
mk.add_statement('NO_XPIDL = 1')
# Add a few necessary variables inherited from configure
for var in (
'PYTHON',
'ACDEFINES',
'MOZ_BUILD_APP',
'MOZ_WIDGET_TOOLKIT',
):
mk.add_statement('%s = %s' % (var, self.environment.substs[var]))
install_manifests_bases = self._install_manifests.keys()
# Add information for chrome manifest generation
manifest_targets = []
for target, entries in self._manifest_entries.iteritems():
manifest_targets.append(target)
install_target = mozpath.basedir(target, install_manifests_bases)
self._install_manifests[install_target].add_content(
''.join('%s\n' % e for e in sorted(entries)),
mozpath.relpath(target, install_target))
# Add information for install manifests.
mk.add_statement('INSTALL_MANIFESTS = %s'
% ' '.join(self._install_manifests.keys()))
# Add dependencies we infered:
for target, deps in self._dependencies.iteritems():
mk.create_rule([target]).add_dependencies(
'$(TOPOBJDIR)/%s' % d for d in deps)
# Add backend dependencies:
mk.create_rule([self._backend_output_list_file]).add_dependencies(
self.backend_input_files)
mk.add_statement('include $(TOPSRCDIR)/config/faster/rules.mk')
for base, install_manifest in self._install_manifests.iteritems():
with self._write_file(
mozpath.join(self.environment.topobjdir, 'faster',
'install_%s' % base.replace('/', '_'))) as fh:
install_manifest.write(fileobj=fh)
with self._write_file(
mozpath.join(self.environment.topobjdir, 'faster',
'Makefile')) as fh:
mk.dump(fh, removal_guard=False)
def close(self):
'''
Push all instructions to the formatter.
'''
self._closed = True
bases = self.get_bases()
broken_bases = sorted(
m for m, includer in six.iteritems(self._included_manifests)
if mozpath.basedir(m, bases) != mozpath.basedir(includer, bases))
for m in broken_bases:
errors.fatal(
'"%s" is included from "%s", which is outside "%s"' %
(m, self._included_manifests[m], mozpath.basedir(m, bases)))
for base in sorted(bases):
self.formatter.add_base(base, self._addons.get(base, False))
self._chrome_queue.execute()
self._queue.execute()
self._file_queue.execute()
def close(self):
'''
Push all instructions to the formatter.
'''
self._closed = True
bases = self.get_bases()
broken_bases = sorted(
m for m, includer in self._included_manifests.iteritems()
if mozpath.basedir(m, bases) != mozpath.basedir(includer, bases))
for m in broken_bases:
errors.fatal('"%s" is included from "%s", which is outside "%s"' %
(m, self._included_manifests[m],
mozpath.basedir(m, bases)))
for base in sorted(bases):
self.formatter.add_base(base, self._addons.get(base, False))
self._chrome_queue.execute()
self._queue.execute()
self._file_queue.execute()
def _get_files_info(self, paths, rev=None):
from mozbuild.frontend.reader import default_finder
from mozpack.files import FileFinder, MercurialRevisionFinder
# Normalize to relative from topsrcdir.
relpaths = []
for p in paths:
a = mozpath.abspath(p)
if not mozpath.basedir(a, [self.topsrcdir]):
raise InvalidPathException('path is outside topsrcdir: %s' % p)
relpaths.append(mozpath.relpath(a, self.topsrcdir))
repo = None
if rev:
hg_path = os.path.join(self.topsrcdir, '.hg')
if not os.path.exists(hg_path):
raise InvalidPathException('a Mercurial repo is required '
'when specifying a revision')
repo = self.topsrcdir
# We need two finders because the reader's finder operates on
# absolute paths.
finder = FileFinder(self.topsrcdir)
if repo:
reader_finder = MercurialRevisionFinder(repo,
rev=rev,
recognize_repo_paths=True)
else:
reader_finder = default_finder
# Expand wildcards.
# One variable is for ordering. The other for membership tests.
# (Membership testing on a list can be slow.)
allpaths = []
all_paths_set = set()
for p in relpaths:
if '*' not in p:
if p not in all_paths_set:
all_paths_set.add(p)
allpaths.append(p)
continue
if repo:
raise InvalidPathException(
'cannot use wildcard in version control mode')
for path, f in finder.find(p):
if path not in all_paths_set:
all_paths_set.add(path)
allpaths.append(path)
reader = self._get_reader(finder=reader_finder)
return reader.files_info(allpaths)
def _get_files_info(self, paths):
relpaths = []
for p in paths:
a = mozpath.abspath(p)
if not mozpath.basedir(a, [self.topsrcdir]):
print('path is not inside topsrcdir: %s' % p)
return 1
relpaths.append(mozpath.relpath(a, self.topsrcdir))
reader = self._get_reader()
return reader.files_info(relpaths)
def _get_files_info(self, paths, rev=None):
from mozbuild.frontend.reader import default_finder
from mozpack.files import FileFinder, MercurialRevisionFinder
# Normalize to relative from topsrcdir.
relpaths = []
for p in paths:
a = mozpath.abspath(p)
if not mozpath.basedir(a, [self.topsrcdir]):
raise InvalidPathException('path is outside topsrcdir: %s' % p)
relpaths.append(mozpath.relpath(a, self.topsrcdir))
repo = None
if rev:
hg_path = os.path.join(self.topsrcdir, '.hg')
if not os.path.exists(hg_path):
raise InvalidPathException('a Mercurial repo is required '
'when specifying a revision')
repo = self.topsrcdir
# We need two finders because the reader's finder operates on
# absolute paths.
finder = FileFinder(self.topsrcdir)
if repo:
reader_finder = MercurialRevisionFinder(repo, rev=rev,
recognize_repo_paths=True)
else:
reader_finder = default_finder
# Expand wildcards.
# One variable is for ordering. The other for membership tests.
# (Membership testing on a list can be slow.)
allpaths = []
all_paths_set = set()
for p in relpaths:
if '*' not in p:
if p not in all_paths_set:
all_paths_set.add(p)
allpaths.append(p)
continue
if repo:
raise InvalidPathException('cannot use wildcard in version control mode')
for path, f in finder.find(p):
if path not in all_paths_set:
all_paths_set.add(path)
allpaths.append(path)
reader = self._get_reader(finder=reader_finder)
return reader.files_info(allpaths)
def add(self, t, flavor, topsrcdir):
t = dict(t)
t['flavor'] = flavor
path = mozpath.normpath(t['path'])
assert mozpath.basedir(path, [topsrcdir])
key = path[len(topsrcdir)+1:]
t['file_relpath'] = key
t['dir_relpath'] = mozpath.dirname(key)
self.tests_by_path[key].append(t)
def match(self, pattern):
"""
Return the list of paths, stored in the container, matching the
given pattern. See the mozpack.path.match documentation for a
description of the handled patterns.
"""
if "*" in pattern:
return [p for p in self.paths() if mozpath.match(p, pattern)]
if pattern == "":
return self.paths()
if pattern in self._files:
return [pattern]
return [p for p in self.paths() if mozpath.basedir(p, [pattern]) == pattern]
def excluded(dest):
if platform.system() != "Windows":
return False
# Exclude local resources from symlinking since the sandbox on Windows
# does not allow accessing reparse points. See bug 1695556.
from buildconfig import topobjdir
denylist = [("dist", "bin"), ("_tests", "modules")]
fulllist = [os.path.join(topobjdir, *paths) for paths in denylist]
fulldest = os.path.join(os.path.abspath(os.curdir), dest)
return mozpath.basedir(fulldest, fulllist) is not None
def match(self, pattern):
'''
Return the list of paths, stored in the container, matching the
given pattern. See the mozpack.path.match documentation for a
description of the handled patterns.
'''
if '*' in pattern:
return [p for p in self.paths()
if mozpath.match(p, pattern)]
if pattern == '':
return self.paths()
if pattern in self._files:
return [pattern]
return [p for p in self.paths()
if mozpath.basedir(p, [pattern]) == pattern]
def _find_relevant_mozbuilds(self, paths):
"""Given a set of filesystem paths, find all relevant moz.build files.
We assume that a moz.build file in the directory ancestry of a given path
is relevant to that path. Let's say we have the following files on disk::
moz.build
foo/moz.build
foo/baz/moz.build
foo/baz/file1
other/moz.build
other/file2
If ``foo/baz/file1`` is passed in, the relevant moz.build files are
``moz.build``, ``foo/moz.build``, and ``foo/baz/moz.build``. For
``other/file2``, the relevant moz.build files are ``moz.build`` and
``other/moz.build``.
Returns a dict of input paths to a list of relevant moz.build files.
The root moz.build file is first and the leaf-most moz.build is last.
"""
root = self.config.topsrcdir
result = {}
@memoize
def exists(path):
return os.path.exists(path)
def itermozbuild(path):
subpath = ''
yield 'moz.build'
for part in mozpath.split(path):
subpath = mozpath.join(subpath, part)
yield mozpath.join(subpath, 'moz.build')
for path in sorted(paths):
path = mozpath.normpath(path)
if os.path.isabs(path):
if not mozpath.basedir(path, [root]):
raise Exception('Path outside topsrcdir: %s' % path)
path = mozpath.relpath(path, root)
result[path] = [
p for p in itermozbuild(path) if exists(mozpath.join(root, p))
]
return result
def _find_relevant_mozbuilds(self, paths):
"""Given a set of filesystem paths, find all relevant moz.build files.
We assume that a moz.build file in the directory ancestry of a given path
is relevant to that path. Let's say we have the following files on disk::
moz.build
foo/moz.build
foo/baz/moz.build
foo/baz/file1
other/moz.build
other/file2
If ``foo/baz/file1`` is passed in, the relevant moz.build files are
``moz.build``, ``foo/moz.build``, and ``foo/baz/moz.build``. For
``other/file2``, the relevant moz.build files are ``moz.build`` and
``other/moz.build``.
Returns a dict of input paths to a list of relevant moz.build files.
The root moz.build file is first and the leaf-most moz.build is last.
"""
root = self.config.topsrcdir
result = {}
@memoize
def exists(path):
return os.path.exists(path)
def itermozbuild(path):
subpath = ''
yield 'moz.build'
for part in mozpath.split(path):
subpath = mozpath.join(subpath, part)
yield mozpath.join(subpath, 'moz.build')
for path in sorted(paths):
path = mozpath.normpath(path)
if os.path.isabs(path):
if not mozpath.basedir(path, [root]):
raise Exception('Path outside topsrcdir: %s' % path)
path = mozpath.relpath(path, root)
result[path] = [p for p in itermozbuild(path)
if exists(mozpath.join(root, p))]
return result
def add(self, t, flavor=None, topsrcdir=None):
t = dict(t)
t["flavor"] = flavor
if topsrcdir is None:
topsrcdir = self.topsrcdir
else:
topsrcdir = mozpath.normpath(topsrcdir)
path = mozpath.normpath(t["path"])
assert mozpath.basedir(path, [topsrcdir])
key = path[len(topsrcdir) + 1 :]
t["file_relpath"] = key
t["dir_relpath"] = mozpath.dirname(key)
self.tests_by_path[key].append(t)
def add(self, t, flavor=None, topsrcdir=None):
t = dict(t)
t['flavor'] = flavor
if topsrcdir is None:
topsrcdir = self.topsrcdir
else:
topsrcdir = mozpath.normpath(topsrcdir)
path = mozpath.normpath(t['path'])
assert mozpath.basedir(path, [topsrcdir])
key = path[len(topsrcdir) + 1:]
t['file_relpath'] = key
t['dir_relpath'] = mozpath.dirname(key)
self.tests_by_path[key].append(t)
def _find(self, pattern):
'''
Actual implementation of JarFinder.find(), dispatching to specialized
member functions depending on what kind of pattern was given.
'''
if '*' in pattern:
for p in self._files:
if mozpath.match(p, pattern):
yield p, DeflatedFile(self._files[p])
elif pattern == '':
for p in self._files:
yield p, DeflatedFile(self._files[p])
elif pattern in self._files:
yield pattern, DeflatedFile(self._files[pattern])
else:
for p in self._files:
if mozpath.basedir(p, [pattern]) == pattern:
yield p, DeflatedFile(self._files[p])
def _find(self, pattern):
'''
Actual implementation of JarFinder.find(), dispatching to specialized
member functions depending on what kind of pattern was given.
'''
if '*' in pattern:
for p in self._files:
if mozpath.match(p, pattern):
yield p, DeflatedFile(self._files[p])
elif pattern == '':
for p in self._files:
yield p, DeflatedFile(self._files[p])
elif pattern in self._files:
yield pattern, DeflatedFile(self._files[pattern])
else:
for p in self._files:
if mozpath.basedir(p, [pattern]) == pattern:
yield p, DeflatedFile(self._files[p])
def _get_files_info(command_context, paths, rev=None):
reader = command_context.mozbuild_reader(config_mode="empty",
vcs_revision=rev)
# Normalize to relative from topsrcdir.
relpaths = []
for p in paths:
a = mozpath.abspath(p)
if not mozpath.basedir(a, [command_context.topsrcdir]):
raise InvalidPathException("path is outside topsrcdir: %s" % p)
relpaths.append(mozpath.relpath(a, command_context.topsrcdir))
# Expand wildcards.
# One variable is for ordering. The other for membership tests.
# (Membership testing on a list can be slow.)
allpaths = []
all_paths_set = set()
for p in relpaths:
if "*" not in p:
if p not in all_paths_set:
if not os.path.exists(
mozpath.join(command_context.topsrcdir, p)):
print("(%s does not exist; ignoring)" % p, file=sys.stderr)
continue
all_paths_set.add(p)
allpaths.append(p)
continue
if rev:
raise InvalidPathException(
"cannot use wildcard in version control mode")
# finder is rooted at / for now.
# TODO bug 1171069 tracks changing to relative.
search = mozpath.join(command_context.topsrcdir, p)[1:]
for path, f in reader.finder.find(search):
path = path[len(command_context.topsrcdir):]
if path not in all_paths_set:
all_paths_set.add(path)
allpaths.append(path)
return reader.files_info(allpaths)
def _get_files_info(self, paths, rev=None):
reader = self.mozbuild_reader(config_mode='empty', vcs_revision=rev)
# Normalize to relative from topsrcdir.
relpaths = []
for p in paths:
a = mozpath.abspath(p)
if not mozpath.basedir(a, [self.topsrcdir]):
raise InvalidPathException('path is outside topsrcdir: %s' % p)
relpaths.append(mozpath.relpath(a, self.topsrcdir))
# Expand wildcards.
# One variable is for ordering. The other for membership tests.
# (Membership testing on a list can be slow.)
allpaths = []
all_paths_set = set()
for p in relpaths:
if '*' not in p:
if p not in all_paths_set:
if not os.path.exists(mozpath.join(self.topsrcdir, p)):
print('(%s does not exist; ignoring)' % p,
file=sys.stderr)
continue
all_paths_set.add(p)
allpaths.append(p)
continue
if rev:
raise InvalidPathException('cannot use wildcard in version control mode')
# finder is rooted at / for now.
# TODO bug 1171069 tracks changing to relative.
search = mozpath.join(self.topsrcdir, p)[1:]
for path, f in reader.finder.find(search):
path = path[len(self.topsrcdir):]
if path not in all_paths_set:
all_paths_set.add(path)
allpaths.append(path)
return reader.files_info(allpaths)
def _process_final_target_files(self, obj):
target = obj.install_target
path = mozpath.basedir(target, (
'dist/bin',
'dist/xpi-stage',
'_tests',
'dist/include',
'dist/branding',
'dist/sdk',
))
if not path:
raise Exception("Cannot install to " + target)
reltarget = mozpath.relpath(target, path)
for path, files in obj.files.walk():
backend_file = self._get_backend_file(mozpath.join(target, path))
for f in files:
assert not isinstance(f, RenamedSourcePath)
dest = mozpath.join(reltarget, path, f.target_basename)
if not isinstance(f, ObjDirPath):
if '*' in f:
if f.startswith('/') or isinstance(f, AbsolutePath):
basepath, wild = os.path.split(f.full_path)
if '*' in basepath:
raise Exception("Wildcards are only supported in the filename part of "
"srcdir-relative or absolute paths.")
# TODO: This is only needed for Windows, so we can
# skip this for now.
pass
else:
# TODO: This is needed for tests
pass
else:
backend_file.symlink_rule(f.full_path, output_group=self._installed_files)
else:
# TODO: Support installing generated files
pass
def _find_helper(self, pattern, files, file_getter):
"""Generic implementation of _find.
A few *Finder implementations share logic for returning results.
This function implements the custom logic.
The ``file_getter`` argument is a callable that receives a path
that is known to exist. The callable should return a ``BaseFile``
instance.
"""
if '*' in pattern:
for p in files:
if mozpath.match(p, pattern):
yield p, file_getter(p)
elif pattern == '':
for p in files:
yield p, file_getter(p)
elif pattern in files:
yield pattern, file_getter(pattern)
else:
for p in files:
if mozpath.basedir(p, [pattern]) == pattern:
yield p, file_getter(p)
def add(self, t, flavor, topsrcdir, default_supp_files):
t = dict(t)
t["flavor"] = flavor
path = mozpath.normpath(t["path"])
assert mozpath.basedir(path, [topsrcdir])
key = path[len(topsrcdir) + 1 :]
t["file_relpath"] = key
t["dir_relpath"] = mozpath.dirname(key)
# Support files are propagated from the default section to individual
# tests by the manifest parser, but we end up storing a lot of
# redundant data due to the huge number of support files.
# So if we have support files that are the same as the manifest default
# we track that separately, per-manifest instead of per-test, to save
# space.
supp_files = t.get("support-files")
if supp_files and supp_files == default_supp_files:
self.manifest_default_support_files[t["manifest"]] = default_supp_files
del t["support-files"]
self.tests_by_path[key].append(t)
def add_manifest(self, entry):
# Store manifest entries in a single manifest per directory, named
# after their parent directory, except for root manifests, all named
# chrome.manifest.
base = self._get_base(entry.base)
if entry.base == base:
name = 'chrome'
else:
name = mozpath.basename(entry.base)
path = mozpath.normpath(mozpath.join(entry.base,
'%s.manifest' % name))
if not self.copier.contains(path):
assert mozpath.basedir(entry.base, [base]) == base
# Add a reference to the manifest file in the parent manifest, if
# the manifest file is not a root manifest.
if len(entry.base) > len(base):
parent = mozpath.dirname(entry.base)
relbase = mozpath.basename(entry.base)
relpath = mozpath.join(relbase,
mozpath.basename(path))
FlatFormatter.add_manifest(self, Manifest(parent, relpath))
self.copier.add(path, ManifestFile(entry.base))
self.copier[path].add(entry)
def exec_file(self, path):
'''Execute one file within the sandbox. Users of this class probably
want to use `run` instead.'''
if self._paths:
path = mozpath.join(mozpath.dirname(self._paths[-1]), path)
if not mozpath.basedir(path, (mozpath.dirname(self._paths[0]),)):
raise ConfigureError(
'Cannot include `%s` because it is not in a subdirectory '
'of `%s`' % (path, mozpath.dirname(self._paths[0])))
else:
path = mozpath.realpath(mozpath.abspath(path))
if path in self._paths:
raise ConfigureError(
'Cannot include `%s` because it was included already.' % path)
self._paths.append(path)
source = open(path, 'rb').read()
code = compile(source, path, 'exec')
exec(code, self)
self._paths.pop(-1)
def _repack(app_finder, l10n_finder, copier, formatter, non_chrome=set()):
app = LocaleManifestFinder(app_finder)
l10n = LocaleManifestFinder(l10n_finder)
# The code further below assumes there's only one locale replaced with
# another one.
if len(app.locales) > 1 or len(l10n.locales) > 1:
errors.fatal("Multiple locales aren't supported")
locale = app.locales[0]
l10n_locale = l10n.locales[0]
# For each base directory, store what path a locale chrome package name
# corresponds to.
# e.g., for the following entry under app/chrome:
# locale foo en-US path/to/files
# keep track that the locale path for foo in app is
# app/chrome/path/to/files.
l10n_paths = {}
for e in l10n.entries:
if isinstance(e, ManifestChrome):
base = mozpath.basedir(e.path, app.bases)
l10n_paths.setdefault(base, {})
l10n_paths[base][e.name] = e.path
# For chrome and non chrome files or directories, store what langpack path
# corresponds to a package path.
paths = {}
for e in app.entries:
if isinstance(e, ManifestEntryWithRelPath):
base = mozpath.basedir(e.path, app.bases)
if base not in l10n_paths:
errors.fatal("Locale doesn't contain %s/" % base)
# Allow errors to accumulate
continue
if e.name not in l10n_paths[base]:
errors.fatal("Locale doesn't have a manifest entry for '%s'" %
e.name)
# Allow errors to accumulate
continue
paths[e.path] = l10n_paths[base][e.name]
for pattern in non_chrome:
for base in app.bases:
path = mozpath.join(base, pattern)
left = set(p for p, f in app_finder.find(path))
right = set(p for p, f in l10n_finder.find(path))
for p in right:
paths[p] = p
for p in left - right:
paths[p] = None
# Create a new package, with non localized bits coming from the original
# package, and localized bits coming from the langpack.
packager = SimplePackager(formatter)
for p, f in app_finder:
if is_manifest(p):
# Remove localized manifest entries.
for e in [e for e in f if e.localized]:
f.remove(e)
# If the path is one that needs a locale replacement, use the
# corresponding file from the langpack.
path = None
if p in paths:
path = paths[p]
if not path:
continue
else:
base = mozpath.basedir(p, paths.keys())
if base:
subpath = mozpath.relpath(p, base)
path = mozpath.normpath(mozpath.join(paths[base],
subpath))
if path:
files = [f for p, f in l10n_finder.find(path)]
if not len(files):
if base not in non_chrome:
errors.error("Missing file: %s" %
os.path.join(l10n_finder.base, path))
else:
packager.add(path, files[0])
else:
packager.add(p, f)
# Add localized manifest entries from the langpack.
l10n_manifests = []
for base in set(e.base for e in l10n.entries):
m = ManifestFile(base, [e for e in l10n.entries if e.base == base])
path = mozpath.join(base, 'chrome.%s.manifest' % l10n_locale)
l10n_manifests.append((path, m))
bases = packager.get_bases()
for path, m in l10n_manifests:
base = mozpath.basedir(path, bases)
packager.add(path, m)
# Add a "manifest $path" entry in the top manifest under that base.
m = ManifestFile(base)
m.add(Manifest(base, mozpath.relpath(path, base)))
packager.add(mozpath.join(base, 'chrome.manifest'), m)
packager.close()
# Add any remaining non chrome files.
for pattern in non_chrome:
for base in bases:
for p, f in l10n_finder.find(mozpath.join(base, pattern)):
if not formatter.contains(p):
formatter.add(p, f)
# Transplant jar preloading information.
for path, log in app_finder.jarlogs.iteritems():
assert isinstance(copier[path], Jarrer)
copier[path].preload([l.replace(locale, l10n_locale) for l in log])
def _read_mozbuild(self, path, config, descend, metadata):
path = mozpath.normpath(path)
log(self._log, logging.DEBUG, 'read_mozbuild', {'path': path},
'Reading file: {path}')
if path in self._read_files:
log(self._log, logging.WARNING, 'read_already', {'path': path},
'File already read. Skipping: {path}')
return
self._read_files.add(path)
time_start = time.time()
topobjdir = config.topobjdir
if not mozpath.basedir(path, [config.topsrcdir]):
external = config.external_source_dir
if external and mozpath.basedir(path, [external]):
config = ConfigEnvironment.from_config_status(
mozpath.join(topobjdir, 'config.status'))
config.topsrcdir = external
config.external_source_dir = None
relpath = mozpath.relpath(path, config.topsrcdir)
reldir = mozpath.dirname(relpath)
if mozpath.dirname(relpath) == 'js/src' and \
not config.substs.get('JS_STANDALONE'):
config = ConfigEnvironment.from_config_status(
mozpath.join(topobjdir, reldir, 'config.status'))
config.topobjdir = topobjdir
config.external_source_dir = None
context = Context(VARIABLES, config)
sandbox = MozbuildSandbox(context, metadata=metadata)
sandbox.exec_file(path)
context.execution_time = time.time() - time_start
# Yield main context before doing any processing. This gives immediate
# consumers an opportunity to change state before our remaining
# processing is performed.
yield context
# We first collect directories populated in variables.
dir_vars = ['DIRS']
if context.config.substs.get('ENABLE_TESTS', False) == '1':
dir_vars.append('TEST_DIRS')
dirs = [(v, context[v]) for v in dir_vars if v in context]
curdir = mozpath.dirname(path)
gyp_contexts = []
for target_dir in context.get('GYP_DIRS', []):
gyp_dir = context['GYP_DIRS'][target_dir]
for v in ('input', 'variables'):
if not getattr(gyp_dir, v):
raise SandboxValidationError('Missing value for '
'GYP_DIRS["%s"].%s' % (target_dir, v), context)
# The make backend assumes contexts for sub-directories are
# emitted after their parent, so accumulate the gyp contexts.
# We could emit the parent context before processing gyp
# configuration, but we need to add the gyp objdirs to that context
# first.
from .gyp_reader import read_from_gyp
non_unified_sources = set()
for s in gyp_dir.non_unified_sources:
source = mozpath.normpath(mozpath.join(curdir, s))
if not os.path.exists(source):
raise SandboxValidationError('Cannot find %s.' % source,
context)
non_unified_sources.add(source)
for gyp_context in read_from_gyp(context.config,
mozpath.join(curdir, gyp_dir.input),
mozpath.join(context.objdir,
target_dir),
gyp_dir.variables,
non_unified_sources = non_unified_sources):
gyp_context.update(gyp_dir.sandbox_vars)
gyp_contexts.append(gyp_context)
for gyp_context in gyp_contexts:
context['DIRS'].append(mozpath.relpath(gyp_context.objdir, context.objdir))
sandbox.subcontexts.append(gyp_context)
for subcontext in sandbox.subcontexts:
yield subcontext
# Traverse into referenced files.
# It's very tempting to use a set here. Unfortunately, the recursive
# make backend needs order preserved. Once we autogenerate all backend
# files, we should be able to convert this to a set.
recurse_info = OrderedDict()
for var, var_dirs in dirs:
for d in var_dirs:
if d in recurse_info:
raise SandboxValidationError(
'Directory (%s) registered multiple times in %s' % (
mozpath.relpath(d, context.srcdir), var), context)
recurse_info[d] = {}
for key in sandbox.metadata:
if key == 'exports':
sandbox.recompute_exports()
recurse_info[d][key] = dict(sandbox.metadata[key])
for path, child_metadata in recurse_info.items():
child_path = path.join('moz.build')
# Ensure we don't break out of the topsrcdir. We don't do realpath
# because it isn't necessary. If there are symlinks in the srcdir,
# that's not our problem. We're not a hosted application: we don't
# need to worry about security too much.
if not is_read_allowed(child_path, context.config):
raise SandboxValidationError(
'Attempting to process file outside of allowed paths: %s' %
child_path, context)
if not descend:
continue
for res in self.read_mozbuild(child_path, context.config,
metadata=child_metadata):
yield res
self._execution_stack.pop()
def __iter__(self):
for p, f in self._registry:
if mozpath.basedir(p, [self._base]):
yield mozpath.relpath(p, self._base), f
def _process_final_target_files(self, obj):
target = obj.install_target
if not isinstance(obj, ObjdirFiles):
path = mozpath.basedir(target, (
'dist/bin',
'dist/xpi-stage',
'_tests',
'dist/include',
'dist/sdk',
))
if not path:
raise Exception("Cannot install to " + target)
if target.startswith('_tests'):
# TODO: TEST_HARNESS_FILES present a few challenges for the tup
# backend (bug 1372381).
return
for path, files in obj.files.walk():
for f in files:
if not isinstance(f, ObjDirPath):
backend_file = self._get_backend_file(mozpath.join(target, path))
if '*' in f:
if f.startswith('/') or isinstance(f, AbsolutePath):
basepath, wild = os.path.split(f.full_path)
if '*' in basepath:
raise Exception("Wildcards are only supported in the filename part of "
"srcdir-relative or absolute paths.")
# TODO: This is only needed for Windows, so we can
# skip this for now.
pass
else:
def _prefix(s):
for p in mozpath.split(s):
if '*' not in p:
yield p + '/'
prefix = ''.join(_prefix(f.full_path))
self.backend_input_files.add(prefix)
finder = FileFinder(prefix)
for p, _ in finder.find(f.full_path[len(prefix):]):
backend_file.symlink_rule(mozpath.join(prefix, p),
output=mozpath.join(f.target_basename, p),
output_group=self._installed_files)
else:
backend_file.symlink_rule(f.full_path, output=f.target_basename, output_group=self._installed_files)
else:
if (self.environment.is_artifact_build and
any(mozpath.match(f.target_basename, p) for p in self._compile_env_gen_files)):
# If we have an artifact build we never would have generated this file,
# so do not attempt to install it.
continue
# We're not generating files in these directories yet, so
# don't attempt to install files generated from them.
if f.context.relobjdir not in ('layout/style/test',
'toolkit/library'):
output = mozpath.join('$(MOZ_OBJ_ROOT)', target, path,
f.target_basename)
gen_backend_file = self._get_backend_file(f.context.relobjdir)
gen_backend_file.symlink_rule(f.full_path, output=output,
output_group=self._installed_files)
def _get_base(self, path):
'''
Return the deepest base directory containing the given path.
'''
self._frozen_bases = True
return mozpath.basedir(path, self._bases)