def test_serialization(self):
m = self._get_test_manifest()
p = self.tmppath('m')
m.write(path=p)
self.assertTrue(os.path.isfile(p))
with open(p, 'rb') as fh:
c = fh.read()
self.assertEqual(c.count('\n'), 5)
lines = c.splitlines()
self.assertEqual(len(lines), 5)
self.assertEqual(lines[0], '2')
self.assertEqual(lines[1], '2\x1fc_dest\x1f%s' %
self.tmppath('c_source'))
self.assertEqual(lines[2], '3\x1fe_dest')
self.assertEqual(lines[3], '4\x1fo_dest')
self.assertEqual(lines[4], '1\x1fs_dest\x1f%s' %
self.tmppath('s_source'))
m2 = InstallManifest(path=p)
self.assertEqual(m, m2)
p2 = self.tmppath('m2')
m2.write(path=p2)
with open(p2, 'rb') as fh:
c2 = fh.read()
self.assertEqual(c, c2)
def test_serialization(self):
m = self._get_test_manifest()
p = self.tmppath("m")
m.write(path=p)
self.assertTrue(os.path.isfile(p))
with open(p, "rb") as fh:
c = fh.read()
self.assertEqual(c.count("\n"), 4)
lines = c.splitlines()
self.assertEqual(len(lines), 4)
self.assertEqual(lines[0], "1")
self.assertEqual(lines[1], "2\x1fc_dest\x1f%s" % self.tmppath("c_source"))
self.assertEqual(lines[2], "3\x1fe_dest")
self.assertEqual(lines[3], "1\x1fs_dest\x1f%s" % self.tmppath("s_source"))
m2 = InstallManifest(path=p)
self.assertEqual(m, m2)
p2 = self.tmppath("m2")
m2.write(path=p2)
with open(p2, "rb") as fh:
c2 = fh.read()
self.assertEqual(c, c2)
def test_serialization(self):
m = self._get_test_manifest()
p = self.tmppath('m')
m.write(path=p)
self.assertTrue(os.path.isfile(p))
with open(p, 'rb') as fh:
c = fh.read()
self.assertEqual(c.count('\n'), 9)
lines = c.splitlines()
self.assertEqual(len(lines), 9)
self.assertEqual(lines[0], '5')
m2 = InstallManifest(path=p)
self.assertEqual(m, m2)
p2 = self.tmppath('m2')
m2.write(path=p2)
with open(p2, 'rb') as fh:
c2 = fh.read()
self.assertEqual(c, c2)
def test_serialization(self):
m = self._get_test_manifest()
p = self.tmppath("m")
m.write(path=p)
self.assertTrue(os.path.isfile(p))
with open(p, "rb") as fh:
c = fh.read()
self.assertEqual(c.count("\n"), 8)
lines = c.splitlines()
self.assertEqual(len(lines), 8)
self.assertEqual(lines[0], "4")
m2 = InstallManifest(path=p)
self.assertEqual(m, m2)
p2 = self.tmppath("m2")
m2.write(path=p2)
with open(p2, "rb") as fh:
c2 = fh.read()
self.assertEqual(c, c2)
def process_manifest(destdir, paths, remove_unaccounted=True):
manifest = InstallManifest()
for path in paths:
manifest |= InstallManifest(path=path)
copier = FileCopier()
manifest.populate_registry(copier)
return copier.copy(destdir, remove_unaccounted=remove_unaccounted)
def process_manifest(destdir, *paths):
manifest = InstallManifest()
for path in paths:
manifest |= InstallManifest(path=path)
copier = FileCopier()
manifest.populate_registry(copier)
return copier.copy(destdir)
def test_final_target_files_wildcard(self):
"""Ensure that wildcards in FINAL_TARGET_FILES work properly."""
env = self._consume('final-target-files-wildcard', FasterMakeBackend)
m = InstallManifest(path=mozpath.join(env.topobjdir,
'faster', 'install_dist_bin'))
self.assertEqual(len(m), 1)
reg = FileRegistry()
m.populate_registry(reg)
expected = [('foo/bar.xyz', 'bar.xyz'), ('foo/foo.xyz', 'foo.xyz')]
actual = [(path, mozpath.relpath(f.path, env.topsrcdir)) for (path, f) in reg]
self.assertEqual(expected, actual)
def _get_test_manifest(self):
m = InstallManifest()
m.add_symlink(self.tmppath('s_source'), 's_dest')
m.add_copy(self.tmppath('c_source'), 'c_dest')
m.add_required_exists('e_dest')
m.add_optional_exists('o_dest')
return m
def file_copier(self):
# TODO: invalidate the file copier when the build system
# itself changes, i.e., the underlying unified manifest
# changes.
file_copier = FileCopier()
unified_manifest = InstallManifest(
mozpath.join(self.config_environment.topobjdir,
'faster', 'unified_install_dist_bin'))
unified_manifest.populate_registry(file_copier, defines_override=self.defines)
return file_copier
def test_old_install_manifest_deleted(self):
# Simulate an install manifest from a previous backend version. Ensure
# it is deleted.
env = self._get_environment('stub0')
purge_dir = mozpath.join(env.topobjdir, '_build_manifests', 'install')
manifest_path = mozpath.join(purge_dir, 'old_manifest')
os.makedirs(purge_dir)
m = InstallManifest()
m.write(path=manifest_path)
self.assertTrue(os.path.exists(manifest_path))
self._consume('stub0', RecursiveMakeBackend, env)
self.assertFalse(os.path.exists(manifest_path))
def test_or(self):
m1 = self._get_test_manifest()
m2 = InstallManifest()
m2.add_symlink("s_source2", "s_dest2")
m2.add_copy("c_source2", "c_dest2")
m1 |= m2
self.assertEqual(len(m2), 2)
self.assertEqual(len(m1), 5)
self.assertIn("s_dest2", m1)
self.assertIn("c_dest2", m1)
def process_manifest(destdir, paths,
remove_unaccounted=True,
remove_all_directory_symlinks=True,
remove_empty_directories=True):
manifest = InstallManifest()
for path in paths:
manifest |= InstallManifest(path=path)
copier = FileCopier()
manifest.populate_registry(copier)
return copier.copy(destdir,
remove_unaccounted=remove_unaccounted,
remove_all_directory_symlinks=remove_all_directory_symlinks,
remove_empty_directories=remove_empty_directories)
def find_generated_harness_files():
# TEST_HARNESS_FILES end up in an install manifest at
# $topsrcdir/_build_manifests/install/_tests.
manifest = InstallManifest(mozpath.join(buildconfig.topobjdir,
'_build_manifests',
'install',
'_tests'))
registry = FileRegistry()
manifest.populate_registry(registry)
# Conveniently, the generated files we care about will already
# exist in the objdir, so we can identify relevant files if
# they're an `ExistingFile` instance.
return [mozpath.join('_tests', p) for p in registry.paths()
if isinstance(registry[p], ExistingFile)]
def _init(self):
CommonBackend._init(self)
self._backend_files = {}
self._ipdl_sources = set()
def detailed(summary):
return '{:d} total backend files. {:d} created; {:d} updated; {:d} unchanged'.format(
summary.managed_count, summary.created_count,
summary.updated_count, summary.unchanged_count)
# This is a little kludgy and could be improved with a better API.
self.summary.backend_detailed_summary = types.MethodType(
detailed, self.summary)
self.xpcshell_manifests = []
self.backend_input_files.add(
os.path.join(self.environment.topobjdir, 'config', 'autoconf.mk'))
self._install_manifests = dict()
self._purge_manifests = dict(
dist_bin=PurgeManifest(relpath='dist/bin'),
dist_include=PurgeManifest(relpath='dist/include'),
dist_private=PurgeManifest(relpath='dist/private'),
dist_public=PurgeManifest(relpath='dist/public'),
dist_sdk=PurgeManifest(relpath='dist/sdk'),
tests=PurgeManifest(relpath='_tests'),
xpidl=PurgeManifest(relpath='config/makefiles/xpidl'),
)
self._install_manifests = dict(dist_idl=InstallManifest(), )
def _init(self):
CommonBackend._init(self)
self._backend_files = {}
self._ipdl_sources = set()
self._webidl_sources = set()
self._generated_events_webidl_sources = set()
self._test_webidl_sources = set()
self._preprocessed_test_webidl_sources = set()
self._preprocessed_webidl_sources = set()
self._generated_webidl_sources = set()
def detailed(summary):
s = '{:d} total backend files. {:d} created; {:d} updated; {:d} unchanged'.format(
summary.created_count + summary.updated_count +
summary.unchanged_count, summary.created_count,
summary.updated_count, summary.unchanged_count)
if summary.deleted_count:
s += '; {:d} deleted'.format(summary.deleted_count)
return s
# This is a little kludgy and could be improved with a better API.
self.summary.backend_detailed_summary = types.MethodType(
detailed, self.summary)
self._test_manifests = {}
self.backend_input_files.add(
os.path.join(self.environment.topobjdir, 'config', 'autoconf.mk'))
self._install_manifests = {
k: InstallManifest()
for k in [
'dist_bin',
'dist_idl',
'dist_include',
'dist_public',
'dist_private',
'dist_sdk',
'tests',
'xpidl',
]
}
self._traversal = RecursiveMakeTraversal()
self._may_skip = {
'export': set(),
'compile': set(),
'binaries': set(),
'libs': set(),
'tools': set(),
}
derecurse = self.environment.substs.get('MOZ_PSEUDO_DERECURSE',
'').split(',')
self._parallel_export = False
self._no_skip = False
if derecurse != ['']:
self._parallel_export = 'no-parallel-export' not in derecurse
self._no_skip = 'no-skip' in derecurse
def find_generated_harness_files():
# TEST_HARNESS_FILES end up in an install manifest at
# $topsrcdir/_build_manifests/install/_tests.
manifest = InstallManifest(
mozpath.join(buildconfig.topobjdir, "_build_manifests", "install", "_tests")
)
registry = FileRegistry()
manifest.populate_registry(registry)
# Conveniently, the generated files we care about will already
# exist in the objdir, so we can identify relevant files if
# they're an `ExistingFile` instance.
return [
mozpath.join("_tests", p)
for p in registry.paths()
if isinstance(registry[p], ExistingFile)
]
def test_old_install_manifest_deleted(self):
# Simulate an install manifest from a previous backend version. Ensure
# it is deleted.
env = self._get_environment('stub0')
purge_dir = mozpath.join(env.topobjdir, '_build_manifests', 'install')
manifest_path = mozpath.join(purge_dir, 'old_manifest')
os.makedirs(purge_dir)
m = InstallManifest()
m.write(path=manifest_path)
with open(mozpath.join(env.topobjdir, 'backend.RecursiveMakeBackend'),
'w') as f:
f.write('%s\n' % manifest_path)
self.assertTrue(os.path.exists(manifest_path))
self._consume('stub0', RecursiveMakeBackend, env)
self.assertFalse(os.path.exists(manifest_path))
def process_manifest(destdir,
paths,
remove_unaccounted=True,
remove_all_directory_symlinks=True,
remove_empty_directories=True):
manifest = InstallManifest()
for path in paths:
manifest |= InstallManifest(path=path)
copier = FileCopier()
manifest.populate_registry(copier)
return copier.copy(
destdir,
remove_unaccounted=remove_unaccounted,
remove_all_directory_symlinks=remove_all_directory_symlinks,
remove_empty_directories=remove_empty_directories)
def test_install_manifests_package_tests(self):
"""Ensure test suites honor package_tests=False."""
env = self._consume('test-manifests-package-tests',
RecursiveMakeBackend)
tests_dir = mozpath.join(env.topobjdir, '_tests')
all_tests_path = mozpath.join(env.topobjdir, 'all-tests.json')
self.assertTrue(os.path.exists(all_tests_path))
with open(all_tests_path, 'rt') as fh:
o = json.load(fh)
self.assertIn('mochitest.js', o)
self.assertIn('not_packaged.java', o)
man_dir = mozpath.join(env.topobjdir, '_build_manifests', 'install')
self.assertTrue(os.path.isdir(man_dir))
full = mozpath.join(man_dir, '_tests')
self.assertTrue(os.path.exists(full))
m = InstallManifest(path=full)
# Only mochitest.js should be in the install manifest.
self.assertTrue('testing/mochitest/tests/mochitest.js' in m)
# The path is odd here because we do not normalize at test manifest
# processing time. This is a fragile test because there's currently no
# way to iterate the manifest.
self.assertFalse('instrumentation/./not_packaged.java' in m)
def assertNotInManifest(self, project_name, *args):
manifest_path = mozpath.join(self.env.topobjdir, 'android_eclipse',
'%s.manifest' % project_name)
manifest = InstallManifest(manifest_path)
for arg in args:
self.assertNotIn(
arg, manifest,
'%s not in manifest for project %s' % (arg, project_name))
def describe_install_manifest(manifest, dest_dir):
try:
manifest = InstallManifest(manifest)
except UnreadableInstallManifest:
raise IOError(errno.EINVAL, 'Error parsing manifest file', manifest)
reg = FileRegistry()
mapping = {}
manifest.populate_registry(reg)
for dest_file, src in reg:
if hasattr(src, 'path'):
dest_path = mozpath.join(dest_dir, dest_file)
relsrc_path = mozpath.relpath(src.path, buildconfig.topsrcdir)
mapping[dest_path] = relsrc_path
return mapping
def test_test_manifests_duplicate_support_files(self):
"""Ensure duplicate support-files in test manifests work."""
env = self._consume('test-manifests-duplicate-support-files',
RecursiveMakeBackend)
p = os.path.join(env.topobjdir, '_build_manifests', 'install', 'tests')
m = InstallManifest(p)
self.assertIn('testing/mochitest/tests/support-file.txt', m)
def _synchronize_docs(self):
m = InstallManifest()
m.add_symlink(self._conf_py_path, 'conf.py')
for dest, source in sorted(self._trees.items()):
source_dir = os.path.join(self._topsrcdir, source)
for root, dirs, files in os.walk(source_dir):
for f in files:
source_path = os.path.join(root, f)
rel_source = source_path[len(source_dir) + 1:]
m.add_symlink(source_path, os.path.join(dest, rel_source))
stage_dir = os.path.join(self._output_dir, 'staging')
copier = FileCopier()
m.populate_registry(copier)
copier.copy(stage_dir)
with open(self._index_path, 'rb') as fh:
data = fh.read()
indexes = ['%s/index' % p for p in sorted(self._trees.keys())]
indexes = '\n '.join(indexes)
packages = [os.path.basename(p) for p in self._python_package_dirs]
packages = ['python/%s' % p for p in packages]
packages = '\n '.join(sorted(packages))
data = data.format(indexes=indexes, python_packages=packages)
with open(os.path.join(stage_dir, 'index.rst'), 'wb') as fh:
fh.write(data)
def _synchronize_docs(self):
m = InstallManifest()
m.add_symlink(self._conf_py_path, "conf.py")
for dest, source in sorted(self._trees.items()):
source_dir = os.path.join(self._topsrcdir, source)
for root, dirs, files in os.walk(source_dir):
for f in files:
source_path = os.path.join(root, f)
rel_source = source_path[len(source_dir) + 1 :]
m.add_symlink(source_path, os.path.join(dest, rel_source))
copier = FileCopier()
m.populate_registry(copier)
copier.copy(self._docs_dir)
with open(self._index_path, "rb") as fh:
data = fh.read()
indexes = ["%s/index" % p for p in sorted(self._trees.keys())]
indexes = "\n ".join(indexes)
packages = [os.path.basename(p) for p in self._python_package_dirs]
packages = ["python/%s" % p for p in packages]
packages = "\n ".join(sorted(packages))
data = data.format(indexes=indexes, python_packages=packages)
with open(os.path.join(self._docs_dir, "index.rst"), "wb") as fh:
fh.write(data)
def _get_test_manifest(self):
m = InstallManifest()
m.add_symlink(self.tmppath("s_source"), "s_dest")
m.add_copy(self.tmppath("c_source"), "c_dest")
m.add_required_exists("e_dest")
return m
def test_test_support_files_tracked(self):
env = self._consume('test-support-binaries-tracked', RecursiveMakeBackend)
m = InstallManifest(path=mozpath.join(env.topobjdir,
'_build_manifests', 'install', '_tests'))
self.assertEqual(len(m), 4)
self.assertIn('xpcshell/tests/mozbuildtest/test-library.dll', m)
self.assertIn('xpcshell/tests/mozbuildtest/test-one.exe', m)
self.assertIn('xpcshell/tests/mozbuildtest/test-two.exe', m)
self.assertIn('xpcshell/tests/mozbuildtest/host-test-library.dll', m)
def test_pattern_expansion(self):
source = self.tmppath("source")
os.mkdir(source)
os.mkdir("%s/base" % source)
os.mkdir("%s/base/foo" % source)
with open("%s/base/foo/file1" % source, "a"):
pass
with open("%s/base/foo/file2" % source, "a"):
pass
m = InstallManifest()
m.add_pattern_symlink("%s/base" % source, "**", "dest")
c = FileCopier()
m.populate_registry(c)
self.assertEqual(c.paths(), ["dest/foo/file1", "dest/foo/file2"])
def test_pattern_expansion(self):
source = self.tmppath('source')
os.mkdir(source)
os.mkdir('%s/base' % source)
os.mkdir('%s/base/foo' % source)
with open('%s/base/foo/file1' % source, 'a'):
pass
with open('%s/base/foo/file2' % source, 'a'):
pass
m = InstallManifest()
m.add_pattern_symlink('%s/base' % source, '**', 'dest')
c = FileCopier()
m.populate_registry(c)
self.assertEqual(c.paths(), ['dest/foo/file1', 'dest/foo/file2'])
def describe_install_manifest(manifest, dest_dir):
try:
manifest = InstallManifest(manifest)
except UnreadableInstallManifest:
raise IOError(errno.EINVAL, 'Error parsing manifest file', manifest)
reg = FileRegistry()
mapping = {}
manifest.populate_registry(reg)
dest_dir = mozpath.join(buildconfig.topobjdir, dest_dir)
for dest_file, src in reg:
if hasattr(src, 'path'):
dest_path = mozpath.join(dest_dir, dest_file)
relsrc_path = mozpath.relpath(src.path, buildconfig.topsrcdir)
mapping[dest_path] = relsrc_path
return mapping
def consume_finished(self):
mp = os.path.join(self.environment.topobjdir, '_build_manifests', 'install', '_tests')
install_manifest = InstallManifest(mp)
reg = FileRegistry()
install_manifest.populate_registry(reg)
for dest, src in reg:
if not hasattr(src, 'path'):
continue
if not os.path.isabs(dest):
dest = '_tests/' + dest
obj_path = mozpath.join(self.environment.topobjdir, dest)
if isinstance(src, PreprocessedFile):
assert os.path.exists(obj_path), '%s should exist' % obj_path
pp_info = generate_pp_info(obj_path, self.environment.topsrcdir)
else:
pp_info = None
self._install_mapping[dest] = src.path, pp_info
# Our result has four parts:
# A map from url prefixes to objdir directories:
# { "chrome://mozapps/content/": [ "dist/bin/chrome/toolkit/content/mozapps" ], ... }
# A map of overrides.
# A map from objdir paths to sourcedir paths, and an object storing mapping information for preprocessed files:
# { "dist/bin/browser/chrome/browser/content/browser/aboutSessionRestore.js":
# [ "$topsrcdir/browser/components/sessionstore/content/aboutSessionRestore.js", {} ], ... }
# An object containing build configuration information.
outputfile = os.path.join(self.environment.topobjdir, 'chrome-map.json')
with self._write_file(outputfile) as fh:
chrome_mapping = self.manifest_handler.chrome_mapping
overrides = self.manifest_handler.overrides
json.dump([
{k: list(v) for k, v in chrome_mapping.iteritems()},
overrides,
self._install_mapping,
{
'topobjdir': mozpath.normpath(self.environment.topobjdir),
'MOZ_APP_NAME': self.environment.substs.get('MOZ_APP_NAME'),
'OMNIJAR_NAME': self.environment.substs.get('OMNIJAR_NAME'),
'MOZ_MACBUNDLE_NAME': self.environment.substs.get('MOZ_MACBUNDLE_NAME'),
}
], fh, sort_keys=True, indent=2)
def test_install_manifests_written(self):
env, objs = self._emit("stub0")
backend = RecursiveMakeBackend(env)
m = InstallManifest()
backend._install_manifests["testing"] = m
m.add_symlink(__file__, "self")
backend.consume(objs)
man_dir = os.path.join(env.topobjdir, "_build_manifests", "install")
self.assertTrue(os.path.isdir(man_dir))
expected = ["testing"]
for e in expected:
full = os.path.join(man_dir, e)
self.assertTrue(os.path.exists(full))
m2 = InstallManifest(path=full)
self.assertEqual(m, m2)
def process_manifest(destdir,
paths,
track=None,
remove_unaccounted=True,
remove_all_directory_symlinks=True,
remove_empty_directories=True,
defines={}):
if track:
if os.path.exists(track):
# We use the same format as install manifests for the tracking
# data.
manifest = InstallManifest(path=track)
remove_unaccounted = FileRegistry()
dummy_file = BaseFile()
finder = FileFinder(destdir,
find_executables=False,
find_dotfiles=True)
for dest in manifest._dests:
if '*' in dest:
for p, f in finder.find(dest):
remove_unaccounted.add(p, dummy_file)
else:
remove_unaccounted.add(dest, dummy_file)
else:
# If tracking is enabled and there is no file, we don't want to
# be removing anything.
remove_unaccounted = False
remove_empty_directories = False
remove_all_directory_symlinks = False
manifest = InstallManifest()
for path in paths:
manifest |= InstallManifest(path=path)
copier = FileCopier()
manifest.populate_registry(copier, defines_override=defines)
result = copier.copy(
destdir,
remove_unaccounted=remove_unaccounted,
remove_all_directory_symlinks=remove_all_directory_symlinks,
remove_empty_directories=remove_empty_directories)
if track:
manifest.write(path=track)
return result
def process_manifest(destdir, paths, track=None,
remove_unaccounted=True,
remove_all_directory_symlinks=True,
remove_empty_directories=True,
no_symlinks=False,
defines={}):
if track:
if os.path.exists(track):
# We use the same format as install manifests for the tracking
# data.
manifest = InstallManifest(path=track)
remove_unaccounted = FileRegistry()
dummy_file = BaseFile()
finder = FileFinder(destdir, find_dotfiles=True)
for dest in manifest._dests:
for p, f in finder.find(dest):
remove_unaccounted.add(p, dummy_file)
else:
# If tracking is enabled and there is no file, we don't want to
# be removing anything.
remove_unaccounted=False
remove_empty_directories=False
remove_all_directory_symlinks=False
manifest_cls = InstallManifestNoSymlinks if no_symlinks else InstallManifest
manifest = manifest_cls()
for path in paths:
manifest |= manifest_cls(path=path)
copier = FileCopier()
manifest.populate_registry(copier, defines_override=defines)
result = copier.copy(destdir,
remove_unaccounted=remove_unaccounted,
remove_all_directory_symlinks=remove_all_directory_symlinks,
remove_empty_directories=remove_empty_directories)
if track:
manifest.write(path=track)
return result
def test_install_manifests_written(self):
env, objs = self._emit('stub0')
backend = RecursiveMakeBackend(env)
m = InstallManifest()
backend._install_manifests['testing'] = m
m.add_symlink(__file__, 'self')
backend.consume(objs)
man_dir = mozpath.join(env.topobjdir, '_build_manifests', 'install')
self.assertTrue(os.path.isdir(man_dir))
expected = ['testing']
for e in expected:
full = mozpath.join(man_dir, e)
self.assertTrue(os.path.exists(full))
m2 = InstallManifest(path=full)
self.assertEqual(m, m2)
def test_install_manifests_written(self):
env, objs = self._emit('stub0')
backend = RecursiveMakeBackend(env)
m = InstallManifest()
backend._install_manifests['testing'] = m
m.add_symlink(__file__, 'self')
backend.consume(objs)
man_dir = mozpath.join(env.topobjdir, '_build_manifests', 'install')
self.assertTrue(os.path.isdir(man_dir))
expected = ['testing']
for e in expected:
full = mozpath.join(man_dir, e)
self.assertTrue(os.path.exists(full))
m2 = InstallManifest(path=full)
self.assertEqual(m, m2)
def install_test_files(topsrcdir, topobjdir, tests_root, test_objs):
"""Installs the requested test files to the objdir. This is invoked by
test runners to avoid installing tens of thousands of test files when
only a few tests need to be run.
"""
flavor_info = {flavor: (root, prefix, install)
for (flavor, root, prefix, install) in TEST_MANIFESTS.values()}
objdir_dest = mozpath.join(topobjdir, tests_root)
converter = SupportFilesConverter()
install_info = TestInstallInfo()
for o in test_objs:
flavor = o['flavor']
if flavor not in flavor_info:
# This is a test flavor that isn't installed by the build system.
continue
root, prefix, install = flavor_info[flavor]
if not install:
# This flavor isn't installed to the objdir.
continue
manifest_path = o['manifest']
manifest_dir = mozpath.dirname(manifest_path)
out_dir = mozpath.join(root, prefix, manifest_dir[len(topsrcdir) + 1:])
file_relpath = o['file_relpath']
source = mozpath.join(topsrcdir, file_relpath)
dest = mozpath.join(root, prefix, file_relpath)
if 'install-to-subdir' in o:
out_dir = mozpath.join(out_dir, o['install-to-subdir'])
manifest_relpath = mozpath.relpath(source, mozpath.dirname(manifest_path))
dest = mozpath.join(out_dir, manifest_relpath)
install_info.installs.append((source, dest))
install_info |= converter.convert_support_files(o, root,
manifest_dir,
out_dir)
manifest = InstallManifest()
for source, dest in set(install_info.installs):
if dest in install_info.external_installs:
continue
manifest.add_symlink(source, dest)
for base, pattern, dest in install_info.pattern_installs:
manifest.add_pattern_symlink(base, pattern, dest)
_resolve_installs(install_info.deferred_installs, topobjdir, manifest)
# Harness files are treated as a monolith and installed each time we run tests.
# Fortunately there are not very many.
manifest |= InstallManifest(mozpath.join(topobjdir,
'_build_manifests',
'install', tests_root))
copier = FileCopier()
manifest.populate_registry(copier)
copier.copy(objdir_dest,
remove_unaccounted=False)
def test_install_manifests_written(self):
env, objs = self._emit("stub0")
backend = RecursiveMakeBackend(env)
m = InstallManifest()
backend._install_manifests["testing"] = m
m.add_link(__file__, "self")
backend.consume(objs)
man_dir = mozpath.join(env.topobjdir, "_build_manifests", "install")
self.assertTrue(os.path.isdir(man_dir))
expected = ["testing"]
for e in expected:
full = mozpath.join(man_dir, e)
self.assertTrue(os.path.exists(full))
m2 = InstallManifest(path=full)
self.assertEqual(m, m2)
def test_process_manifest(self):
source = self.tmppath('source')
os.mkdir(source)
os.mkdir('%s/base' % source)
os.mkdir('%s/base/foo' % source)
os.mkdir('%s/base2' % source)
with open('%s/base/foo/file1' % source, 'a'):
pass
with open('%s/base/foo/file2' % source, 'a'):
pass
with open('%s/base2/file3' % source, 'a'):
pass
m = InstallManifest()
m.add_pattern_link('%s/base' % source, '**', '')
m.add_link('%s/base2/file3' % source, 'foo/file3')
p = self.tmppath('m')
m.write(path=p)
dest = self.tmppath('dest')
track = self.tmppath('track')
for i in range(2):
process_install_manifest.process_manifest(dest, [p], track)
self.assertTrue(os.path.exists(self.tmppath('dest/foo/file1')))
self.assertTrue(os.path.exists(self.tmppath('dest/foo/file2')))
self.assertTrue(os.path.exists(self.tmppath('dest/foo/file3')))
m = InstallManifest()
m.write(path=p)
for i in range(2):
process_install_manifest.process_manifest(dest, [p], track)
self.assertFalse(os.path.exists(self.tmppath('dest/foo/file1')))
self.assertFalse(os.path.exists(self.tmppath('dest/foo/file2')))
self.assertFalse(os.path.exists(self.tmppath('dest/foo/file3')))
def test_branding_files(self):
"""Ensure BRANDING_FILES is handled properly."""
env = self._consume('branding-files', RecursiveMakeBackend)
#BRANDING_FILES should appear in the dist_branding install manifest.
m = InstallManifest(path=os.path.join(
env.topobjdir, '_build_manifests', 'install', 'dist_branding'))
self.assertEqual(len(m), 3)
self.assertIn('bar.ico', m)
self.assertIn('quux.png', m)
self.assertIn('icons/foo.ico', m)
def test_exports(self):
"""Ensure EXPORTS is handled properly."""
env = self._consume('exports', RecursiveMakeBackend)
# EXPORTS files should appear in the dist_include install manifest.
m = InstallManifest(path=mozpath.join(
env.topobjdir, '_build_manifests', 'install', 'dist_include'))
self.assertEqual(len(m), 7)
self.assertIn('foo.h', m)
self.assertIn('mozilla/mozilla1.h', m)
self.assertIn('mozilla/dom/dom2.h', m)
def test_test_manifest_pattern_matches_recorded(self):
"""Pattern matches in test manifests' support-files should be recorded."""
env = self._consume('test-manifests-written', RecursiveMakeBackend)
m = InstallManifest(path=mozpath.join(
env.topobjdir, '_build_manifests', 'install', 'tests'))
# This is not the most robust test in the world, but it gets the job
# done.
entries = [e for e in m._dests.keys() if '**' in e]
self.assertEqual(len(entries), 1)
self.assertIn('support/**', entries[0])
def process_manifest(destdir, paths, track,
no_symlinks=False,
defines={}):
if os.path.exists(track):
# We use the same format as install manifests for the tracking
# data.
manifest = InstallManifest(path=track)
remove_unaccounted = FileRegistry()
dummy_file = BaseFile()
finder = FileFinder(destdir, find_dotfiles=True)
for dest in manifest._dests:
for p, f in finder.find(dest):
remove_unaccounted.add(p, dummy_file)
remove_empty_directories = True
remove_all_directory_symlinks = True
else:
# If tracking is enabled and there is no file, we don't want to
# be removing anything.
remove_unaccounted = False
remove_empty_directories = False
remove_all_directory_symlinks = False
manifest = InstallManifest()
for path in paths:
manifest |= InstallManifest(path=path)
copier = FileCopier()
link_policy = "copy" if no_symlinks else "symlink"
manifest.populate_registry(
copier, defines_override=defines, link_policy=link_policy
)
result = copier.copy(destdir,
remove_unaccounted=remove_unaccounted,
remove_all_directory_symlinks=remove_all_directory_symlinks,
remove_empty_directories=remove_empty_directories)
if track:
# We should record files that we actually copied.
# It is too late to expand wildcards when the track file is read.
manifest.write(path=track, expand_pattern=True)
return result
def install_test_files(topsrcdir, topobjdir, tests_root):
"""Installs the requested test files to the objdir. This is invoked by
test runners to avoid installing tens of thousands of test files when
only a few tests need to be run.
"""
manifest = InstallManifest(
mozpath.join(topobjdir, '_build_manifests', 'install', '_test_files'))
harness_files_manifest = mozpath.join(topobjdir, '_build_manifests',
'install', tests_root)
if os.path.isfile(harness_files_manifest):
# If the backend has generated an install manifest for test harness
# files they are treated as a monolith and installed each time we
# run tests. Fortunately there are not very many.
manifest |= InstallManifest(harness_files_manifest)
copier = FileCopier()
manifest.populate_registry(copier)
copier.copy(mozpath.join(topobjdir, tests_root), remove_unaccounted=False)
def test_xpidl_generation(self):
"""Ensure xpidl files and directories are written out."""
env = self._consume("xpidl", RecursiveMakeBackend)
# Install manifests should contain entries.
install_dir = mozpath.join(env.topobjdir, "_build_manifests",
"install")
self.assertTrue(os.path.isfile(mozpath.join(install_dir, "xpidl")))
m = InstallManifest(path=mozpath.join(install_dir, "xpidl"))
self.assertIn(".deps/my_module.pp", m)
m = InstallManifest(path=mozpath.join(install_dir, "xpidl"))
self.assertIn("my_module.xpt", m)
m = InstallManifest(path=mozpath.join(install_dir, "dist_include"))
self.assertIn("foo.h", m)
p = mozpath.join(env.topobjdir, "config/makefiles/xpidl")
self.assertTrue(os.path.isdir(p))
self.assertTrue(os.path.isfile(mozpath.join(p, "Makefile")))
def test_xpidl_generation(self):
"""Ensure xpidl files and directories are written out."""
env = self._consume('xpidl', RecursiveMakeBackend)
# Install manifests should contain entries.
install_dir = mozpath.join(env.topobjdir, '_build_manifests',
'install')
self.assertTrue(os.path.isfile(mozpath.join(install_dir, 'xpidl')))
m = InstallManifest(path=mozpath.join(install_dir, 'xpidl'))
self.assertIn('.deps/my_module.pp', m)
m = InstallManifest(path=mozpath.join(install_dir, 'xpidl'))
self.assertIn('my_module.xpt', m)
m = InstallManifest(path=mozpath.join(install_dir, 'dist_include'))
self.assertIn('foo.h', m)
p = mozpath.join(env.topobjdir, 'config/makefiles/xpidl')
self.assertTrue(os.path.isdir(p))
self.assertTrue(os.path.isfile(mozpath.join(p, 'Makefile')))
def test_test_manifest_deffered_installs_written(self):
"""Shared support files are written to their own data file by the backend."""
env = self._consume('test-manifest-shared-support', RecursiveMakeBackend)
all_tests_path = mozpath.join(env.topobjdir, 'all-tests.json')
self.assertTrue(os.path.exists(all_tests_path))
test_installs_path = mozpath.join(env.topobjdir, 'test-installs.json')
with open(test_installs_path, 'r') as fh:
test_installs = json.load(fh)
self.assertEqual(set(test_installs.keys()),
set(['child/test_sub.js',
'child/data/**',
'child/another-file.sjs']))
for key in test_installs.keys():
self.assertIn(key, test_installs)
test_files_manifest = mozpath.join(env.topobjdir,
'_build_manifests',
'install',
'_test_files')
# First, read the generated for ini manifest contents.
m = InstallManifest(path=test_files_manifest)
# Then, synthesize one from the test-installs.json file. This should
# allow us to re-create a subset of the above.
synthesized_manifest = InstallManifest()
for item, installs in test_installs.items():
for install_info in installs:
if len(install_info) == 3:
synthesized_manifest.add_pattern_symlink(*install_info)
if len(install_info) == 2:
synthesized_manifest.add_symlink(*install_info)
self.assertEqual(len(synthesized_manifest), 3)
for item, info in synthesized_manifest._dests.items():
self.assertIn(item, m)
self.assertEqual(info, m._dests[item])
def test_exports_generated(self):
"""Ensure EXPORTS that are listed in GENERATED_FILES
are handled properly."""
env = self._consume('exports-generated', RecursiveMakeBackend)
# EXPORTS files should appear in the dist_include install manifest.
m = InstallManifest(path=mozpath.join(
env.topobjdir, '_build_manifests', 'install', 'dist_include'))
self.assertEqual(len(m), 8)
self.assertIn('foo.h', m)
self.assertIn('mozilla/mozilla1.h', m)
self.assertIn('mozilla/dom/dom1.h', m)
self.assertIn('gfx/gfx.h', m)
self.assertIn('bar.h', m)
self.assertIn('mozilla/mozilla2.h', m)
self.assertIn('mozilla/dom/dom2.h', m)
self.assertIn('mozilla/dom/dom3.h', m)
# EXPORTS files that are also GENERATED_FILES should be handled as
# INSTALL_TARGETS.
backend_path = mozpath.join(env.topobjdir, 'backend.mk')
lines = [l.strip() for l in open(backend_path, 'rt').readlines()[2:]]
expected = [
'export:: bar.h',
'GARBAGE += bar.h',
'EXTRA_MDDEPEND_FILES += bar.h.pp',
'export:: mozilla2.h',
'GARBAGE += mozilla2.h',
'EXTRA_MDDEPEND_FILES += mozilla2.h.pp',
'export:: dom2.h',
'GARBAGE += dom2.h',
'EXTRA_MDDEPEND_FILES += dom2.h.pp',
'export:: dom3.h',
'GARBAGE += dom3.h',
'EXTRA_MDDEPEND_FILES += dom3.h.pp',
'dist_include_FILES += bar.h',
'dist_include_DEST := $(DEPTH)/dist/include/',
'dist_include_TARGET := export',
'INSTALL_TARGETS += dist_include',
'dist_include_mozilla_FILES += mozilla2.h',
'dist_include_mozilla_DEST := $(DEPTH)/dist/include/mozilla',
'dist_include_mozilla_TARGET := export',
'INSTALL_TARGETS += dist_include_mozilla',
'dist_include_mozilla_dom_FILES += dom2.h',
'dist_include_mozilla_dom_FILES += dom3.h',
'dist_include_mozilla_dom_DEST := $(DEPTH)/dist/include/mozilla/dom',
'dist_include_mozilla_dom_TARGET := export',
'INSTALL_TARGETS += dist_include_mozilla_dom',
]
self.maxDiff = None
self.assertEqual(lines, expected)
def setUp(self):
HelperMixin.setUp(self)
self.srcdir = os.path.join(self.test_dir, 'src')
os.mkdir(self.srcdir)
self.objdir = os.path.join(self.test_dir, 'obj')
os.mkdir(self.objdir)
self.manifest = InstallManifest()
self.canonical_mapping = {}
for s in ['src1', 'src2']:
srcfile = os.path.join(self.srcdir, s)
objfile = os.path.join(self.objdir, s)
self.canonical_mapping[objfile] = srcfile
self.manifest.add_copy(srcfile, s)
self.manifest_file = os.path.join(self.test_dir, 'install-manifest')
self.manifest.write(self.manifest_file)
def _synchronize_docs(self):
m = InstallManifest()
m.add_link(self.conf_py_path, 'conf.py')
for dest, source in sorted(self.trees.items()):
source_dir = os.path.join(self.topsrcdir, source)
for root, dirs, files in os.walk(source_dir):
for f in files:
source_path = os.path.join(root, f)
rel_source = source_path[len(source_dir) + 1:]
m.add_link(source_path, os.path.join(dest, rel_source))
copier = FileCopier()
m.populate_registry(copier)
copier.copy(self.staging_dir)
with open(self.index_path, 'rb') as fh:
data = fh.read()
def is_toplevel(key):
"""Whether the tree is nested under the toplevel index, or is
nested under another tree's index.
"""
for k in self.trees:
if k == key:
continue
if key.startswith(k):
return False
return True
toplevel_trees = {
k: v
for k, v in self.trees.items() if is_toplevel(k)
}
indexes = ['%s/index' % p for p in sorted(toplevel_trees.keys())]
indexes = '\n '.join(indexes)
packages = [os.path.basename(p) for p in self.python_package_dirs]
packages = ['python/%s' % p for p in packages]
packages = '\n '.join(sorted(packages))
data = data.format(indexes=indexes, python_packages=packages)
with open(os.path.join(self.staging_dir, 'index.rst'), 'wb') as fh:
fh.write(data)
def setUp(self):
HelperMixin.setUp(self)
self.srcdir = os.path.join(self.test_dir, "src")
os.mkdir(self.srcdir)
self.objdir = os.path.join(self.test_dir, "obj")
os.mkdir(self.objdir)
self.manifest = InstallManifest()
self.canonical_mapping = {}
for s in ["src1", "src2"]:
srcfile = os.path.join(self.srcdir, s)
objfile = os.path.join(self.objdir, s)
self.canonical_mapping[objfile] = srcfile
self.manifest.add_copy(srcfile, s)
self.manifest_file = os.path.join(self.test_dir, "install-manifest")
self.manifest.write(self.manifest_file)
def test_resources(self):
"""Ensure RESOURCE_FILES is handled properly."""
env = self._consume('resources', RecursiveMakeBackend)
# RESOURCE_FILES should appear in the dist_bin install manifest.
m = InstallManifest(path=os.path.join(
env.topobjdir, '_build_manifests', 'install', 'dist_bin'))
self.assertEqual(len(m), 10)
self.assertIn('res/foo.res', m)
self.assertIn('res/fonts/font1.ttf', m)
self.assertIn('res/fonts/desktop/desktop2.ttf', m)
self.assertIn('res/bar.res', m)
self.assertIn('res/tests/test.manifest', m)
self.assertIn('res/tests/extra.manifest', m)
def test_or(self):
m1 = self._get_test_manifest()
m2 = InstallManifest()
m2.add_symlink('s_source2', 's_dest2')
m2.add_copy('c_source2', 'c_dest2')
m1 |= m2
self.assertEqual(len(m2), 2)
self.assertEqual(len(m1), 6)
self.assertIn('s_dest2', m1)
self.assertIn('c_dest2', m1)
def test_write_expand_pattern(self):
source = self.tmppath('source')
os.mkdir(source)
os.mkdir('%s/base' % source)
os.mkdir('%s/base/foo' % source)
with open('%s/base/foo/file1' % source, 'a'):
pass
with open('%s/base/foo/file2' % source, 'a'):
pass
m = InstallManifest()
m.add_pattern_link('%s/base' % source, '**', 'dest')
track = self.tmppath('track')
m.write(path=track, expand_pattern=True)
m = InstallManifest(path=track)
self.assertEqual([dest for dest in m._dests],
['dest/foo/file1', 'dest/foo/file2'])
def test_write_expand_pattern(self):
source = self.tmppath("source")
os.mkdir(source)
os.mkdir("%s/base" % source)
os.mkdir("%s/base/foo" % source)
with open("%s/base/foo/file1" % source, "a"):
pass
with open("%s/base/foo/file2" % source, "a"):
pass
m = InstallManifest()
m.add_pattern_link("%s/base" % source, "**", "dest")
track = self.tmppath("track")
m.write(path=track, expand_pattern=True)
m = InstallManifest(path=track)
self.assertEqual(sorted(dest for dest in m._dests),
["dest/foo/file1", "dest/foo/file2"])
def test_or(self):
m1 = self._get_test_manifest()
orig_length = len(m1)
m2 = InstallManifest()
m2.add_link("s_source2", "s_dest2")
m2.add_copy("c_source2", "c_dest2")
m1 |= m2
self.assertEqual(len(m2), 2)
self.assertEqual(len(m1), orig_length + 2)
self.assertIn("s_dest2", m1)
self.assertIn("c_dest2", m1)
def _make_install_manifest(topsrcdir, topobjdir, test_objs):
flavor_info = {
flavor: (root, prefix, install)
for (flavor, root, prefix, install) in TEST_MANIFESTS.values()
}
converter = SupportFilesConverter()
install_info = TestInstallInfo()
for o in test_objs:
flavor = o['flavor']
if flavor not in flavor_info:
# This is a test flavor that isn't installed by the build system.
continue
root, prefix, install = flavor_info[flavor]
if not install:
# This flavor isn't installed to the objdir.
continue
manifest_path = o['manifest']
manifest_dir = mozpath.dirname(manifest_path)
out_dir = mozpath.join(root, prefix, manifest_dir[len(topsrcdir) + 1:])
file_relpath = o['file_relpath']
source = mozpath.join(topsrcdir, file_relpath)
dest = mozpath.join(root, prefix, file_relpath)
if 'install-to-subdir' in o:
out_dir = mozpath.join(out_dir, o['install-to-subdir'])
manifest_relpath = mozpath.relpath(source,
mozpath.dirname(manifest_path))
dest = mozpath.join(out_dir, manifest_relpath)
install_info.installs.append((source, dest))
install_info |= converter.convert_support_files(
o, root, manifest_dir, out_dir)
manifest = InstallManifest()
for source, dest in set(install_info.installs):
if dest in install_info.external_installs:
continue
manifest.add_link(source, dest)
for base, pattern, dest in install_info.pattern_installs:
manifest.add_pattern_link(base, pattern, dest)
_resolve_installs(install_info.deferred_installs, topobjdir, manifest)
return manifest
def test_install_manifests_package_tests(self):
"""Ensure test suites honor package_tests=False."""
env = self._consume("test-manifests-package-tests",
RecursiveMakeBackend)
man_dir = mozpath.join(env.topobjdir, "_build_manifests", "install")
self.assertTrue(os.path.isdir(man_dir))
full = mozpath.join(man_dir, "_test_files")
self.assertTrue(os.path.exists(full))
m = InstallManifest(path=full)
# Only mochitest.js should be in the install manifest.
self.assertTrue("testing/mochitest/tests/mochitest.js" in m)
# The path is odd here because we do not normalize at test manifest
# processing time. This is a fragile test because there's currently no
# way to iterate the manifest.
self.assertFalse("instrumentation/./not_packaged.java" in m)
def test_write_expand_pattern(self):
source = self.tmppath('source')
os.mkdir(source)
os.mkdir('%s/base' % source)
os.mkdir('%s/base/foo' % source)
with open('%s/base/foo/file1' % source, 'a'):
pass
with open('%s/base/foo/file2' % source, 'a'):
pass
m = InstallManifest()
m.add_pattern_link('%s/base' % source, '**', 'dest')
track = self.tmppath('track')
m.write(path=track, expand_pattern=True)
m = InstallManifest(path=track)
self.assertEqual([dest for dest in m._dests],
['dest/foo/file1', 'dest/foo/file2'])
def test_adds(self):
m = InstallManifest()
m.add_symlink('s_source', 's_dest')
m.add_copy('c_source', 'c_dest')
m.add_required_exists('e_dest')
m.add_optional_exists('o_dest')
self.assertEqual(len(m), 4)
self.assertIn('s_dest', m)
self.assertIn('c_dest', m)
self.assertIn('e_dest', m)
self.assertIn('o_dest', m)
with self.assertRaises(ValueError):
m.add_symlink('s_other', 's_dest')
with self.assertRaises(ValueError):
m.add_copy('c_other', 'c_dest')
with self.assertRaises(ValueError):
m.add_required_exists('e_dest')
with self.assertRaises(ValueError):
m.add_optional_exists('o_dest')
