def test_process_manifest(self):
source = self.tmppath('source')
os.mkdir(source)
os.mkdir('%s/base' % source)
os.mkdir('%s/base/foo' % source)
os.mkdir('%s/base2' % source)
with open('%s/base/foo/file1' % source, 'a'):
pass
with open('%s/base/foo/file2' % source, 'a'):
pass
with open('%s/base2/file3' % source, 'a'):
pass
m = InstallManifest()
m.add_pattern_link('%s/base' % source, '**', '')
m.add_link('%s/base2/file3' % source, 'foo/file3')
p = self.tmppath('m')
m.write(path=p)
dest = self.tmppath('dest')
track = self.tmppath('track')
for i in range(2):
process_install_manifest.process_manifest(dest, [p], track)
self.assertTrue(os.path.exists(self.tmppath('dest/foo/file1')))
self.assertTrue(os.path.exists(self.tmppath('dest/foo/file2')))
self.assertTrue(os.path.exists(self.tmppath('dest/foo/file3')))
def install_test_files(topsrcdir, topobjdir, tests_root, test_objs):
"""Installs the requested test files to the objdir. This is invoked by
test runners to avoid installing tens of thousands of test files when
only a few tests need to be run.
"""
flavor_info = {
flavor: (root, prefix, install)
for (flavor, root, prefix, install) in TEST_MANIFESTS.values()
}
objdir_dest = mozpath.join(topobjdir, tests_root)
converter = SupportFilesConverter()
install_info = TestInstallInfo()
for o in test_objs:
flavor = o['flavor']
if flavor not in flavor_info:
# This is a test flavor that isn't installed by the build system.
continue
root, prefix, install = flavor_info[flavor]
if not install:
# This flavor isn't installed to the objdir.
continue
manifest_path = o['manifest']
manifest_dir = mozpath.dirname(manifest_path)
out_dir = mozpath.join(root, prefix, manifest_dir[len(topsrcdir) + 1:])
file_relpath = o['file_relpath']
source = mozpath.join(topsrcdir, file_relpath)
dest = mozpath.join(root, prefix, file_relpath)
if 'install-to-subdir' in o:
out_dir = mozpath.join(out_dir, o['install-to-subdir'])
manifest_relpath = mozpath.relpath(source,
mozpath.dirname(manifest_path))
dest = mozpath.join(out_dir, manifest_relpath)
install_info.installs.append((source, dest))
install_info |= converter.convert_support_files(
o, root, manifest_dir, out_dir)
manifest = InstallManifest()
for source, dest in set(install_info.installs):
if dest in install_info.external_installs:
continue
manifest.add_link(source, dest)
for base, pattern, dest in install_info.pattern_installs:
manifest.add_pattern_link(base, pattern, dest)
_resolve_installs(install_info.deferred_installs, topobjdir, manifest)
# Harness files are treated as a monolith and installed each time we run tests.
# Fortunately there are not very many.
manifest |= InstallManifest(
mozpath.join(topobjdir, '_build_manifests', 'install', tests_root))
copier = FileCopier()
manifest.populate_registry(copier)
copier.copy(objdir_dest, remove_unaccounted=False)
def test_add_entries_from(self):
source = self.tmppath("source")
os.mkdir(source)
os.mkdir("%s/base" % source)
os.mkdir("%s/base/foo" % source)
with open("%s/base/foo/file1" % source, "a"):
pass
with open("%s/base/foo/file2" % source, "a"):
pass
m = InstallManifest()
m.add_pattern_link("%s/base" % source, "**", "dest")
p = InstallManifest()
p.add_entries_from(m)
self.assertEqual(len(p), 1)
c = FileCopier()
p.populate_registry(c)
self.assertEqual(c.paths(), ["dest/foo/file1", "dest/foo/file2"])
q = InstallManifest()
q.add_entries_from(m, base="target")
self.assertEqual(len(q), 1)
d = FileCopier()
q.populate_registry(d)
self.assertEqual(d.paths(),
["target/dest/foo/file1", "target/dest/foo/file2"])
# Some of the values in an InstallManifest include destination
# information that is present in the keys. Verify that we can
# round-trip serialization.
r = InstallManifest()
r.add_entries_from(m)
r.add_entries_from(m, base="target")
self.assertEqual(len(r), 2)
temp_path = self.tmppath("temp_path")
r.write(path=temp_path)
s = InstallManifest(path=temp_path)
e = FileCopier()
s.populate_registry(e)
self.assertEqual(
e.paths(),
[
"dest/foo/file1",
"dest/foo/file2",
"target/dest/foo/file1",
"target/dest/foo/file2",
],
)
def _get_test_manifest(self):
m = InstallManifest()
m.add_link(self.tmppath('s_source'), 's_dest')
m.add_copy(self.tmppath('c_source'), 'c_dest')
m.add_preprocess(self.tmppath('p_source'), 'p_dest', self.tmppath('p_source.pp'), '#', {'FOO':'BAR', 'BAZ':'QUX'})
m.add_required_exists('e_dest')
m.add_optional_exists('o_dest')
m.add_pattern_link('ps_base', '*', 'ps_dest')
m.add_pattern_copy('pc_base', '**', 'pc_dest')
m.add_content('the content\non\nmultiple lines', 'content')
return m
def _get_test_manifest(self):
m = InstallManifest()
m.add_link(self.tmppath('s_source'), 's_dest')
m.add_copy(self.tmppath('c_source'), 'c_dest')
m.add_preprocess(self.tmppath('p_source'), 'p_dest', self.tmppath('p_source.pp'), '#', {'FOO':'BAR', 'BAZ':'QUX'})
m.add_required_exists('e_dest')
m.add_optional_exists('o_dest')
m.add_pattern_link('ps_base', '*', 'ps_dest')
m.add_pattern_copy('pc_base', '**', 'pc_dest')
m.add_content('the content\non\nmultiple lines', 'content')
return m
def test_add_entries_from(self):
source = self.tmppath('source')
os.mkdir(source)
os.mkdir('%s/base' % source)
os.mkdir('%s/base/foo' % source)
with open('%s/base/foo/file1' % source, 'a'):
pass
with open('%s/base/foo/file2' % source, 'a'):
pass
m = InstallManifest()
m.add_pattern_link('%s/base' % source, '**', 'dest')
p = InstallManifest()
p.add_entries_from(m)
self.assertEqual(len(p), 1)
c = FileCopier()
p.populate_registry(c)
self.assertEqual(c.paths(), ['dest/foo/file1', 'dest/foo/file2'])
q = InstallManifest()
q.add_entries_from(m, base='target')
self.assertEqual(len(q), 1)
d = FileCopier()
q.populate_registry(d)
self.assertEqual(d.paths(),
['target/dest/foo/file1', 'target/dest/foo/file2'])
# Some of the values in an InstallManifest include destination
# information that is present in the keys. Verify that we can
# round-trip serialization.
r = InstallManifest()
r.add_entries_from(m)
r.add_entries_from(m, base='target')
self.assertEqual(len(r), 2)
temp_path = self.tmppath('temp_path')
r.write(path=temp_path)
s = InstallManifest(path=temp_path)
e = FileCopier()
s.populate_registry(e)
self.assertEqual(e.paths(), [
'dest/foo/file1', 'dest/foo/file2', 'target/dest/foo/file1',
'target/dest/foo/file2'
])
def test_add_entries_from(self):
source = self.tmppath('source')
os.mkdir(source)
os.mkdir('%s/base' % source)
os.mkdir('%s/base/foo' % source)
with open('%s/base/foo/file1' % source, 'a'):
pass
with open('%s/base/foo/file2' % source, 'a'):
pass
m = InstallManifest()
m.add_pattern_link('%s/base' % source, '**', 'dest')
p = InstallManifest()
p.add_entries_from(m)
self.assertEqual(len(p), 1)
c = FileCopier()
p.populate_registry(c)
self.assertEqual(c.paths(), ['dest/foo/file1', 'dest/foo/file2'])
q = InstallManifest()
q.add_entries_from(m, base='target')
self.assertEqual(len(q), 1)
d = FileCopier()
q.populate_registry(d)
self.assertEqual(d.paths(), ['target/dest/foo/file1', 'target/dest/foo/file2'])
# Some of the values in an InstallManifest include destination
# information that is present in the keys. Verify that we can
# round-trip serialization.
r = InstallManifest()
r.add_entries_from(m)
r.add_entries_from(m, base='target')
self.assertEqual(len(r), 2)
temp_path = self.tmppath('temp_path')
r.write(path=temp_path)
s = InstallManifest(path=temp_path)
e = FileCopier()
s.populate_registry(e)
self.assertEqual(e.paths(),
['dest/foo/file1', 'dest/foo/file2',
'target/dest/foo/file1', 'target/dest/foo/file2'])
def _make_install_manifest(topsrcdir, topobjdir, test_objs):
flavor_info = {
flavor: (root, prefix, install)
for (flavor, root, prefix, install) in TEST_MANIFESTS.values()
}
converter = SupportFilesConverter()
install_info = TestInstallInfo()
for o in test_objs:
flavor = o['flavor']
if flavor not in flavor_info:
# This is a test flavor that isn't installed by the build system.
continue
root, prefix, install = flavor_info[flavor]
if not install:
# This flavor isn't installed to the objdir.
continue
manifest_path = o['manifest']
manifest_dir = mozpath.dirname(manifest_path)
out_dir = mozpath.join(root, prefix, manifest_dir[len(topsrcdir) + 1:])
file_relpath = o['file_relpath']
source = mozpath.join(topsrcdir, file_relpath)
dest = mozpath.join(root, prefix, file_relpath)
if 'install-to-subdir' in o:
out_dir = mozpath.join(out_dir, o['install-to-subdir'])
manifest_relpath = mozpath.relpath(source,
mozpath.dirname(manifest_path))
dest = mozpath.join(out_dir, manifest_relpath)
install_info.installs.append((source, dest))
install_info |= converter.convert_support_files(
o, root, manifest_dir, out_dir)
manifest = InstallManifest()
for source, dest in set(install_info.installs):
if dest in install_info.external_installs:
continue
manifest.add_link(source, dest)
for base, pattern, dest in install_info.pattern_installs:
manifest.add_pattern_link(base, pattern, dest)
_resolve_installs(install_info.deferred_installs, topobjdir, manifest)
return manifest
def test_pattern_expansion(self):
source = self.tmppath("source")
os.mkdir(source)
os.mkdir("%s/base" % source)
os.mkdir("%s/base/foo" % source)
with open("%s/base/foo/file1" % source, "a"):
pass
with open("%s/base/foo/file2" % source, "a"):
pass
m = InstallManifest()
m.add_pattern_link("%s/base" % source, "**", "dest")
c = FileCopier()
m.populate_registry(c)
self.assertEqual(c.paths(), ["dest/foo/file1", "dest/foo/file2"])
def test_pattern_expansion(self):
source = self.tmppath('source')
os.mkdir(source)
os.mkdir('%s/base' % source)
os.mkdir('%s/base/foo' % source)
with open('%s/base/foo/file1' % source, 'a'):
pass
with open('%s/base/foo/file2' % source, 'a'):
pass
m = InstallManifest()
m.add_pattern_link('%s/base' % source, '**', 'dest')
c = FileCopier()
m.populate_registry(c)
self.assertEqual(c.paths(), ['dest/foo/file1', 'dest/foo/file2'])
def test_pattern_expansion(self):
source = self.tmppath('source')
os.mkdir(source)
os.mkdir('%s/base' % source)
os.mkdir('%s/base/foo' % source)
with open('%s/base/foo/file1' % source, 'a'):
pass
with open('%s/base/foo/file2' % source, 'a'):
pass
m = InstallManifest()
m.add_pattern_link('%s/base' % source, '**', 'dest')
c = FileCopier()
m.populate_registry(c)
self.assertEqual(c.paths(), ['dest/foo/file1', 'dest/foo/file2'])
def test_adds(self):
m = InstallManifest()
m.add_link('s_source', 's_dest')
m.add_copy('c_source', 'c_dest')
m.add_required_exists('e_dest')
m.add_optional_exists('o_dest')
m.add_pattern_link('ps_base', 'ps/*', 'ps_dest')
m.add_pattern_copy('pc_base', 'pc/**', 'pc_dest')
m.add_preprocess('p_source', 'p_dest', 'p_source.pp')
m.add_content('content', 'content')
self.assertEqual(len(m), 8)
self.assertIn('s_dest', m)
self.assertIn('c_dest', m)
self.assertIn('p_dest', m)
self.assertIn('e_dest', m)
self.assertIn('o_dest', m)
self.assertIn('content', m)
with self.assertRaises(ValueError):
m.add_link('s_other', 's_dest')
with self.assertRaises(ValueError):
m.add_copy('c_other', 'c_dest')
with self.assertRaises(ValueError):
m.add_preprocess('p_other', 'p_dest', 'p_other.pp')
with self.assertRaises(ValueError):
m.add_required_exists('e_dest')
with self.assertRaises(ValueError):
m.add_optional_exists('o_dest')
with self.assertRaises(ValueError):
m.add_pattern_link('ps_base', 'ps/*', 'ps_dest')
with self.assertRaises(ValueError):
m.add_pattern_copy('pc_base', 'pc/**', 'pc_dest')
with self.assertRaises(ValueError):
m.add_content('content', 'content')
def test_process_manifest(self):
source = self.tmppath('source')
os.mkdir(source)
os.mkdir('%s/base' % source)
os.mkdir('%s/base/foo' % source)
os.mkdir('%s/base2' % source)
with open('%s/base/foo/file1' % source, 'a'):
pass
with open('%s/base/foo/file2' % source, 'a'):
pass
with open('%s/base2/file3' % source, 'a'):
pass
m = InstallManifest()
m.add_pattern_link('%s/base' % source, '**', '')
m.add_link('%s/base2/file3' % source, 'foo/file3')
p = self.tmppath('m')
m.write(path=p)
dest = self.tmppath('dest')
track = self.tmppath('track')
for i in range(2):
process_install_manifest.process_manifest(dest, [p], track)
self.assertTrue(os.path.exists(self.tmppath('dest/foo/file1')))
self.assertTrue(os.path.exists(self.tmppath('dest/foo/file2')))
self.assertTrue(os.path.exists(self.tmppath('dest/foo/file3')))
m = InstallManifest()
m.write(path=p)
for i in range(2):
process_install_manifest.process_manifest(dest, [p], track)
self.assertFalse(os.path.exists(self.tmppath('dest/foo/file1')))
self.assertFalse(os.path.exists(self.tmppath('dest/foo/file2')))
self.assertFalse(os.path.exists(self.tmppath('dest/foo/file3')))
def test_process_manifest(self):
source = self.tmppath("source")
os.mkdir(source)
os.mkdir("%s/base" % source)
os.mkdir("%s/base/foo" % source)
os.mkdir("%s/base2" % source)
with open("%s/base/foo/file1" % source, "a"):
pass
with open("%s/base/foo/file2" % source, "a"):
pass
with open("%s/base2/file3" % source, "a"):
pass
m = InstallManifest()
m.add_pattern_link("%s/base" % source, "**", "")
m.add_link("%s/base2/file3" % source, "foo/file3")
p = self.tmppath("m")
m.write(path=p)
dest = self.tmppath("dest")
track = self.tmppath("track")
for i in range(2):
process_install_manifest.process_manifest(dest, [p], track)
self.assertTrue(os.path.exists(self.tmppath("dest/foo/file1")))
self.assertTrue(os.path.exists(self.tmppath("dest/foo/file2")))
self.assertTrue(os.path.exists(self.tmppath("dest/foo/file3")))
m = InstallManifest()
m.write(path=p)
for i in range(2):
process_install_manifest.process_manifest(dest, [p], track)
self.assertFalse(os.path.exists(self.tmppath("dest/foo/file1")))
self.assertFalse(os.path.exists(self.tmppath("dest/foo/file2")))
self.assertFalse(os.path.exists(self.tmppath("dest/foo/file3")))
def test_adds(self):
m = InstallManifest()
m.add_link('s_source', 's_dest')
m.add_copy('c_source', 'c_dest')
m.add_required_exists('e_dest')
m.add_optional_exists('o_dest')
m.add_pattern_link('ps_base', 'ps/*', 'ps_dest')
m.add_pattern_copy('pc_base', 'pc/**', 'pc_dest')
m.add_preprocess('p_source', 'p_dest', 'p_source.pp')
m.add_content('content', 'content')
self.assertEqual(len(m), 8)
self.assertIn('s_dest', m)
self.assertIn('c_dest', m)
self.assertIn('p_dest', m)
self.assertIn('e_dest', m)
self.assertIn('o_dest', m)
self.assertIn('content', m)
with self.assertRaises(ValueError):
m.add_link('s_other', 's_dest')
with self.assertRaises(ValueError):
m.add_copy('c_other', 'c_dest')
with self.assertRaises(ValueError):
m.add_preprocess('p_other', 'p_dest', 'p_other.pp')
with self.assertRaises(ValueError):
m.add_required_exists('e_dest')
with self.assertRaises(ValueError):
m.add_optional_exists('o_dest')
with self.assertRaises(ValueError):
m.add_pattern_link('ps_base', 'ps/*', 'ps_dest')
with self.assertRaises(ValueError):
m.add_pattern_copy('pc_base', 'pc/**', 'pc_dest')
with self.assertRaises(ValueError):
m.add_content('content', 'content')
def test_adds(self):
m = InstallManifest()
m.add_link("s_source", "s_dest")
m.add_copy("c_source", "c_dest")
m.add_required_exists("e_dest")
m.add_optional_exists("o_dest")
m.add_pattern_link("ps_base", "ps/*", "ps_dest")
m.add_pattern_copy("pc_base", "pc/**", "pc_dest")
m.add_preprocess("p_source", "p_dest", "p_source.pp")
m.add_content("content", "content")
self.assertEqual(len(m), 8)
self.assertIn("s_dest", m)
self.assertIn("c_dest", m)
self.assertIn("p_dest", m)
self.assertIn("e_dest", m)
self.assertIn("o_dest", m)
self.assertIn("content", m)
with self.assertRaises(ValueError):
m.add_link("s_other", "s_dest")
with self.assertRaises(ValueError):
m.add_copy("c_other", "c_dest")
with self.assertRaises(ValueError):
m.add_preprocess("p_other", "p_dest", "p_other.pp")
with self.assertRaises(ValueError):
m.add_required_exists("e_dest")
with self.assertRaises(ValueError):
m.add_optional_exists("o_dest")
with self.assertRaises(ValueError):
m.add_pattern_link("ps_base", "ps/*", "ps_dest")
with self.assertRaises(ValueError):
m.add_pattern_copy("pc_base", "pc/**", "pc_dest")
with self.assertRaises(ValueError):
m.add_content("content", "content")
def test_write_expand_pattern(self):
source = self.tmppath('source')
os.mkdir(source)
os.mkdir('%s/base' % source)
os.mkdir('%s/base/foo' % source)
with open('%s/base/foo/file1' % source, 'a'):
pass
with open('%s/base/foo/file2' % source, 'a'):
pass
m = InstallManifest()
m.add_pattern_link('%s/base' % source, '**', 'dest')
track = self.tmppath('track')
m.write(path=track, expand_pattern=True)
m = InstallManifest(path=track)
self.assertEqual([dest for dest in m._dests],
['dest/foo/file1', 'dest/foo/file2'])
def test_write_expand_pattern(self):
source = self.tmppath("source")
os.mkdir(source)
os.mkdir("%s/base" % source)
os.mkdir("%s/base/foo" % source)
with open("%s/base/foo/file1" % source, "a"):
pass
with open("%s/base/foo/file2" % source, "a"):
pass
m = InstallManifest()
m.add_pattern_link("%s/base" % source, "**", "dest")
track = self.tmppath("track")
m.write(path=track, expand_pattern=True)
m = InstallManifest(path=track)
self.assertEqual(sorted(dest for dest in m._dests),
["dest/foo/file1", "dest/foo/file2"])
def _get_test_manifest(self):
m = InstallManifest()
m.add_link(self.tmppath("s_source"), "s_dest")
m.add_copy(self.tmppath("c_source"), "c_dest")
m.add_preprocess(
self.tmppath("p_source"),
"p_dest",
self.tmppath("p_source.pp"),
"#",
{
"FOO": "BAR",
"BAZ": "QUX"
},
)
m.add_required_exists("e_dest")
m.add_optional_exists("o_dest")
m.add_pattern_link("ps_base", "*", "ps_dest")
m.add_pattern_copy("pc_base", "**", "pc_dest")
m.add_content("the content\non\nmultiple lines", "content")
return m
def test_write_expand_pattern(self):
source = self.tmppath('source')
os.mkdir(source)
os.mkdir('%s/base' % source)
os.mkdir('%s/base/foo' % source)
with open('%s/base/foo/file1' % source, 'a'):
pass
with open('%s/base/foo/file2' % source, 'a'):
pass
m = InstallManifest()
m.add_pattern_link('%s/base' % source, '**', 'dest')
track = self.tmppath('track')
m.write(path=track, expand_pattern=True)
m = InstallManifest(path=track)
self.assertEqual([dest for dest in m._dests],
['dest/foo/file1', 'dest/foo/file2'])
def test_test_manifest_deffered_installs_written(self):
"""Shared support files are written to their own data file by the backend."""
env = self._consume("test-manifest-shared-support",
RecursiveMakeBackend)
# First, read the generated for ini manifest contents.
test_files_manifest = mozpath.join(env.topobjdir, "_build_manifests",
"install", "_test_files")
m = InstallManifest(path=test_files_manifest)
# Then, synthesize one from the test-installs.pkl file. This should
# allow us to re-create a subset of the above.
env = self._consume("test-manifest-shared-support",
TestManifestBackend)
test_installs_path = mozpath.join(env.topobjdir, "test-installs.pkl")
with open(test_installs_path, "rb") as fh:
test_installs = pickle.load(fh)
self.assertEqual(
set(test_installs.keys()),
set([
"child/test_sub.js", "child/data/**", "child/another-file.sjs"
]),
)
for key in test_installs.keys():
self.assertIn(key, test_installs)
synthesized_manifest = InstallManifest()
for item, installs in test_installs.items():
for install_info in installs:
if len(install_info) == 3:
synthesized_manifest.add_pattern_link(*install_info)
if len(install_info) == 2:
synthesized_manifest.add_link(*install_info)
self.assertEqual(len(synthesized_manifest), 3)
for item, info in synthesized_manifest._dests.items():
self.assertIn(item, m)
self.assertEqual(info, m._dests[item])
def test_test_manifest_deffered_installs_written(self):
"""Shared support files are written to their own data file by the backend."""
env = self._consume('test-manifest-shared-support', RecursiveMakeBackend)
# First, read the generated for ini manifest contents.
test_files_manifest = mozpath.join(env.topobjdir,
'_build_manifests',
'install',
'_test_files')
m = InstallManifest(path=test_files_manifest)
# Then, synthesize one from the test-installs.pkl file. This should
# allow us to re-create a subset of the above.
env = self._consume('test-manifest-shared-support', TestManifestBackend)
test_installs_path = mozpath.join(env.topobjdir, 'test-installs.pkl')
with open(test_installs_path, 'r') as fh:
test_installs = pickle.load(fh)
self.assertEqual(set(test_installs.keys()),
set(['child/test_sub.js',
'child/data/**',
'child/another-file.sjs']))
for key in test_installs.keys():
self.assertIn(key, test_installs)
synthesized_manifest = InstallManifest()
for item, installs in test_installs.items():
for install_info in installs:
if len(install_info) == 3:
synthesized_manifest.add_pattern_link(*install_info)
if len(install_info) == 2:
synthesized_manifest.add_link(*install_info)
self.assertEqual(len(synthesized_manifest), 3)
for item, info in synthesized_manifest._dests.items():
self.assertIn(item, m)
self.assertEqual(info, m._dests[item])