def test_jar_formatter_with_base(self):
registry = FileRegistry()
formatter = JarFormatter(registry)
fill_formatter(formatter, CONTENTS_WITH_BASE)
self.assertEqual(get_contents(registry), RESULT_JAR_WITH_BASE)
self.do_test_contents(formatter, CONTENTS_WITH_BASE)
def test_omnijar_formatter_with_base(self):
registry = FileRegistry()
formatter = OmniJarFormatter(registry, 'omni.foo')
fill_formatter(formatter, CONTENTS_WITH_BASE)
self.assertEqual(get_contents(registry), RESULT_OMNIJAR_WITH_BASE)
self.do_test_contents(formatter, CONTENTS_WITH_BASE)
def test_flat_formatter(self):
registry = FileRegistry()
formatter = FlatFormatter(registry)
fill_formatter(formatter, CONTENTS)
self.assertEqual(get_contents(registry), RESULT_FLAT)
self.do_test_contents(formatter, CONTENTS)
def test_jar_formatter(self):
registry = FileRegistry()
formatter = JarFormatter(registry)
fill_formatter(formatter, CONTENTS)
self.assertEqual(get_contents(registry), RESULT_JAR)
self.do_test_contents(formatter, CONTENTS)
def test_required_directories(self):
self.registry = FileRegistry()
self.registry.add('foo', GeneratedFile('foo'))
self.assertEqual(self.registry.required_directories(), set())
self.registry.add('bar/baz', GeneratedFile('barbaz'))
self.assertEqual(self.registry.required_directories(), {'bar'})
self.registry.add('bar/zot', GeneratedFile('barzot'))
self.assertEqual(self.registry.required_directories(), {'bar'})
self.registry.add('bar/zap/zot', GeneratedFile('barzapzot'))
self.assertEqual(self.registry.required_directories(),
{'bar', 'bar/zap'})
self.registry.remove('bar/zap/zot')
self.assertEqual(self.registry.required_directories(), {'bar'})
self.registry.remove('bar/baz')
self.assertEqual(self.registry.required_directories(), {'bar'})
self.registry.remove('bar/zot')
self.assertEqual(self.registry.required_directories(), set())
self.registry.add('x/y/z', GeneratedFile('xyz'))
self.assertEqual(self.registry.required_directories(), {'x', 'x/y'})
def test_omnijar_formatter_with_subpath(self):
registry = FileRegistry()
formatter = OmniJarFormatter(registry, 'bar/omni.foo')
fill_formatter(formatter, CONTENTS)
self.assertEqual(get_contents(registry), RESULT_OMNIJAR_WITH_SUBPATH)
self.do_test_contents(formatter, CONTENTS)
def test_populate_registry(self):
m = self._get_test_manifest()
r = FileRegistry()
m.populate_registry(r)
self.assertEqual(len(r), 6)
self.assertEqual(r.paths(), ['c_dest', 'content', 'e_dest', 'o_dest',
'p_dest', 's_dest'])
def test_partial_paths(self):
cases = {
"foo/bar/baz/zot": ["foo/bar/baz", "foo/bar", "foo"],
"foo/bar": ["foo"],
"bar": [],
}
reg = FileRegistry()
for path, parts in six.iteritems(cases):
self.assertEqual(reg._partial_paths(path), parts)
def test_partial_paths(self):
cases = {
'foo/bar/baz/zot': ['foo/bar/baz', 'foo/bar', 'foo'],
'foo/bar': ['foo'],
'bar': [],
}
reg = FileRegistry()
for path, parts in cases.iteritems():
self.assertEqual(reg._partial_paths(path), parts)
def test_populate_registry(self):
m = self._get_test_manifest()
r = FileRegistry()
m.populate_registry(r)
self.assertEqual(len(r), 6)
self.assertEqual(
r.paths(),
["c_dest", "content", "e_dest", "o_dest", "p_dest", "s_dest"])
def test_flat_formatter(self):
registry = FileRegistry()
formatter = FlatFormatter(registry)
formatter.add_base('app')
formatter.add('f/oo/bar', GeneratedFile('foobar'))
formatter.add('f/oo/baz', GeneratedFile('foobaz'))
formatter.add('f/oo/qux', GeneratedFile('fooqux'))
formatter.add_manifest(ManifestContent('f/oo', 'bar', 'bar'))
formatter.add_manifest(ManifestContent('f/oo', 'qux', 'qux'))
self.assertEqual(registry.paths(), [
'f/oo/bar', 'f/oo/baz', 'f/oo/qux', 'chrome.manifest',
'f/f.manifest', 'f/oo/oo.manifest'
])
self.assertEqual(registry['chrome.manifest'].open().read(),
'manifest f/f.manifest\n')
self.assertEqual(registry['f/f.manifest'].open().read(),
'manifest oo/oo.manifest\n')
self.assertEqual(registry['f/oo/oo.manifest'].open().read(), ''.join([
'content bar bar\n',
'content qux qux\n',
]))
formatter.add_interfaces('components/foo.xpt', foo_xpt)
formatter.add_interfaces('components/bar.xpt', bar_xpt)
self.assertEqual(registry.paths(), [
'f/oo/bar', 'f/oo/baz', 'f/oo/qux', 'chrome.manifest',
'f/f.manifest', 'f/oo/oo.manifest',
'components/components.manifest', 'components/interfaces.xpt'
])
self.assertEqual(
registry['chrome.manifest'].open().read(), ''.join([
'manifest f/f.manifest\n',
'manifest components/components.manifest\n',
]))
self.assertEqual(
registry['components/components.manifest'].open().read(),
'interfaces interfaces.xpt\n')
registry['components/interfaces.xpt'] \
.copy(self.tmppath('interfaces.xpt'))
linked = read_interfaces(self.tmppath('interfaces.xpt'))
foo = read_interfaces(foo_xpt.open())
bar = read_interfaces(bar_xpt.open())
self.assertEqual(foo['foo'], linked['foo'])
self.assertEqual(bar['bar'], linked['bar'])
formatter.add_manifest(ManifestContent('app/chrome', 'content',
'foo/'))
self.assertEqual(
registry['chrome.manifest'].open().read(), ''.join([
'manifest f/f.manifest\n',
'manifest components/components.manifest\n',
]))
self.assertEqual(registry['app/chrome.manifest'].open().read(),
'manifest chrome/chrome.manifest\n')
self.assertEqual(registry['app/chrome/chrome.manifest'].open().read(),
'content content foo/\n')
def test_bases(self):
formatter = FlatFormatter(FileRegistry())
formatter.add_base('')
formatter.add_base('browser')
formatter.add_base('webapprt')
self.assertEqual(formatter._get_base('platform.ini'), '')
self.assertEqual(formatter._get_base('browser/application.ini'),
'browser')
self.assertEqual(formatter._get_base('webapprt/webapprt.ini'),
'webapprt')
def _unpack_test(self, cls):
# Format a package with the given formatter class
copier = self._get_copier(cls)
copier.copy(self.tmpdir)
# Unpack that package. Its content is expected to match that of a Flat
# formatted package.
registry = FileRegistry()
unpack_to_registry(self.tmpdir, registry)
self.assertEqual(get_contents(registry, read_all=True), self.contents)
def consume_finished(self):
mp = os.path.join(self.environment.topobjdir, '_build_manifests',
'install', '_tests')
install_manifest = InstallManifest(mp)
reg = FileRegistry()
install_manifest.populate_registry(reg)
for dest, src in reg:
if not hasattr(src, 'path'):
continue
if not os.path.isabs(dest):
dest = '_tests/' + dest
obj_path = mozpath.join(self.environment.topobjdir, dest)
if isinstance(src, PreprocessedFile):
assert os.path.exists(obj_path), '%s should exist' % obj_path
pp_info = generate_pp_info(obj_path,
self.environment.topsrcdir)
else:
pp_info = None
rel_src = mozpath.relpath(src.path, self.environment.topsrcdir)
self._install_mapping[dest] = rel_src, pp_info
# Our result has four parts:
# A map from url prefixes to objdir directories:
# { "chrome://mozapps/content/": [ "dist/bin/chrome/toolkit/content/mozapps" ], ... }
# A map of overrides.
# A map from objdir paths to sourcedir paths, and an object storing mapping
# information for preprocessed files:
# { "dist/bin/browser/chrome/browser/content/browser/aboutSessionRestore.js":
# [ "$topsrcdir/browser/components/sessionstore/content/aboutSessionRestore.js", {} ],
# ... }
# An object containing build configuration information.
outputfile = os.path.join(self.environment.topobjdir,
'chrome-map.json')
with self._write_file(outputfile) as fh:
chrome_mapping = self.manifest_handler.chrome_mapping
overrides = self.manifest_handler.overrides
json.dump([{k: list(v)
for k, v in chrome_mapping.iteritems()}, overrides,
self._install_mapping, {
'topobjdir':
mozpath.normpath(self.environment.topobjdir),
'MOZ_APP_NAME':
self.environment.substs.get('MOZ_APP_NAME'),
'OMNIJAR_NAME':
self.environment.substs.get('OMNIJAR_NAME'),
'MOZ_MACBUNDLE_NAME':
self.environment.substs.get('MOZ_MACBUNDLE_NAME'),
}],
fh,
sort_keys=True,
indent=2)
def __init__(self, source, omnijar_name=None, unpack_xpi=True):
if isinstance(source, BaseFinder):
self._finder = source
else:
self._finder = FileFinder(source)
self.base = self._finder.base
self.files = FileRegistry()
self.kind = "flat"
if omnijar_name:
self.omnijar = omnijar_name
else:
# Can't include globally because of bootstrapping issues.
from buildconfig import substs
self.omnijar = substs.get("OMNIJAR_NAME", "omni.ja")
self.jarlogs = {}
self.compressed = False
self._unpack_xpi = unpack_xpi
jars = set()
for p, f in self._finder.find("*"):
# Skip the precomplete file, which is generated at packaging time.
if p == "precomplete":
continue
base = mozpath.dirname(p)
# If the file matches the omnijar pattern, it is an omnijar.
# All the files it contains go in the directory containing the full
# pattern. Manifests are merged if there is a corresponding manifest
# in the directory.
if self._maybe_zip(f) and mozpath.match(p, "**/%s" % self.omnijar):
jar = self._open_jar(p, f)
if "chrome.manifest" in jar:
self.kind = "omni"
self._fill_with_jar(p[: -len(self.omnijar) - 1], jar)
continue
# If the file is a manifest, scan its entries for some referencing
# jar: urls. If there are some, the files contained in the jar they
# point to, go under a directory named after the jar.
if is_manifest(p):
m = self.files[p] if self.files.contains(p) else ManifestFile(base)
for e in parse_manifest(
self.base, p, codecs.getreader("utf-8")(f.open())
):
m.add(self._handle_manifest_entry(e, jars))
if self.files.contains(p):
continue
f = m
# If we're unpacking packed addons and the file is a packed addon,
# unpack it under a directory named after the xpi.
if self._unpack_xpi and p.endswith(".xpi") and self._maybe_zip(f):
self._fill_with_jar(p[:-4], self._open_jar(p, f))
continue
if p not in jars:
self.files.add(p, f)
def make_file_mapping(install_manifests):
file_mapping = {}
for manifest, destination in install_manifests:
destination = os.path.abspath(destination)
reg = FileRegistry()
manifest.populate_registry(reg)
for dst, src in reg:
if hasattr(src, 'path'):
abs_dest = os.path.normpath(os.path.join(destination, dst))
file_mapping[abs_dest] = src.path
return file_mapping
def test_bases(self):
formatter = FlatFormatter(FileRegistry())
formatter.add_base('')
formatter.add_base('browser')
formatter.add_base('addon0', addon=True)
self.assertEqual(formatter._get_base('platform.ini'),
('', 'platform.ini'))
self.assertEqual(formatter._get_base('browser/application.ini'),
('browser', 'application.ini'))
self.assertEqual(formatter._get_base('addon0/install.rdf'),
('addon0', 'install.rdf'))
def get_registry(paths):
used_paths = set()
registry = FileRegistry()
for base, path in paths:
full_path = mozpath.join(buildconfig.topobjdir, path)
if not os.path.exists(full_path):
continue
used_paths.add(full_path)
reg = FileRegistry()
InstallManifest(full_path).populate_registry(reg)
for p, f in reg:
path = mozpath.join(base, p)
if not registry.contains(path):
registry.add(path, f)
return registry, used_paths
def __init__(self, finders):
# Can't import globally, because of the dependency of mozpack.copier
# on this module.
from mozpack.copier import FileRegistry
self.files = FileRegistry()
for base, finder in sorted(finders.iteritems()):
if self.files.contains(base):
self.files.remove(base)
for p, f in finder.find(''):
self.files.add(mozpath.join(base, p), f)
def test_final_target_files_wildcard(self):
"""Ensure that wildcards in FINAL_TARGET_FILES work properly."""
env = self._consume("final-target-files-wildcard", FasterMakeBackend)
m = InstallManifest(
path=mozpath.join(env.topobjdir, "faster", "install_dist_bin"))
self.assertEqual(len(m), 1)
reg = FileRegistry()
m.populate_registry(reg)
expected = [("foo/bar.xyz", "bar.xyz"), ("foo/foo.xyz", "foo.xyz")]
actual = [(path, mozpath.relpath(f.path, env.topsrcdir))
for (path, f) in reg]
self.assertEqual(expected, actual)
def make_file_mapping(install_manifests):
file_mapping = {}
for manifest, destination in install_manifests:
destination = os.path.abspath(destination)
reg = FileRegistry()
manifest.populate_registry(reg)
for dst, src in reg:
if hasattr(src, 'path'):
# Any paths that get compared to source file names need to go through normpath.
abs_dest = normpath(os.path.join(destination, dst))
file_mapping[abs_dest] = normpath(src.path)
return file_mapping
def _load_manifest(self, path, root):
install_manifest = InstallManifest(path)
reg = FileRegistry()
install_manifest.populate_registry(reg)
for dest, src in reg:
if hasattr(src, 'path'):
if not os.path.isabs(dest):
dest = root + dest
self._install_mapping[dest] = (src.path,
isinstance(
src, PreprocessedFile))
def test_file_registry(self):
self.registry = FileRegistry()
self.registry.add('foo', GeneratedFile('foo'))
bar = GeneratedFile('bar')
self.registry.add('bar', bar)
self.assertEqual(self.registry.paths(), ['foo', 'bar'])
self.assertEqual(self.registry['bar'], bar)
self.assertRaises(ErrorMessage, self.registry.add, 'foo',
GeneratedFile('foo2'))
self.assertRaises(ErrorMessage, self.registry.remove, 'qux')
self.assertRaises(ErrorMessage, self.registry.add, 'foo/bar',
GeneratedFile('foobar'))
self.assertRaises(ErrorMessage, self.registry.add, 'foo/bar/baz',
GeneratedFile('foobar'))
self.assertEqual(self.registry.paths(), ['foo', 'bar'])
self.registry.remove('foo')
self.assertEqual(self.registry.paths(), ['bar'])
self.registry.remove('bar')
self.assertEqual(self.registry.paths(), [])
self.prepare_match_test()
self.do_match_test()
self.assertTrue(self.checked)
self.assertEqual(self.registry.paths(), [
'bar',
'foo/bar',
'foo/baz',
'foo/qux/1',
'foo/qux/bar',
'foo/qux/2/test',
'foo/qux/2/test2',
])
self.registry.remove('foo/qux')
self.assertEqual(self.registry.paths(), ['bar', 'foo/bar', 'foo/baz'])
self.registry.add('foo/qux', GeneratedFile('fooqux'))
self.assertEqual(self.registry.paths(),
['bar', 'foo/bar', 'foo/baz', 'foo/qux'])
self.registry.remove('foo/b*')
self.assertEqual(self.registry.paths(), ['bar', 'foo/qux'])
self.assertEqual([f for f, c in self.registry], ['bar', 'foo/qux'])
self.assertEqual(len(self.registry), 2)
self.add('foo/.foo')
self.assertTrue(self.registry.contains('foo/.foo'))
def test_bases(self):
formatter = FlatFormatter(FileRegistry())
formatter.add_base("")
formatter.add_base("addon0", addon=True)
formatter.add_base("browser")
self.assertEqual(formatter._get_base("platform.ini"),
("", "platform.ini"))
self.assertEqual(
formatter._get_base("browser/application.ini"),
("browser", "application.ini"),
)
self.assertEqual(formatter._get_base("addon0/install.rdf"),
("addon0", "install.rdf"))
def __init__(self, source):
if isinstance(source, BaseFinder):
self._finder = source
else:
self._finder = FileFinder(source)
self.base = self._finder.base
self.files = FileRegistry()
self.kind = 'flat'
self.omnijar = None
self.jarlogs = {}
self.optimizedjars = False
self.compressed = False
jars = set()
for p, f in self._finder.find('*'):
# Skip the precomplete file, which is generated at packaging time.
if p == 'precomplete':
continue
base = mozpath.dirname(p)
# If the file is a zip/jar that is not a .xpi, and contains a
# chrome.manifest, it is an omnijar. All the files it contains
# go in the directory containing the omnijar. Manifests are merged
# if there is a corresponding manifest in the directory.
if not p.endswith('.xpi') and self._maybe_zip(f) and \
(mozpath.basename(p) == self.omnijar or
not self.omnijar):
jar = self._open_jar(p, f)
if 'chrome.manifest' in jar:
self.kind = 'omni'
self.omnijar = mozpath.basename(p)
self._fill_with_jar(base, jar)
continue
# If the file is a manifest, scan its entries for some referencing
# jar: urls. If there are some, the files contained in the jar they
# point to, go under a directory named after the jar.
if is_manifest(p):
m = self.files[p] if self.files.contains(p) \
else ManifestFile(base)
for e in parse_manifest(self.base, p, f.open()):
m.add(self._handle_manifest_entry(e, jars))
if self.files.contains(p):
continue
f = m
# If the file is a packed addon, unpack it under a directory named
# after the xpi.
if p.endswith('.xpi') and self._maybe_zip(f):
self._fill_with_jar(p[:-4], self._open_jar(p, f))
continue
if not p in jars:
self.files.add(p, f)
def find_generated_harness_files():
# TEST_HARNESS_FILES end up in an install manifest at
# $topsrcdir/_build_manifests/install/_tests.
manifest = InstallManifest(mozpath.join(buildconfig.topobjdir,
'_build_manifests',
'install',
'_tests'))
registry = FileRegistry()
manifest.populate_registry(registry)
# Conveniently, the generated files we care about will already
# exist in the objdir, so we can identify relevant files if
# they're an `ExistingFile` instance.
return [mozpath.join('_tests', p) for p in registry.paths()
if isinstance(registry[p], ExistingFile)]
def process_manifest(destdir,
paths,
track=None,
remove_unaccounted=True,
remove_all_directory_symlinks=True,
remove_empty_directories=True,
defines={}):
if track:
if os.path.exists(track):
# We use the same format as install manifests for the tracking
# data.
manifest = InstallManifest(path=track)
remove_unaccounted = FileRegistry()
dummy_file = BaseFile()
finder = FileFinder(destdir,
find_executables=False,
find_dotfiles=True)
for dest in manifest._dests:
if '*' in dest:
for p, f in finder.find(dest):
remove_unaccounted.add(p, dummy_file)
else:
remove_unaccounted.add(dest, dummy_file)
else:
# If tracking is enabled and there is no file, we don't want to
# be removing anything.
remove_unaccounted = False
remove_empty_directories = False
remove_all_directory_symlinks = False
manifest = InstallManifest()
for path in paths:
manifest |= InstallManifest(path=path)
copier = FileCopier()
manifest.populate_registry(copier, defines_override=defines)
result = copier.copy(
destdir,
remove_unaccounted=remove_unaccounted,
remove_all_directory_symlinks=remove_all_directory_symlinks,
remove_empty_directories=remove_empty_directories)
if track:
manifest.write(path=track)
return result
def is_resource(base, path):
registry = FileRegistry()
f = OmniJarFormatter(registry, 'omni.foo', non_resources=[
'defaults/messenger/mailViews.dat',
'defaults/foo/*',
'*/dummy',
])
f.add_base('')
f.add_base('app')
f.add(mozpath.join(base, path), GeneratedFile(''))
if f.copier.contains(mozpath.join(base, path)):
return False
self.assertTrue(f.copier.contains(mozpath.join(base, 'omni.foo')))
self.assertTrue(f.copier[mozpath.join(base, 'omni.foo')]
.contains(path))
return True
def describe_install_manifest(manifest, dest_dir):
try:
manifest = InstallManifest(manifest)
except UnreadableInstallManifest:
raise IOError(errno.EINVAL, 'Error parsing manifest file', manifest)
reg = FileRegistry()
mapping = {}
manifest.populate_registry(reg)
for dest_file, src in reg:
if hasattr(src, 'path'):
dest_path = mozpath.join(dest_dir, dest_file)
relsrc_path = mozpath.relpath(src.path, buildconfig.topsrcdir)
mapping[dest_path] = relsrc_path
return mapping
def process_manifest(destdir, paths, track,
no_symlinks=False,
defines={}):
if os.path.exists(track):
# We use the same format as install manifests for the tracking
# data.
manifest = InstallManifest(path=track)
remove_unaccounted = FileRegistry()
dummy_file = BaseFile()
finder = FileFinder(destdir, find_dotfiles=True)
for dest in manifest._dests:
for p, f in finder.find(dest):
remove_unaccounted.add(p, dummy_file)
remove_empty_directories = True
remove_all_directory_symlinks = True
else:
# If tracking is enabled and there is no file, we don't want to
# be removing anything.
remove_unaccounted = False
remove_empty_directories = False
remove_all_directory_symlinks = False
manifest = InstallManifest()
for path in paths:
manifest |= InstallManifest(path=path)
copier = FileCopier()
link_policy = "copy" if no_symlinks else "symlink"
manifest.populate_registry(
copier, defines_override=defines, link_policy=link_policy
)
result = copier.copy(destdir,
remove_unaccounted=remove_unaccounted,
remove_all_directory_symlinks=remove_all_directory_symlinks,
remove_empty_directories=remove_empty_directories)
if track:
# We should record files that we actually copied.
# It is too late to expand wildcards when the track file is read.
manifest.write(path=track, expand_pattern=True)
return result
