def process_manifest(destdir,
paths,
remove_unaccounted=True,
remove_all_directory_symlinks=True,
remove_empty_directories=True):
manifest = InstallManifest()
for path in paths:
manifest |= InstallManifest(path=path)
copier = FileCopier()
manifest.populate_registry(copier)
return copier.copy(
destdir,
remove_unaccounted=remove_unaccounted,
remove_all_directory_symlinks=remove_all_directory_symlinks,
remove_empty_directories=remove_empty_directories)
def repack(source, l10n, extra_l10n={}, non_resources=[], non_chrome=set()):
'''
Replace localized data from the `source` directory with localized data
from `l10n` and `extra_l10n`.
The `source` argument points to a directory containing a packaged
application (in omnijar, jar or flat form).
The `l10n` argument points to a directory containing the main localized
data (usually in the form of a language pack addon) to use to replace
in the packaged application.
The `extra_l10n` argument contains a dict associating relative paths in
the source to separate directories containing localized data for them.
This can be used to point at different language pack addons for different
parts of the package application.
The `non_resources` argument gives a list of relative paths in the source
that should not be added in an omnijar in case the packaged application
is in that format.
The `non_chrome` argument gives a list of file/directory patterns for
localized files that are not listed in a chrome.manifest.
'''
app_finder = UnpackFinder(source)
l10n_finder = UnpackFinder(l10n)
if extra_l10n:
finders = {
'': l10n_finder,
}
for base, path in extra_l10n.iteritems():
finders[base] = UnpackFinder(path)
l10n_finder = ComposedFinder(finders)
copier = FileCopier()
if app_finder.kind == 'flat':
formatter = FlatFormatter(copier)
elif app_finder.kind == 'jar':
formatter = JarFormatter(copier,
optimize=app_finder.optimizedjars,
compress=app_finder.compressed)
elif app_finder.kind == 'omni':
formatter = OmniJarFormatter(copier,
app_finder.omnijar,
optimize=app_finder.optimizedjars,
compress=app_finder.compressed,
non_resources=non_resources)
with errors.accumulate():
_repack(app_finder, l10n_finder, copier, formatter, non_chrome)
copier.copy(source, skip_if_older=False)
generate_precomplete(source)
def _synchronize_docs(self):
m = InstallManifest()
m.add_link(self.conf_py_path, 'conf.py')
for dest, source in sorted(self.trees.items()):
source_dir = os.path.join(self.topsrcdir, source)
for root, dirs, files in os.walk(source_dir):
for f in files:
source_path = os.path.join(root, f)
rel_source = source_path[len(source_dir) + 1:]
m.add_link(source_path, os.path.join(dest, rel_source))
copier = FileCopier()
m.populate_registry(copier)
copier.copy(self.staging_dir)
with open(self.index_path, 'rb') as fh:
data = fh.read()
def is_toplevel(key):
"""Whether the tree is nested under the toplevel index, or is
nested under another tree's index.
"""
for k in self.trees:
if k == key:
continue
if key.startswith(k):
return False
return True
toplevel_trees = {
k: v
for k, v in self.trees.items() if is_toplevel(k)
}
indexes = ['%s/index' % p for p in sorted(toplevel_trees.keys())]
indexes = '\n '.join(indexes)
packages = [os.path.basename(p) for p in self.python_package_dirs]
packages = ['python/%s' % p for p in packages]
packages = '\n '.join(sorted(packages))
data = data.format(indexes=indexes, python_packages=packages)
with open(os.path.join(self.staging_dir, 'index.rst'), 'wb') as fh:
fh.write(data)
def repack(source, l10n, non_resources=[], non_chrome=set()):
app_finder = UnpackFinder(source)
l10n_finder = UnpackFinder(l10n)
copier = FileCopier()
if app_finder.kind == 'flat':
formatter = FlatFormatter(copier)
elif app_finder.kind == 'jar':
formatter = JarFormatter(copier, optimize=app_finder.optimizedjars)
elif app_finder.kind == 'omni':
formatter = OmniJarFormatter(copier, app_finder.omnijar,
optimize=app_finder.optimizedjars,
non_resources=non_resources)
with errors.accumulate():
_repack(app_finder, l10n_finder, copier, formatter, non_chrome)
copier.copy(source, skip_if_older=False)
generate_precomplete(source)
def process_manifest(destdir, paths, track,
no_symlinks=False,
defines={}):
if os.path.exists(track):
# We use the same format as install manifests for the tracking
# data.
manifest = InstallManifest(path=track)
remove_unaccounted = FileRegistry()
dummy_file = BaseFile()
finder = FileFinder(destdir, find_dotfiles=True)
for dest in manifest._dests:
for p, f in finder.find(dest):
remove_unaccounted.add(p, dummy_file)
remove_empty_directories = True
remove_all_directory_symlinks = True
else:
# If tracking is enabled and there is no file, we don't want to
# be removing anything.
remove_unaccounted = False
remove_empty_directories = False
remove_all_directory_symlinks = False
manifest = InstallManifest()
for path in paths:
manifest |= InstallManifest(path=path)
copier = FileCopier()
link_policy = "copy" if no_symlinks else "symlink"
manifest.populate_registry(
copier, defines_override=defines, link_policy=link_policy
)
result = copier.copy(destdir,
remove_unaccounted=remove_unaccounted,
remove_all_directory_symlinks=remove_all_directory_symlinks,
remove_empty_directories=remove_empty_directories)
if track:
# We should record files that we actually copied.
# It is too late to expand wildcards when the track file is read.
manifest.write(path=track, expand_pattern=True)
return result
def test_copier_application(self):
dest = self.tmppath('dest')
os.mkdir(dest)
to_delete = self.tmppath('dest/to_delete')
with open(to_delete, 'a'):
pass
with open(self.tmppath('s_source'), 'wt') as fh:
fh.write('symlink!')
with open(self.tmppath('c_source'), 'wt') as fh:
fh.write('copy!')
with open(self.tmppath('dest/e_dest'), 'a'):
pass
with open(self.tmppath('dest/o_dest'), 'a'):
pass
m = self._get_test_manifest()
c = FileCopier()
m.populate_registry(c)
result = c.copy(dest)
self.assertTrue(os.path.exists(self.tmppath('dest/s_dest')))
self.assertTrue(os.path.exists(self.tmppath('dest/c_dest')))
self.assertTrue(os.path.exists(self.tmppath('dest/e_dest')))
self.assertTrue(os.path.exists(self.tmppath('dest/o_dest')))
self.assertFalse(os.path.exists(to_delete))
with open(self.tmppath('dest/s_dest'), 'rt') as fh:
self.assertEqual(fh.read(), 'symlink!')
with open(self.tmppath('dest/c_dest'), 'rt') as fh:
self.assertEqual(fh.read(), 'copy!')
self.assertEqual(
result.updated_files,
set(self.tmppath(p) for p in ('dest/s_dest', 'dest/c_dest')))
self.assertEqual(
result.existing_files,
set([self.tmppath('dest/e_dest'),
self.tmppath('dest/o_dest')]))
self.assertEqual(result.removed_files, {to_delete})
self.assertEqual(result.removed_directories, set())
def process_manifest(destdir,
paths,
track=None,
remove_unaccounted=True,
remove_all_directory_symlinks=True,
remove_empty_directories=True,
defines={}):
if track:
if os.path.exists(track):
# We use the same format as install manifests for the tracking
# data.
manifest = InstallManifest(path=track)
remove_unaccounted = FileRegistry()
dummy_file = BaseFile()
finder = FileFinder(destdir,
find_executables=False,
find_dotfiles=True)
for dest in manifest._dests:
for p, f in finder.find(dest):
remove_unaccounted.add(p, dummy_file)
else:
# If tracking is enabled and there is no file, we don't want to
# be removing anything.
remove_unaccounted = False
remove_empty_directories = False
remove_all_directory_symlinks = False
manifest = InstallManifest()
for path in paths:
manifest |= InstallManifest(path=path)
copier = FileCopier()
manifest.populate_registry(copier, defines_override=defines)
result = copier.copy(
destdir,
remove_unaccounted=remove_unaccounted,
remove_all_directory_symlinks=remove_all_directory_symlinks,
remove_empty_directories=remove_empty_directories)
if track:
manifest.write(path=track)
return result
def test_pattern_expansion(self):
source = self.tmppath('source')
os.mkdir(source)
os.mkdir('%s/base' % source)
os.mkdir('%s/base/foo' % source)
with open('%s/base/foo/file1' % source, 'a'):
pass
with open('%s/base/foo/file2' % source, 'a'):
pass
m = InstallManifest()
m.add_pattern_symlink('%s/base' % source, '**', 'dest')
c = FileCopier()
m.populate_registry(c)
self.assertEqual(c.paths(), ['dest/foo/file1', 'dest/foo/file2'])
def test_pattern_expansion(self):
source = self.tmppath("source")
os.mkdir(source)
os.mkdir("%s/base" % source)
os.mkdir("%s/base/foo" % source)
with open("%s/base/foo/file1" % source, "a"):
pass
with open("%s/base/foo/file2" % source, "a"):
pass
m = InstallManifest()
m.add_pattern_link("%s/base" % source, "**", "dest")
c = FileCopier()
m.populate_registry(c)
self.assertEqual(c.paths(), ["dest/foo/file1", "dest/foo/file2"])
def setup_benchmarks(self):
"""Make sure benchmarks are linked to the proper location in the objdir.
Benchmarks can either live in-tree or in an external repository. In the latter
case also clone/update the repository if necessary.
"""
print("Updating external benchmarks from {}".format(
BENCHMARK_REPOSITORY))
# Set up the external repo
external_repo_path = os.path.join(get_state_dir()[0],
'performance-tests')
if not os.path.isdir(external_repo_path):
subprocess.check_call(
['git', 'clone', BENCHMARK_REPOSITORY, external_repo_path])
else:
subprocess.check_call(['git', 'checkout', 'master'],
cwd=external_repo_path)
subprocess.check_call(['git', 'pull'], cwd=external_repo_path)
subprocess.check_call(['git', 'checkout', BENCHMARK_REVISION],
cwd=external_repo_path)
# Link benchmarks to the objdir
benchmark_paths = (
os.path.join(external_repo_path, 'benchmarks'),
os.path.join(self.topsrcdir, 'third_party', 'webkit',
'PerformanceTests'),
)
manifest = InstallManifest()
for benchmark_path in benchmark_paths:
for path in os.listdir(benchmark_path):
abspath = os.path.join(benchmark_path, path)
if not os.path.isdir(abspath) or path.startswith('.'):
continue
manifest.add_link(abspath, path)
copier = FileCopier()
manifest.populate_registry(copier)
copier.copy(
os.path.join(self.topobjdir, 'testing', 'raptor', 'benchmarks'))
def test_file_copier(self):
copier = FileCopier()
copier.add('foo/bar', GeneratedFile('foobar'))
copier.add('foo/qux', GeneratedFile('fooqux'))
copier.add('foo/deep/nested/directory/file', GeneratedFile('fooz'))
copier.add('bar', GeneratedFile('bar'))
copier.add('qux/foo', GeneratedFile('quxfoo'))
copier.add('qux/bar', GeneratedFile(''))
copier.copy(self.tmpdir)
self.assertEqual(self.all_files(self.tmpdir), set(copier.paths()))
self.assertEqual(self.all_dirs(self.tmpdir),
set(['foo/deep/nested/directory', 'qux']))
copier.remove('foo')
copier.add('test', GeneratedFile('test'))
copier.copy(self.tmpdir)
self.assertEqual(self.all_files(self.tmpdir), set(copier.paths()))
self.assertEqual(self.all_dirs(self.tmpdir), set(['qux']))
def main():
parser = argparse.ArgumentParser(
description="Merge two crashreporter symbols directories."
)
parser.add_argument("dir1", help="Directory")
parser.add_argument("dir2", help="Directory to merge")
options = parser.parse_args()
dir1_finder = FileFinder(options.dir1)
dir2_finder = FileFinder(options.dir2)
finder = UnifiedSymbolsFinder(dir1_finder, dir2_finder)
copier = FileCopier()
with errors.accumulate():
for p, f in finder:
copier.add(p, f)
copier.copy(options.dir1, skip_if_older=False)
def test_permissions(self):
"""Ensure files without write permission can be deleted."""
with open(self.tmppath("dummy"), "a"):
pass
p = self.tmppath("no_perms")
with open(p, "a"):
pass
# Make file and directory unwritable. Reminder: making a directory
# unwritable prevents modifications (including deletes) from the list
# of files in that directory.
os.chmod(p, 0o400)
os.chmod(self.tmpdir, 0o400)
copier = FileCopier()
copier.add("dummy", GeneratedFile(b"content"))
result = copier.copy(self.tmpdir)
self.assertEqual(result.removed_files_count, 1)
self.assertFalse(os.path.exists(p))
def install_test_files(topsrcdir, topobjdir, tests_root):
"""Installs the requested test files to the objdir. This is invoked by
test runners to avoid installing tens of thousands of test files when
only a few tests need to be run.
"""
manifest = InstallManifest(
mozpath.join(topobjdir, '_build_manifests', 'install', '_test_files'))
harness_files_manifest = mozpath.join(topobjdir, '_build_manifests',
'install', tests_root)
if os.path.isfile(harness_files_manifest):
# If the backend has generated an install manifest for test harness
# files they are treated as a monolith and installed each time we
# run tests. Fortunately there are not very many.
manifest |= InstallManifest(harness_files_manifest)
copier = FileCopier()
manifest.populate_registry(copier)
copier.copy(mozpath.join(topobjdir, tests_root), remove_unaccounted=False)
def test_no_remove(self):
copier = FileCopier()
copier.add('foo', GeneratedFile('foo'))
with open(self.tmppath('bar'), 'a'):
pass
os.mkdir(self.tmppath('emptydir'))
d = self.tmppath('populateddir')
os.mkdir(d)
with open(self.tmppath('populateddir/foo'), 'a'):
pass
result = copier.copy(self.tmpdir, remove_unaccounted=False)
self.assertEqual(self.all_files(self.tmpdir), set(['foo', 'bar',
'populateddir/foo']))
self.assertEqual(result.removed_files, set())
self.assertEqual(result.removed_directories,
set([self.tmppath('emptydir')]))
def test_no_remove(self):
copier = FileCopier()
copier.add("foo", GeneratedFile(b"foo"))
with open(self.tmppath("bar"), "a"):
pass
os.mkdir(self.tmppath("emptydir"))
d = self.tmppath("populateddir")
os.mkdir(d)
with open(self.tmppath("populateddir/foo"), "a"):
pass
result = copier.copy(self.tmpdir, remove_unaccounted=False)
self.assertEqual(self.all_files(self.tmpdir),
set(["foo", "bar", "populateddir/foo"]))
self.assertEqual(self.all_dirs(self.tmpdir), set(["populateddir"]))
self.assertEqual(result.removed_files, set())
self.assertEqual(result.removed_directories,
set([self.tmppath("emptydir")]))
def test_preprocessor_dependencies(self):
manifest = self.tmppath('m')
deps = self.tmppath('m.pp')
dest = self.tmppath('dest')
source = self.tmppath('p_source')
destfile = self.tmppath('dest/p_dest')
include = self.tmppath('p_incl')
os.mkdir(dest)
with open(source, 'wt') as fh:
fh.write('#define SRC\nSOURCE\n')
time = os.path.getmtime(source) - 3
os.utime(source, (time, time))
with open(include, 'wt') as fh:
fh.write('INCLUDE\n')
time = os.path.getmtime(source) - 3
os.utime(include, (time, time))
# Create and write a manifest with the preprocessed file.
m = InstallManifest()
m.add_preprocess(source, 'p_dest', deps, '#', {
'FOO': 'BAR',
'BAZ': 'QUX'
})
m.write(path=manifest)
time = os.path.getmtime(source) - 5
os.utime(manifest, (time, time))
# Now read the manifest back in, and apply it. This should write out
# our preprocessed file.
m = InstallManifest(path=manifest)
c = FileCopier()
m.populate_registry(c)
self.assertTrue(c.copy(dest))
with open(destfile, 'rt') as fh:
self.assertEqual(fh.read(), 'SOURCE\n')
# Next, modify the source to #INCLUDE another file.
with open(source, 'wt') as fh:
fh.write('SOURCE\n#include p_incl\n')
time = os.path.getmtime(source) - 1
os.utime(destfile, (time, time))
# Apply the manifest, and confirm that it also reads the newly included
# file.
m = InstallManifest(path=manifest)
c = FileCopier()
m.populate_registry(c)
c.copy(dest)
with open(destfile, 'rt') as fh:
self.assertEqual(fh.read(), 'SOURCE\nINCLUDE\n')
# Set the time on the source file back, so it won't be picked up as
# modified in the next test.
time = os.path.getmtime(source) - 1
os.utime(source, (time, time))
# Now, modify the include file (but not the original source).
with open(include, 'wt') as fh:
fh.write('INCLUDE MODIFIED\n')
time = os.path.getmtime(include) - 1
os.utime(destfile, (time, time))
# Apply the manifest, and confirm that the change to the include file
# is detected. That should cause the preprocessor to run again.
m = InstallManifest(path=manifest)
c = FileCopier()
m.populate_registry(c)
c.copy(dest)
with open(destfile, 'rt') as fh:
self.assertEqual(fh.read(), 'SOURCE\nINCLUDE MODIFIED\n')
# ORing an InstallManifest should copy file dependencies
m = InstallManifest()
m |= InstallManifest(path=manifest)
c = FileCopier()
m.populate_registry(c)
e = c._files['p_dest']
self.assertEqual(e.extra_depends, [manifest])
def test_remove_unaccounted_directory_symlinks(self):
"""Directory symlinks in destination that are not in the way are
deleted according to remove_unaccounted and
remove_all_directory_symlinks.
"""
if not self.symlink_supported:
return
dest = self.tmppath("dest")
copier = FileCopier()
copier.add("foo/bar/baz", GeneratedFile(b"foobarbaz"))
os.makedirs(self.tmppath("dest/foo"))
dummy = self.tmppath("dummy")
os.mkdir(dummy)
os.mkdir(self.tmppath("dest/zot"))
link = self.tmppath("dest/zot/zap")
os.symlink(dummy, link)
# If not remove_unaccounted but remove_empty_directories, then
# the symlinked directory remains (as does its containing
# directory).
result = copier.copy(
dest,
remove_unaccounted=False,
remove_empty_directories=True,
remove_all_directory_symlinks=False,
)
st = os.lstat(link)
self.assertTrue(stat.S_ISLNK(st.st_mode))
self.assertFalse(stat.S_ISDIR(st.st_mode))
self.assertEqual(self.all_files(dest), set(copier.paths()))
self.assertEqual(self.all_dirs(dest), set(["foo/bar"]))
self.assertEqual(result.removed_directories, set())
self.assertEqual(len(result.updated_files), 1)
# If remove_unaccounted but not remove_empty_directories, then
# only the symlinked directory is removed.
result = copier.copy(
dest,
remove_unaccounted=True,
remove_empty_directories=False,
remove_all_directory_symlinks=False,
)
st = os.lstat(self.tmppath("dest/zot"))
self.assertFalse(stat.S_ISLNK(st.st_mode))
self.assertTrue(stat.S_ISDIR(st.st_mode))
self.assertEqual(result.removed_files, set([link]))
self.assertEqual(result.removed_directories, set())
self.assertEqual(self.all_files(dest), set(copier.paths()))
self.assertEqual(self.all_dirs(dest), set(["foo/bar", "zot"]))
# If remove_unaccounted and remove_empty_directories, then
# both the symlink and its containing directory are removed.
link = self.tmppath("dest/zot/zap")
os.symlink(dummy, link)
result = copier.copy(
dest,
remove_unaccounted=True,
remove_empty_directories=True,
remove_all_directory_symlinks=False,
)
self.assertEqual(result.removed_files, set([link]))
self.assertEqual(result.removed_directories,
set([self.tmppath("dest/zot")]))
self.assertEqual(self.all_files(dest), set(copier.paths()))
self.assertEqual(self.all_dirs(dest), set(["foo/bar"]))
def main():
parser = ArgumentParser()
parser.add_argument('-D',
dest='defines',
action='append',
metavar="VAR[=VAL]",
help='Define a variable')
parser.add_argument('--format',
default='omni',
help='Choose the chrome format for packaging ' +
'(omni, jar or flat ; default: %(default)s)')
parser.add_argument('--removals',
default=None,
help='removed-files source file')
parser.add_argument('--ignore-errors',
action='store_true',
default=False,
help='Transform errors into warnings.')
parser.add_argument('--minify',
action='store_true',
default=False,
help='Make some files more compact while packaging')
parser.add_argument('--minify-js',
action='store_true',
help='Minify JavaScript files while packaging.')
parser.add_argument('--js-binary',
help='Path to js binary. This is used to verify '
'minified JavaScript. If this is not defined, '
'minification verification will not be performed.')
parser.add_argument('--jarlog',
default='',
help='File containing jar ' + 'access logs')
parser.add_argument('--optimizejars',
action='store_true',
default=False,
help='Enable jar optimizations')
parser.add_argument('--unify',
default='',
help='Base directory of another build to unify with')
parser.add_argument('manifest',
default=None,
nargs='?',
help='Manifest file name')
parser.add_argument('source', help='Source directory')
parser.add_argument('destination', help='Destination directory')
parser.add_argument('--non-resource',
nargs='+',
metavar='PATTERN',
default=[],
help='Extra files not to be considered as resources')
args = parser.parse_args()
defines = dict(buildconfig.defines)
if args.ignore_errors:
errors.ignore_errors()
if args.defines:
for name, value in [split_define(d) for d in args.defines]:
defines[name] = value
copier = FileCopier()
if args.format == 'flat':
formatter = FlatFormatter(copier)
elif args.format == 'jar':
formatter = JarFormatter(copier, optimize=args.optimizejars)
elif args.format == 'omni':
formatter = OmniJarFormatter(copier,
buildconfig.substs['OMNIJAR_NAME'],
optimize=args.optimizejars,
non_resources=args.non_resource)
else:
errors.fatal('Unknown format: %s' % args.format)
# Adjust defines according to the requested format.
if isinstance(formatter, OmniJarFormatter):
defines['MOZ_OMNIJAR'] = 1
elif 'MOZ_OMNIJAR' in defines:
del defines['MOZ_OMNIJAR']
binpath = ''
if 'BINPATH' in defines:
binpath = SimpleManifestSink.normalize_path(defines['BINPATH'])
while binpath.startswith('/'):
binpath = binpath[1:]
if args.unify:
def is_native(path):
path = os.path.abspath(path)
return platform.machine() in mozpack.path.split(path)
# Invert args.unify and args.source if args.unify points to the
# native architecture.
args.source, args.unify = sorted([args.source, args.unify],
key=is_native,
reverse=True)
if is_native(args.source):
launcher.tooldir = args.source
elif not buildconfig.substs['CROSS_COMPILE']:
launcher.tooldir = buildconfig.substs['LIBXUL_DIST']
with errors.accumulate():
finder_args = dict(
minify=args.minify,
minify_js=args.minify_js,
)
if args.js_binary:
finder_args['minify_js_verify_command'] = [
args.js_binary,
os.path.join(os.path.abspath(os.path.dirname(__file__)),
'js-compare-ast.js')
]
if args.unify:
finder = UnifiedBuildFinder(FileFinder(args.source),
FileFinder(args.unify), **finder_args)
else:
finder = FileFinder(args.source, **finder_args)
if 'NO_PKG_FILES' in os.environ:
sinkformatter = NoPkgFilesRemover(formatter, args.manifest
is not None)
else:
sinkformatter = formatter
sink = SimpleManifestSink(finder, sinkformatter)
if args.manifest:
preprocess_manifest(sink, args.manifest, defines)
else:
sink.add(Component(''), 'bin/*')
sink.close(args.manifest is not None)
if args.removals:
lines = [l.lstrip() for l in open(args.removals).readlines()]
removals_in = StringIO(''.join(lines))
removals_in.name = args.removals
removals = RemovedFiles(copier)
preprocess(removals_in, removals, defines)
copier.add(mozpack.path.join(binpath, 'removed-files'), removals)
# shlibsign libraries
if launcher.can_launch():
for lib in SIGN_LIBS:
libbase = mozpack.path.join(binpath, '%s%s') \
% (buildconfig.substs['DLL_PREFIX'], lib)
libname = '%s%s' % (libbase, buildconfig.substs['DLL_SUFFIX'])
if copier.contains(libname):
copier.add(
libbase + '.chk',
LibSignFile(os.path.join(args.destination, libname)))
# Setup preloading
if args.jarlog and os.path.exists(args.jarlog):
from mozpack.mozjar import JarLog
log = JarLog(args.jarlog)
for p, f in copier:
if not isinstance(f, Jarrer):
continue
key = JarLog.canonicalize(os.path.join(args.destination, p))
if key in log:
f.preload(log[key])
# Fill startup cache
if isinstance(formatter, OmniJarFormatter) and launcher.can_launch() \
and buildconfig.substs['MOZ_DISABLE_STARTUPCACHE'] != '1':
if buildconfig.substs['LIBXUL_SDK']:
gre_path = mozpack.path.join(buildconfig.substs['LIBXUL_DIST'],
'bin')
else:
gre_path = None
for base in sorted([[
p for p in [mozpack.path.join('bin', b), b]
if os.path.exists(os.path.join(args.source, p))
][0] for b in sink.packager.get_bases()]):
if not gre_path:
gre_path = base
base_path = sink.normalize_path(base)
if base_path in formatter.omnijars:
precompile_cache(formatter.omnijars[base_path], args.source,
gre_path, base)
copier.copy(args.destination)
generate_precomplete(
os.path.normpath(os.path.join(args.destination, binpath)))
def main(output_dirname, verbose, *input_dirs):
# Map directories to source paths, like
# `{'values-large-v11': ['/path/to/values-large-v11/strings.xml',
# '/path/to/values-large-v11/colors.xml', ...], ...}`.
values = defaultdict(list)
# Map unversioned resource names to maps from versions to source paths, like:
# `{'drawable-large/icon.png':
# {None: '/path/to/drawable-large/icon.png',
# 11: '/path/to/drawable-large-v11/icon.png', ...}, ...}`.
resources = defaultdict(dict)
manifest = InstallManifest()
for p in uniqify(input_dirs):
finder = FileFinder(p, find_executables=False)
values_pattern = 'values*/*.xml'
for path, _ in finder.find('*/*'):
if path in MANIFEST_EXCLUSIONS:
continue
source_path = mozpath.join(finder.base, path)
if mozpath.match(path, values_pattern):
dir, _name = path.split('/')
dir = with_version(dir)
values[dir].append(source_path)
continue
(resource, version) = classify(path)
# Earlier paths are taken in preference to later paths.
# This agrees with aapt.
if version not in resources:
resources[resource][version] = source_path
# Step 1: merge all XML values into one single, sorted
# per-configuration values.xml file. This apes what the Android
# Gradle resource merging algorithm does.
merged_values = defaultdict(list)
for dir, files in values.items():
for file in files:
values = ET.ElementTree(file=file).getroot()
merged_values[dir].extend(values)
values = ET.Element('resources')
# Sort by <type> tag, and then by name. Note that <item
# type="type"> is equivalent to <type>.
key = lambda x: (resource_type.get(x.get('type', x.tag)), x.get('name')
)
values[:] = sorted(merged_values[dir], key=key)
for value in values:
if value.get('name') == 'TextAppearance.Design.Snackbar.Message':
if value.get('{http://schemas.android.com/tools}override',
False):
values.remove(value)
break
merged_values[dir] = values
for dir, values in merged_values.items():
o = mozpath.join(output_dirname, dir, '{}.xml'.format(dir))
ensureParentDir(o)
ET.ElementTree(values).write(o)
manifest.add_required_exists(mozpath.join(dir, '{}.xml'.format(dir)))
# Step 2a: add version numbers for unversioned features
# corresponding to when the feature was introduced. Resource
# qualifiers will never be recognized by Android versions before
# they were introduced. For example, density qualifiers are
# supported only in Android v4 and above. Therefore
# "drawable-hdpi" is implicitly "drawable-hdpi-v4". We version
# such unversioned resources here.
for (resource, versions) in resources.items():
if None in versions:
dir, name = resource.split('/')
new_dir = with_version(dir)
(new_resource,
new_version) = classify('{}/{}'.format(new_dir, name))
if new_resource != resource:
raise ValueError('this is bad')
# `new_version` might be None: for example, `dir` might be "drawable".
source_path = versions.pop(None)
versions[new_version] = source_path
if verbose:
if new_version:
print("Versioning unversioned resource {} as {}-v{}/{}".
format(source_path, dir, new_version, name))
# TODO: make this a command line argument that takes MOZ_ANDROID_MIN_SDK_VERSION.
min_sdk = 15
retained = defaultdict(dict)
# Step 2b: drop resource directories that will never be used by
# Android on device. This depends on the minimum supported
# Android SDK version. Suppose the minimum SDK is 15 and we have
# drawable-v4/icon.png and drawable-v11/icon.png. The v4 version
# will never be chosen, since v15 is always greater than v11.
for (resource, versions) in resources.items():
def key(v):
return 0 if v is None else v
# Versions in descending order.
version_list = sorted(versions.keys(), key=key, reverse=True)
for version in version_list:
retained[resource][version] = versions[version]
if version is not None and version <= min_sdk:
break
if set(retained.keys()) != set(resources.keys()):
raise ValueError('Something terrible has happened; retained '
'resource names do not match input resources '
'names')
if verbose:
for resource in resources:
if resources[resource] != retained[resource]:
for version in sorted(resources[resource].keys(),
reverse=True):
if version in retained[resource]:
print("Keeping reachable resource {}".format(
resources[resource][version]))
else:
print("Dropping unreachable resource {}".format(
resources[resource][version]))
# Populate manifest.
for (resource, versions) in retained.items():
for version in sorted(versions.keys(), reverse=True):
path = resource
if version:
dir, name = resource.split('/')
path = '{}-v{}/{}'.format(dir, version, name)
manifest.add_copy(versions[version], path)
copier = FileCopier()
manifest.populate_registry(copier)
print('mr', os.getcwd())
result = copier.copy(output_dirname,
remove_unaccounted=True,
remove_all_directory_symlinks=False,
remove_empty_directories=True)
if verbose:
print('Updated:', result.updated_files_count)
print('Removed:',
result.removed_files_count + result.removed_directories_count)
print('Existed:', result.existing_files_count)
return 0
def install_test_files(topsrcdir, topobjdir, tests_root, test_objs):
"""Installs the requested test files to the objdir. This is invoked by
test runners to avoid installing tens of thousands of test files when
only a few tests need to be run.
"""
flavor_info = {flavor: (root, prefix, install)
for (flavor, root, prefix, install) in TEST_MANIFESTS.values()}
objdir_dest = mozpath.join(topobjdir, tests_root)
converter = SupportFilesConverter()
install_info = TestInstallInfo()
for o in test_objs:
flavor = o['flavor']
if flavor not in flavor_info:
# This is a test flavor that isn't installed by the build system.
continue
root, prefix, install = flavor_info[flavor]
if not install:
# This flavor isn't installed to the objdir.
continue
manifest_path = o['manifest']
manifest_dir = mozpath.dirname(manifest_path)
out_dir = mozpath.join(root, prefix, manifest_dir[len(topsrcdir) + 1:])
file_relpath = o['file_relpath']
source = mozpath.join(topsrcdir, file_relpath)
dest = mozpath.join(root, prefix, file_relpath)
if 'install-to-subdir' in o:
out_dir = mozpath.join(out_dir, o['install-to-subdir'])
manifest_relpath = mozpath.relpath(source, mozpath.dirname(manifest_path))
dest = mozpath.join(out_dir, manifest_relpath)
install_info.installs.append((source, dest))
install_info |= converter.convert_support_files(o, root,
manifest_dir,
out_dir)
manifest = InstallManifest()
for source, dest in set(install_info.installs):
if dest in install_info.external_installs:
continue
manifest.add_link(source, dest)
for base, pattern, dest in install_info.pattern_installs:
manifest.add_pattern_link(base, pattern, dest)
_resolve_installs(install_info.deferred_installs, topobjdir, manifest)
harness_files_manifest = mozpath.join(topobjdir, '_build_manifests',
'install', tests_root)
if os.path.isfile(harness_files_manifest):
# If the backend has generated an install manifest for test harness
# files they are treated as a monolith and installed each time we
# run tests. Fortunately there are not very many.
manifest |= InstallManifest(harness_files_manifest)
copier = FileCopier()
manifest.populate_registry(copier)
copier.copy(objdir_dest,
remove_unaccounted=False)
def main():
parser = ArgumentParser()
parser.add_argument(
"-D",
dest="defines",
action="append",
metavar="VAR[=VAL]",
help="Define a variable",
)
parser.add_argument(
"--format",
default="omni",
help="Choose the chrome format for packaging " +
"(omni, jar or flat ; default: %(default)s)",
)
parser.add_argument("--removals",
default=None,
help="removed-files source file")
parser.add_argument(
"--ignore-errors",
action="store_true",
default=False,
help="Transform errors into warnings.",
)
parser.add_argument(
"--ignore-broken-symlinks",
action="store_true",
default=False,
help="Do not fail when processing broken symlinks.",
)
parser.add_argument(
"--minify",
action="store_true",
default=False,
help="Make some files more compact while packaging",
)
parser.add_argument(
"--minify-js",
action="store_true",
help="Minify JavaScript files while packaging.",
)
parser.add_argument(
"--js-binary",
help="Path to js binary. This is used to verify "
"minified JavaScript. If this is not defined, "
"minification verification will not be performed.",
)
parser.add_argument("--jarlog",
default="",
help="File containing jar " + "access logs")
parser.add_argument(
"--compress",
choices=("none", "deflate"),
default="deflate",
help="Use given jar compression (default: deflate)",
)
parser.add_argument("manifest",
default=None,
nargs="?",
help="Manifest file name")
parser.add_argument("source", help="Source directory")
parser.add_argument("destination", help="Destination directory")
parser.add_argument(
"--non-resource",
nargs="+",
metavar="PATTERN",
default=[],
help="Extra files not to be considered as resources",
)
args = parser.parse_args()
defines = dict(buildconfig.defines["ALLDEFINES"])
if args.ignore_errors:
errors.ignore_errors()
if args.defines:
for name, value in [split_define(d) for d in args.defines]:
defines[name] = value
compress = {
"none": False,
"deflate": True,
}[args.compress]
copier = FileCopier()
if args.format == "flat":
formatter = FlatFormatter(copier)
elif args.format == "jar":
formatter = JarFormatter(copier, compress=compress)
elif args.format == "omni":
formatter = OmniJarFormatter(
copier,
buildconfig.substs["OMNIJAR_NAME"],
compress=compress,
non_resources=args.non_resource,
)
else:
errors.fatal("Unknown format: %s" % args.format)
# Adjust defines according to the requested format.
if isinstance(formatter, OmniJarFormatter):
defines["MOZ_OMNIJAR"] = 1
elif "MOZ_OMNIJAR" in defines:
del defines["MOZ_OMNIJAR"]
respath = ""
if "RESPATH" in defines:
respath = SimpleManifestSink.normalize_path(defines["RESPATH"])
while respath.startswith("/"):
respath = respath[1:]
with errors.accumulate():
finder_args = dict(
minify=args.minify,
minify_js=args.minify_js,
ignore_broken_symlinks=args.ignore_broken_symlinks,
)
if args.js_binary:
finder_args["minify_js_verify_command"] = [
args.js_binary,
os.path.join(os.path.abspath(os.path.dirname(__file__)),
"js-compare-ast.js"),
]
finder = PackagerFileFinder(args.source,
find_executables=True,
**finder_args)
if "NO_PKG_FILES" in os.environ:
sinkformatter = NoPkgFilesRemover(formatter, args.manifest
is not None)
else:
sinkformatter = formatter
sink = SimpleManifestSink(finder, sinkformatter)
if args.manifest:
preprocess_manifest(sink, args.manifest, defines)
else:
sink.add(Component(""), "bin/*")
sink.close(args.manifest is not None)
if args.removals:
removals_in = StringIO(open(args.removals).read())
removals_in.name = args.removals
removals = RemovedFiles(copier)
preprocess(removals_in, removals, defines)
copier.add(mozpath.join(respath, "removed-files"), removals)
# If a pdb file is present and we were instructed to copy it, include it.
# Run on all OSes to capture MinGW builds
if buildconfig.substs.get("MOZ_COPY_PDBS"):
# We want to mutate the copier while we're iterating through it, so copy
# the items to a list first.
copier_items = [(p, f) for p, f in copier]
for p, f in copier_items:
if isinstance(f, ExecutableFile):
pdbname = os.path.splitext(f.inputs()[0])[0] + ".pdb"
if os.path.exists(pdbname):
copier.add(os.path.basename(pdbname), File(pdbname))
# Setup preloading
if args.jarlog:
if not os.path.exists(args.jarlog):
raise Exception("Cannot find jar log: %s" % args.jarlog)
omnijars = []
if isinstance(formatter, OmniJarFormatter):
omnijars = [
mozpath.join(base, buildconfig.substs["OMNIJAR_NAME"])
for base in sink.packager.get_bases(addons=False)
]
from mozpack.mozjar import JarLog
log = JarLog(args.jarlog)
for p, f in copier:
if not isinstance(f, Jarrer):
continue
if respath:
p = mozpath.relpath(p, respath)
if p in log:
f.preload(log[p])
elif p in omnijars:
raise Exception("No jar log data for %s" % p)
copier.copy(args.destination)
generate_precomplete(
os.path.normpath(os.path.join(args.destination, respath)))
def _get_copier(cls):
copier = FileCopier()
formatter = cls(copier)
fill_formatter(formatter, CONTENTS)
return copier
def fat_aar(distdir,
aars_paths,
no_process=False,
no_compatibility_check=False):
if no_process:
print('Not processing architecture-specific artifact Maven AARs.')
return 0
# Map {filename: {fingerprint: [arch1, arch2, ...]}}.
diffs = defaultdict(lambda: defaultdict(list))
missing_arch_prefs = set()
# Collect multi-architecture inputs to the fat AAR.
copier = FileCopier()
for arch, aar_path in aars_paths.items():
# Map old non-architecture-specific path to new architecture-specific path.
old_rewrite_map = {
'greprefs.js':
'{}/greprefs.js'.format(arch),
'defaults/pref/geckoview-prefs.js':
'defaults/pref/{}/geckoview-prefs.js'.format(arch),
}
# Architecture-specific preferences files.
arch_prefs = set(old_rewrite_map.values())
missing_arch_prefs |= set(arch_prefs)
jar_finder = JarFinder(aar_path, JarReader(aar_path))
for path, fileobj in UnpackFinder(jar_finder):
# Native libraries go straight through.
if mozpath.match(path, 'jni/**'):
copier.add(path, fileobj)
elif path in arch_prefs:
copier.add(path, fileobj)
elif path in ('classes.jar', 'annotations.zip'):
# annotations.zip differs due to timestamps, but the contents should not.
# `JarReader` fails on the non-standard `classes.jar` produced by Gradle/aapt,
# and it's not worth working around, so we use Python's zip functionality
# instead.
z = ZipFile(BytesIO(fileobj.open().read()))
for r in z.namelist():
fingerprint = sha1(z.open(r).read()).hexdigest()
diffs['{}!/{}'.format(path, r)][fingerprint].append(arch)
else:
fingerprint = sha1(fileobj.open().read()).hexdigest()
# There's no need to distinguish `target.maven.zip` from `assets/omni.ja` here,
# since in practice they will never overlap.
diffs[path][fingerprint].append(arch)
missing_arch_prefs.discard(path)
# Some differences are allowed across the architecture-specific AARs. We could allow-list
# the actual content, but it's not necessary right now.
allow_pattern_list = {
'AndroidManifest.xml', # Min SDK version is different for 32- and 64-bit builds.
'classes.jar!/org/mozilla/gecko/util/HardwareUtils.class', # Min SDK as well.
'classes.jar!/org/mozilla/geckoview/BuildConfig.class',
# Each input captures its CPU architecture.
'chrome/toolkit/content/global/buildconfig.html',
# Bug 1556162: localized resources are not deterministic across
# per-architecture builds triggered from the same push.
'**/*.ftl',
'**/*.dtd',
'**/*.properties',
}
not_allowed = OrderedDict()
def format_diffs(ds):
# Like ' armeabi-v7a, arm64-v8a -> XXX\n x86, x86_64 -> YYY'.
return '\n'.join(
sorted(' {archs} -> {fingerprint}'.format(
archs=', '.join(sorted(archs)), fingerprint=fingerprint)
for fingerprint, archs in ds.iteritems()))
for p, ds in sorted(diffs.iteritems()):
if len(ds) <= 1:
# Only one hash across all inputs: roll on.
continue
if any(mozpath.match(p, pat) for pat in allow_pattern_list):
print(
'Allowed: Path "{path}" has architecture-specific versions:\n{ds_repr}'
.format(path=p, ds_repr=format_diffs(ds)))
continue
not_allowed[p] = ds
for p, ds in not_allowed.iteritems():
print(
'Disallowed: Path "{path}" has architecture-specific versions:\n{ds_repr}'
.format(path=p, ds_repr=format_diffs(ds)))
for missing in sorted(missing_arch_prefs):
print(
'Disallowed: Inputs missing expected architecture-specific input: {missing}'
.format(missing=missing))
if not no_compatibility_check and (missing_arch_prefs or not_allowed):
return 1
output_dir = mozpath.join(distdir, 'output')
copier.copy(output_dir)
return 0
def test_preprocessor(self):
manifest = self.tmppath('m')
deps = self.tmppath('m.pp')
dest = self.tmppath('dest')
include = self.tmppath('p_incl')
with open(include, 'wt') as fh:
fh.write('#define INCL\n')
time = os.path.getmtime(include) - 3
os.utime(include, (time, time))
with open(self.tmppath('p_source'), 'wt') as fh:
fh.write('#ifdef FOO\n#if BAZ == QUX\nPASS1\n#endif\n#endif\n')
fh.write('#ifdef DEPTEST\nPASS2\n#endif\n')
fh.write('#include p_incl\n#ifdef INCLTEST\nPASS3\n#endif\n')
time = os.path.getmtime(self.tmppath('p_source')) - 3
os.utime(self.tmppath('p_source'), (time, time))
# Create and write a manifest with the preprocessed file, then apply it.
# This should write out our preprocessed file.
m = InstallManifest()
m.add_preprocess(self.tmppath('p_source'), 'p_dest', deps, '#', {
'FOO': 'BAR',
'BAZ': 'QUX'
})
m.write(path=manifest)
m = InstallManifest(path=manifest)
c = FileCopier()
m.populate_registry(c)
c.copy(dest)
self.assertTrue(os.path.exists(self.tmppath('dest/p_dest')))
with open(self.tmppath('dest/p_dest'), 'rt') as fh:
self.assertEqual(fh.read(), 'PASS1\n')
# Create a second manifest with the preprocessed file, then apply it.
# Since this manifest does not exist on the disk, there should not be a
# dependency on it, and the preprocessed file should not be modified.
m2 = InstallManifest()
m2.add_preprocess(self.tmppath('p_source'), 'p_dest', deps, '#',
{'DEPTEST': True})
c = FileCopier()
m2.populate_registry(c)
result = c.copy(dest)
self.assertFalse(self.tmppath('dest/p_dest') in result.updated_files)
self.assertTrue(self.tmppath('dest/p_dest') in result.existing_files)
# Write out the second manifest, then load it back in from the disk.
# This should add the dependency on the manifest file, so our
# preprocessed file should be regenerated with the new defines.
# We also set the mtime on the destination file back, so it will be
# older than the manifest file.
m2.write(path=manifest)
time = os.path.getmtime(manifest) - 1
os.utime(self.tmppath('dest/p_dest'), (time, time))
m2 = InstallManifest(path=manifest)
c = FileCopier()
m2.populate_registry(c)
self.assertTrue(c.copy(dest))
with open(self.tmppath('dest/p_dest'), 'rt') as fh:
self.assertEqual(fh.read(), 'PASS2\n')
# Set the time on the manifest back, so it won't be picked up as
# modified in the next test
time = os.path.getmtime(manifest) - 1
os.utime(manifest, (time, time))
# Update the contents of a file included by the source file. This should
# cause the destination to be regenerated.
with open(include, 'wt') as fh:
fh.write('#define INCLTEST\n')
time = os.path.getmtime(include) - 1
os.utime(self.tmppath('dest/p_dest'), (time, time))
c = FileCopier()
m2.populate_registry(c)
self.assertTrue(c.copy(dest))
with open(self.tmppath('dest/p_dest'), 'rt') as fh:
self.assertEqual(fh.read(), 'PASS2\nPASS3\n')
def main():
parser = ArgumentParser()
parser.add_argument('-D', dest='defines', action='append',
metavar="VAR[=VAL]", help='Define a variable')
parser.add_argument('--format', default='omni',
help='Choose the chrome format for packaging ' +
'(omni, jar or flat ; default: %(default)s)')
parser.add_argument('--removals', default=None,
help='removed-files source file')
parser.add_argument('--ignore-errors', action='store_true', default=False,
help='Transform errors into warnings.')
parser.add_argument('--ignore-broken-symlinks', action='store_true', default=False,
help='Do not fail when processing broken symlinks.')
parser.add_argument('--minify', action='store_true', default=False,
help='Make some files more compact while packaging')
parser.add_argument('--minify-js', action='store_true',
help='Minify JavaScript files while packaging.')
parser.add_argument('--js-binary',
help='Path to js binary. This is used to verify '
'minified JavaScript. If this is not defined, '
'minification verification will not be performed.')
parser.add_argument('--jarlog', default='', help='File containing jar ' +
'access logs')
parser.add_argument('--compress', choices=('none', 'deflate', 'brotli'),
default='deflate',
help='Use given jar compression (default: deflate)')
parser.add_argument('manifest', default=None, nargs='?',
help='Manifest file name')
parser.add_argument('source', help='Source directory')
parser.add_argument('destination', help='Destination directory')
parser.add_argument('--non-resource', nargs='+', metavar='PATTERN',
default=[],
help='Extra files not to be considered as resources')
args = parser.parse_args()
defines = dict(buildconfig.defines['ALLDEFINES'])
if args.ignore_errors:
errors.ignore_errors()
if args.defines:
for name, value in [split_define(d) for d in args.defines]:
defines[name] = value
compress = {
'none': False,
'deflate': True,
'brotli': JAR_BROTLI,
}[args.compress]
copier = FileCopier()
if args.format == 'flat':
formatter = FlatFormatter(copier)
elif args.format == 'jar':
formatter = JarFormatter(copier, compress=compress)
elif args.format == 'omni':
formatter = OmniJarFormatter(copier,
buildconfig.substs['OMNIJAR_NAME'],
compress=compress,
non_resources=args.non_resource)
else:
errors.fatal('Unknown format: %s' % args.format)
# Adjust defines according to the requested format.
if isinstance(formatter, OmniJarFormatter):
defines['MOZ_OMNIJAR'] = 1
elif 'MOZ_OMNIJAR' in defines:
del defines['MOZ_OMNIJAR']
respath = ''
if 'RESPATH' in defines:
respath = SimpleManifestSink.normalize_path(defines['RESPATH'])
while respath.startswith('/'):
respath = respath[1:]
if not buildconfig.substs['CROSS_COMPILE']:
launcher.tooldir = mozpath.join(buildconfig.topobjdir, 'dist')
with errors.accumulate():
finder_args = dict(
minify=args.minify,
minify_js=args.minify_js,
ignore_broken_symlinks=args.ignore_broken_symlinks,
)
if args.js_binary:
finder_args['minify_js_verify_command'] = [
args.js_binary,
os.path.join(os.path.abspath(os.path.dirname(__file__)),
'js-compare-ast.js')
]
finder = FileFinder(args.source, find_executables=True,
**finder_args)
if 'NO_PKG_FILES' in os.environ:
sinkformatter = NoPkgFilesRemover(formatter,
args.manifest is not None)
else:
sinkformatter = formatter
sink = SimpleManifestSink(finder, sinkformatter)
if args.manifest:
preprocess_manifest(sink, args.manifest, defines)
else:
sink.add(Component(''), 'bin/*')
sink.close(args.manifest is not None)
if args.removals:
removals_in = StringIO(open(args.removals).read())
removals_in.name = args.removals
removals = RemovedFiles(copier)
preprocess(removals_in, removals, defines)
copier.add(mozpath.join(respath, 'removed-files'), removals)
# shlibsign libraries
if launcher.can_launch():
if not mozinfo.isMac and buildconfig.substs.get('COMPILE_ENVIRONMENT'):
for lib in SIGN_LIBS:
libbase = mozpath.join(respath, '%s%s') \
% (buildconfig.substs['DLL_PREFIX'], lib)
libname = '%s%s' % (libbase, buildconfig.substs['DLL_SUFFIX'])
if copier.contains(libname):
copier.add(libbase + '.chk',
LibSignFile(os.path.join(args.destination,
libname)))
# If a pdb file is present and we were instructed to copy it, include it.
# Run on all OSes to capture MinGW builds
if buildconfig.substs.get('MOZ_COPY_PDBS'):
for p, f in copier:
if isinstance(f, ExecutableFile):
pdbname = os.path.splitext(f.inputs()[0])[0] + '.pdb'
if os.path.exists(pdbname):
copier.add(os.path.basename(pdbname), File(pdbname))
# Setup preloading
if args.jarlog:
if not os.path.exists(args.jarlog):
raise Exception('Cannot find jar log: %s' % args.jarlog)
omnijars = []
if isinstance(formatter, OmniJarFormatter):
omnijars = [mozpath.join(base, buildconfig.substs['OMNIJAR_NAME'])
for base in sink.packager.get_bases(addons=False)]
from mozpack.mozjar import JarLog
log = JarLog(args.jarlog)
for p, f in copier:
if not isinstance(f, Jarrer):
continue
if respath:
p = mozpath.relpath(p, respath)
if p in log:
f.preload(log[p])
elif p in omnijars:
raise Exception('No jar log data for %s' % p)
copier.copy(args.destination)
generate_precomplete(os.path.normpath(os.path.join(args.destination,
respath)))
def _process_android_eclipse_project_data(self, data, srcdir, objdir):
# This can't be relative to the environment's topsrcdir,
# because during testing topsrcdir is faked.
template_directory = os.path.abspath(
mozpath.join(os.path.dirname(__file__), 'templates',
'android_eclipse'))
project_directory = mozpath.join(self.environment.topobjdir,
'android_eclipse', data.name)
manifest_path = mozpath.join(self.environment.topobjdir,
'android_eclipse',
'%s.manifest' % data.name)
manifest = self._manifest_for_project(srcdir, data)
ensureParentDir(manifest_path)
manifest.write(path=manifest_path)
classpathentries = []
for cpe in sorted(data._classpathentries, key=lambda x: x.path):
e = self._Element_for_classpathentry(cpe)
classpathentries.append(ET.tostring(e))
for name in sorted(data.referenced_projects):
e = self._Element_for_referenced_project(name)
classpathentries.append(ET.tostring(e))
for name in sorted(data.extra_jars):
e = self._Element_for_extra_jar(mozpath.join(srcdir, name))
classpathentries.append(ET.tostring(e))
defines = {}
defines['IDE_OBJDIR'] = objdir
defines['IDE_TOPOBJDIR'] = self.environment.topobjdir
defines['IDE_SRCDIR'] = srcdir
defines['IDE_TOPSRCDIR'] = self.environment.topsrcdir
defines['IDE_PROJECT_NAME'] = data.name
defines['IDE_PACKAGE_NAME'] = data.package_name
defines['IDE_PROJECT_DIRECTORY'] = project_directory
defines['IDE_RELSRCDIR'] = mozpath.relpath(srcdir,
self.environment.topsrcdir)
defines['IDE_CLASSPATH_ENTRIES'] = '\n'.join(
'\t' + cpe for cpe in classpathentries)
defines['IDE_RECURSIVE_MAKE_TARGETS'] = ' '.join(
sorted(data.recursive_make_targets))
# Like android.library=true
defines[
'IDE_PROJECT_LIBRARY_SETTING'] = 'android.library=true' if data.is_library else ''
# Like android.library.reference.1=FennecBrandingResources
defines['IDE_PROJECT_LIBRARY_REFERENCES'] = '\n'.join(
'android.library.reference.%s=%s' % (i + 1, ref)
for i, ref in enumerate(sorted(data.included_projects)))
if data.filtered_resources:
filteredResources = self._Element_for_filtered_resources(
data.filtered_resources)
defines['IDE_PROJECT_FILTERED_RESOURCES'] = pretty_print(
filteredResources).strip()
else:
defines['IDE_PROJECT_FILTERED_RESOURCES'] = ''
defines['ANDROID_TARGET_SDK'] = self.environment.substs[
'ANDROID_TARGET_SDK']
defines['MOZ_ANDROID_MIN_SDK_VERSION'] = self.environment.defines[
'MOZ_ANDROID_MIN_SDK_VERSION']
copier = FileCopier()
finder = FileFinder(template_directory)
for input_filename, f in itertools.chain(finder.find('**'),
finder.find('.**')):
if input_filename == 'AndroidManifest.xml' and not data.is_library:
# Main projects supply their own manifests.
continue
copier.add(
input_filename,
PreprocessedFile(
mozpath.join(finder.base, input_filename),
depfile_path=None,
marker='#',
defines=defines,
extra_depends={mozpath.join(finder.base, input_filename)}))
# When we re-create the build backend, we kill everything that was there.
if os.path.isdir(project_directory):
self._updated_count += 1
else:
self._created_count += 1
copier.copy(project_directory,
skip_if_older=False,
remove_unaccounted=True)
def _synchronize_docs(self, app):
m = InstallManifest()
with open(os.path.join(MAIN_DOC_PATH, "config.yml"), "r") as fh:
tree_config = yaml.safe_load(fh)["categories"]
m.add_link(self.conf_py_path, "conf.py")
for dest, source in sorted(self.trees.items()):
source_dir = os.path.join(self.topsrcdir, source)
for root, _, files in os.walk(source_dir):
for f in files:
source_path = os.path.normpath(os.path.join(root, f))
rel_source = source_path[len(source_dir) + 1:]
target = os.path.normpath(os.path.join(dest, rel_source))
if source_path.endswith(".md"):
self._process_markdown(m, source_path,
os.path.join(".", target))
else:
m.add_link(source_path, target)
copier = FileCopier()
m.populate_registry(copier)
copier.copy(self.staging_dir, remove_empty_directories=False)
with open(self.index_path, "r") as fh:
data = fh.read()
def is_toplevel(key):
"""Whether the tree is nested under the toplevel index, or is
nested under another tree's index.
"""
for k in self.trees:
if k == key:
continue
if key.startswith(k):
return False
return True
def format_paths(paths):
source_doc = ["%s/index" % p for p in paths]
return "\n ".join(source_doc)
toplevel_trees = {
k: v
for k, v in self.trees.items() if is_toplevel(k)
}
CATEGORIES = {}
# generate the datastructure to deal with the tree
for t in tree_config:
CATEGORIES[t] = format_paths(tree_config[t])
# During livereload, we don't correctly rebuild the full document
# tree (Bug 1557020). The page is no longer referenced within the index
# tree, thus we shall check categorisation only if complete tree is being rebuilt.
if app.srcdir == self.topsrcdir:
indexes = set([
os.path.normpath(os.path.join(p, "index"))
for p in toplevel_trees.keys()
])
# Format categories like indexes
cats = "\n".join(CATEGORIES.values()).split("\n")
# Remove heading spaces
cats = [os.path.normpath(x.strip()) for x in cats]
indexes = tuple(set(indexes) - set(cats))
if indexes:
# In case a new doc isn't categorized
print(indexes)
raise Exception(
"Uncategorized documentation. Please add it in docs/config.yml"
)
data = data.format(**CATEGORIES)
with open(os.path.join(self.staging_dir, "index.rst"), "w") as fh:
fh.write(data)
def _synchronize_docs(self):
m = InstallManifest()
with open(os.path.join(MAIN_DOC_PATH, 'config.yml'), 'r') as fh:
tree_config = yaml.safe_load(fh)['categories']
m.add_link(self.conf_py_path, 'conf.py')
for dest, source in sorted(self.trees.items()):
source_dir = os.path.join(self.topsrcdir, source)
for root, dirs, files in os.walk(source_dir):
for f in files:
source_path = os.path.join(root, f)
rel_source = source_path[len(source_dir) + 1:]
m.add_link(source_path, os.path.join(dest, rel_source))
copier = FileCopier()
m.populate_registry(copier)
copier.copy(self.staging_dir)
with open(self.index_path, 'rb') as fh:
data = fh.read()
def is_toplevel(key):
"""Whether the tree is nested under the toplevel index, or is
nested under another tree's index.
"""
for k in self.trees:
if k == key:
continue
if key.startswith(k):
return False
return True
def format_paths(paths):
source_doc = ['%s/index' % p for p in paths]
return '\n '.join(source_doc)
toplevel_trees = {
k: v
for k, v in self.trees.items() if is_toplevel(k)
}
CATEGORIES = {}
# generate the datastructure to deal with the tree
for t in tree_config:
CATEGORIES[t] = format_paths(tree_config[t])
indexes = set(['%s/index' % p for p in toplevel_trees.keys()])
# Format categories like indexes
cats = '\n'.join(CATEGORIES.values()).split("\n")
# Remove heading spaces
cats = [x.strip() for x in cats]
indexes = tuple(set(indexes) - set(cats))
if indexes:
# In case a new doc isn't categorized
print(indexes)
raise Exception(
"Uncategorized documentation. Please add it in tools/docs/config.yml"
)
data = data.format(**CATEGORIES)
with open(os.path.join(self.staging_dir, 'index.rst'), 'wb') as fh:
fh.write(data)
def main():
parser = ArgumentParser()
parser.add_argument('-D', dest='defines', action='append',
metavar="VAR[=VAL]", help='Define a variable')
parser.add_argument('--format', default='omni',
help='Choose the chrome format for packaging ' +
'(omni, jar or flat ; default: %(default)s)')
parser.add_argument('--removals', default=None,
help='removed-files source file')
parser.add_argument('--ignore-errors', action='store_true', default=False,
help='Transform errors into warnings.')
parser.add_argument('--minify', action='store_true', default=False,
help='Make some files more compact while packaging')
parser.add_argument('--minify-js', action='store_true',
help='Minify JavaScript files while packaging.')
parser.add_argument('--js-binary',
help='Path to js binary. This is used to verify '
'minified JavaScript. If this is not defined, '
'minification verification will not be performed.')
parser.add_argument('--jarlog', default='', help='File containing jar ' +
'access logs')
parser.add_argument('--optimizejars', action='store_true', default=False,
help='Enable jar optimizations')
parser.add_argument('--disable-compression', action='store_false',
dest='compress', default=True,
help='Disable jar compression')
parser.add_argument('manifest', default=None, nargs='?',
help='Manifest file name')
parser.add_argument('source', help='Source directory')
parser.add_argument('destination', help='Destination directory')
parser.add_argument('--non-resource', nargs='+', metavar='PATTERN',
default=[],
help='Extra files not to be considered as resources')
args = parser.parse_args()
defines = dict(buildconfig.defines)
if args.ignore_errors:
errors.ignore_errors()
if args.defines:
for name, value in [split_define(d) for d in args.defines]:
defines[name] = value
copier = FileCopier()
if args.format == 'flat':
formatter = FlatFormatter(copier)
elif args.format == 'jar':
formatter = JarFormatter(copier, compress=args.compress, optimize=args.optimizejars)
elif args.format == 'omni':
formatter = OmniJarFormatter(copier,
buildconfig.substs['OMNIJAR_NAME'],
compress=args.compress,
optimize=args.optimizejars,
non_resources=args.non_resource)
else:
errors.fatal('Unknown format: %s' % args.format)
# Adjust defines according to the requested format.
if isinstance(formatter, OmniJarFormatter):
defines['MOZ_OMNIJAR'] = 1
elif 'MOZ_OMNIJAR' in defines:
del defines['MOZ_OMNIJAR']
respath = ''
if 'RESPATH' in defines:
respath = SimpleManifestSink.normalize_path(defines['RESPATH'])
while respath.startswith('/'):
respath = respath[1:]
if not buildconfig.substs['CROSS_COMPILE']:
launcher.tooldir = mozpath.join(buildconfig.topobjdir, 'dist')
with errors.accumulate():
finder_args = dict(
minify=args.minify,
minify_js=args.minify_js,
)
if args.js_binary:
finder_args['minify_js_verify_command'] = [
args.js_binary,
os.path.join(os.path.abspath(os.path.dirname(__file__)),
'js-compare-ast.js')
]
finder = FileFinder(args.source, find_executables=True,
**finder_args)
if 'NO_PKG_FILES' in os.environ:
sinkformatter = NoPkgFilesRemover(formatter,
args.manifest is not None)
else:
sinkformatter = formatter
sink = SimpleManifestSink(finder, sinkformatter)
if args.manifest:
preprocess_manifest(sink, args.manifest, defines)
else:
sink.add(Component(''), 'bin/*')
sink.close(args.manifest is not None)
if args.removals:
removals_in = StringIO(open(args.removals).read())
removals_in.name = args.removals
removals = RemovedFiles(copier)
preprocess(removals_in, removals, defines)
copier.add(mozpath.join(respath, 'removed-files'), removals)
# shlibsign libraries
if launcher.can_launch():
if not mozinfo.isMac and buildconfig.substs.get('COMPILE_ENVIRONMENT'):
for lib in SIGN_LIBS:
libbase = mozpath.join(respath, '%s%s') \
% (buildconfig.substs['DLL_PREFIX'], lib)
libname = '%s%s' % (libbase, buildconfig.substs['DLL_SUFFIX'])
if copier.contains(libname):
copier.add(libbase + '.chk',
LibSignFile(os.path.join(args.destination,
libname)))
# Setup preloading
if args.jarlog and os.path.exists(args.jarlog):
from mozpack.mozjar import JarLog
log = JarLog(args.jarlog)
for p, f in copier:
if not isinstance(f, Jarrer):
continue
key = JarLog.canonicalize(os.path.join(args.destination, p))
if key in log:
f.preload(log[key])
copier.copy(args.destination)
generate_precomplete(os.path.normpath(os.path.join(args.destination,
respath)))
