def test_error_loop(self):
with self.assertRaises(AccumulatedErrors):
with errors.accumulate():
for i in range(3):
errors.error('%d' % i)
self.assertEquals(self.get_output(),
['Error: 0', 'Error: 1', 'Error: 2'])
def main():
parser = argparse.ArgumentParser(
description="Merge two directories, creating Universal binaries for "
"executables and libraries they contain.")
parser.add_argument("dir1", help="Directory")
parser.add_argument("dir2", help="Directory to merge")
options = parser.parse_args()
buildconfig.substs["OS_ARCH"] = "Darwin"
buildconfig.substs["LIPO"] = os.environ.get("LIPO")
dir1_finder = FileFinder(options.dir1,
find_executables=True,
find_dotfiles=True)
dir2_finder = FileFinder(options.dir2,
find_executables=True,
find_dotfiles=True)
finder = UnifiedTestFinder(dir1_finder, dir2_finder)
copier = FileCopier()
with errors.accumulate():
for p, f in finder:
copier.add(p, f)
copier.copy(options.dir1, skip_if_older=False)
def test_error_loop(self):
with self.assertRaises(AccumulatedErrors):
with errors.accumulate():
for i in range(3):
errors.error("%d" % i)
self.assertEquals(self.get_output(),
["Error: 0", "Error: 1", "Error: 2"])
def test_unified_build_finder(self):
self.create_both('chrome.manifest', 'a\nb\nc\n')
self.create_one('a', 'chrome/chrome.manifest', 'a\nb\nc\n')
self.create_one('b', 'chrome/chrome.manifest', 'b\nc\na\n')
self.create_one('a', 'chrome/browser/foo/buildconfig.html',
'\n'.join([
'<html>',
'<body>',
'<h1>about:buildconfig</h1>',
'<div>foo</div>',
'</body>',
'</html>',
]))
self.create_one('b', 'chrome/browser/foo/buildconfig.html',
'\n'.join([
'<html>',
'<body>',
'<h1>about:buildconfig</h1>',
'<div>bar</div>',
'</body>',
'</html>',
]))
finder = UnifiedBuildFinder(FileFinder(self.tmppath('a')),
FileFinder(self.tmppath('b')))
self.assertEqual(sorted([(f, c.open().read()) for f, c in
finder.find('**/chrome.manifest')]),
[('chrome.manifest', 'a\nb\nc\n'),
('chrome/chrome.manifest', 'a\nb\nc\n')])
self.assertEqual(sorted([(f, c.open().read()) for f, c in
finder.find('**/buildconfig.html')]),
[('chrome/browser/foo/buildconfig.html', '\n'.join([
'<html>',
'<body>',
'<h1>about:buildconfig</h1>',
'<div>foo</div>',
'<hr> </hr>',
'<div>bar</div>',
'</body>',
'</html>',
]))])
xpi = MockDest()
with JarWriter(fileobj=xpi, compress=True) as jar:
jar.add('foo', 'foo')
jar.add('bar', 'bar')
foo_xpi = xpi.read()
self.create_both('foo.xpi', foo_xpi)
with JarWriter(fileobj=xpi, compress=True) as jar:
jar.add('foo', 'bar')
self.create_one('a', 'bar.xpi', foo_xpi)
self.create_one('b', 'bar.xpi', xpi.read())
errors.out = StringIO()
with self.assertRaises(AccumulatedErrors), errors.accumulate():
self.assertEqual([(f, c.open().read()) for f, c in
finder.find('*.xpi')],
[('foo.xpi', foo_xpi)])
errors.out = sys.stderr
def main(args):
parser = argparse.ArgumentParser()
parser.add_argument("-C",
metavar='DIR',
default=".",
help="Change to given directory before considering "
"other paths")
parser.add_argument("--strip",
action='store_true',
help="Strip executables")
parser.add_argument("-x",
metavar='EXCLUDE',
default=[],
action='append',
help="Exclude files that match the pattern")
parser.add_argument("zip", help="Path to zip file to write")
parser.add_argument("input", nargs="+", help="Path to files to add to zip")
args = parser.parse_args(args)
jarrer = Jarrer()
with errors.accumulate():
finder = FileFinder(args.C, find_executables=args.strip)
for path in args.input:
for p, f in finder.find(path):
if not any([match(p, exclude) for exclude in args.x]):
jarrer.add(p, f)
jarrer.copy(mozpath.join(args.C, args.zip))
def test_multiple_errors(self):
with self.assertRaises(AccumulatedErrors):
with errors.accumulate():
errors.error('foo')
for i in range(3):
if i == 2:
errors.warn('%d' % i)
else:
errors.error('%d' % i)
errors.error('bar')
self.assertEquals(self.get_output(),
['Error: foo', 'Error: 0', 'Error: 1',
'Warning: 2', 'Error: bar'])
def test_multiple_errors(self):
with self.assertRaises(AccumulatedErrors):
with errors.accumulate():
errors.error("foo")
for i in range(3):
if i == 2:
errors.warn("%d" % i)
else:
errors.error("%d" % i)
errors.error("bar")
self.assertEquals(
self.get_output(),
["Error: foo", "Error: 0", "Error: 1", "Warning: 2", "Error: bar"],
)
def main():
parser = argparse.ArgumentParser(
description=
"Merge two builds of a Gecko-based application into a Universal build")
parser.add_argument("app1", help="Directory containing the application")
parser.add_argument("app2",
help="Directory containing the application to merge")
parser.add_argument(
"--non-resource",
nargs="+",
metavar="PATTERN",
default=[],
help="Extra files not to be considered as resources",
)
options = parser.parse_args()
buildconfig.substs["OS_ARCH"] = "Darwin"
buildconfig.substs["LIPO"] = os.environ.get("LIPO")
app1_finder = UnpackFinder(FileFinder(options.app1, find_executables=True))
app2_finder = UnpackFinder(FileFinder(options.app2, find_executables=True))
app_finder = UnifiedBuildFinderWasmHack(app1_finder, app2_finder)
copier = FileCopier()
compress = min(app1_finder.compressed, JAR_DEFLATED)
if app1_finder.kind == "flat":
formatter = FlatFormatter(copier)
elif app1_finder.kind == "jar":
formatter = JarFormatter(copier, compress=compress)
elif app1_finder.kind == "omni":
formatter = OmniJarFormatter(
copier,
app1_finder.omnijar,
compress=compress,
non_resources=options.non_resource,
)
with errors.accumulate():
packager = SimplePackager(formatter)
for p, f in app_finder:
packager.add(p, f)
packager.close()
# Transplant jar preloading information.
for path, log in six.iteritems(app1_finder.jarlogs):
assert isinstance(copier[path], Jarrer)
copier[path].preload(log)
copier.copy(options.app1, skip_if_older=False)
def repack(source, l10n, extra_l10n={}, non_resources=[], non_chrome=set()):
'''
Replace localized data from the `source` directory with localized data
from `l10n` and `extra_l10n`.
The `source` argument points to a directory containing a packaged
application (in omnijar, jar or flat form).
The `l10n` argument points to a directory containing the main localized
data (usually in the form of a language pack addon) to use to replace
in the packaged application.
The `extra_l10n` argument contains a dict associating relative paths in
the source to separate directories containing localized data for them.
This can be used to point at different language pack addons for different
parts of the package application.
The `non_resources` argument gives a list of relative paths in the source
that should not be added in an omnijar in case the packaged application
is in that format.
The `non_chrome` argument gives a list of file/directory patterns for
localized files that are not listed in a chrome.manifest.
'''
app_finder = UnpackFinder(source)
l10n_finder = UnpackFinder(l10n)
if extra_l10n:
finders = {
'': l10n_finder,
}
for base, path in extra_l10n.iteritems():
finders[base] = UnpackFinder(path)
l10n_finder = ComposedFinder(finders)
copier = FileCopier()
compress = min(app_finder.compressed, JAR_DEFLATED)
if app_finder.kind == 'flat':
formatter = FlatFormatter(copier)
elif app_finder.kind == 'jar':
formatter = JarFormatter(copier,
optimize=app_finder.optimizedjars,
compress=compress)
elif app_finder.kind == 'omni':
formatter = OmniJarFormatter(copier,
app_finder.omnijar,
optimize=app_finder.optimizedjars,
compress=compress,
non_resources=non_resources)
with errors.accumulate():
_repack(app_finder, l10n_finder, copier, formatter, non_chrome)
copier.copy(source, skip_if_older=False)
generate_precomplete(source)
def repack(source, l10n, extra_l10n={}, non_resources=[], non_chrome=set()):
'''
Replace localized data from the `source` directory with localized data
from `l10n` and `extra_l10n`.
The `source` argument points to a directory containing a packaged
application (in omnijar, jar or flat form).
The `l10n` argument points to a directory containing the main localized
data (usually in the form of a language pack addon) to use to replace
in the packaged application.
The `extra_l10n` argument contains a dict associating relative paths in
the source to separate directories containing localized data for them.
This can be used to point at different language pack addons for different
parts of the package application.
The `non_resources` argument gives a list of relative paths in the source
that should not be added in an omnijar in case the packaged application
is in that format.
The `non_chrome` argument gives a list of file/directory patterns for
localized files that are not listed in a chrome.manifest.
'''
app_finder = UnpackFinder(source)
l10n_finder = UnpackFinder(l10n)
if extra_l10n:
finders = {
'': l10n_finder,
}
for base, path in extra_l10n.iteritems():
finders[base] = UnpackFinder(path)
l10n_finder = ComposedFinder(finders)
copier = FileCopier()
compress = min(app_finder.compressed, JAR_DEFLATED)
if app_finder.kind == 'flat':
formatter = FlatFormatter(copier)
elif app_finder.kind == 'jar':
formatter = JarFormatter(copier,
optimize=app_finder.optimizedjars,
compress=compress)
elif app_finder.kind == 'omni':
formatter = OmniJarFormatter(copier, app_finder.omnijar,
optimize=app_finder.optimizedjars,
compress=compress,
non_resources=non_resources)
with errors.accumulate():
_repack(app_finder, l10n_finder, copier, formatter, non_chrome)
copier.copy(source, skip_if_older=False)
generate_precomplete(source)
def test_errors_context(self):
with self.assertRaises(AccumulatedErrors):
with errors.accumulate():
self.assertEqual(errors.get_context(), None)
with errors.context('foo', 1):
self.assertEqual(errors.get_context(), ('foo', 1))
errors.error('a')
with errors.context('bar', 2):
self.assertEqual(errors.get_context(), ('bar', 2))
errors.error('b')
self.assertEqual(errors.get_context(), ('foo', 1))
errors.error('c')
self.assertEqual(self.get_output(), [
'Error: foo:1: a',
'Error: bar:2: b',
'Error: foo:1: c',
])
def repack(source, l10n, non_resources=[], non_chrome=set()):
app_finder = UnpackFinder(source)
l10n_finder = UnpackFinder(l10n)
copier = FileCopier()
if app_finder.kind == 'flat':
formatter = FlatFormatter(copier)
elif app_finder.kind == 'jar':
formatter = JarFormatter(copier, optimize=app_finder.optimizedjars)
elif app_finder.kind == 'omni':
formatter = OmniJarFormatter(copier, app_finder.omnijar,
optimize=app_finder.optimizedjars,
non_resources=non_resources)
with errors.accumulate():
_repack(app_finder, l10n_finder, copier, formatter, non_chrome)
copier.copy(source, skip_if_older=False)
generate_precomplete(source)
def main(args):
parser = argparse.ArgumentParser()
parser.add_argument("-C", metavar='DIR', default=".",
help="Change to given directory before considering "
"other paths")
parser.add_argument("zip", help="Path to zip file to write")
parser.add_argument("input", nargs="+",
help="Path to files to add to zip")
args = parser.parse_args(args)
jarrer = Jarrer(optimize=False)
with errors.accumulate():
finder = FileFinder(args.C)
for path in args.input:
for p, f in finder.find(path):
jarrer.add(p, f)
jarrer.copy(mozpath.join(args.C, args.zip))
def main(args):
parser = argparse.ArgumentParser()
parser.add_argument("--base-dir",
default=os.path.join(buildconfig.topobjdir, "dist",
"bin"),
help="Store paths relative to this directory")
parser.add_argument("zip", help="Path to zip file to write")
parser.add_argument("input", nargs="+", help="Path to files to add to zip")
args = parser.parse_args(args)
jarrer = Jarrer(optimize=False)
with errors.accumulate():
finder = FileFinder(args.base_dir)
for i in args.input:
path = mozpath.relpath(i, args.base_dir)
for p, f in finder.find(path):
jarrer.add(p, f)
jarrer.copy(args.zip)
def test_parse_manifest_errors(self):
manifest = [
"skin global classic/1.0 content/skin/classic/ platform",
"",
"binary-component bar.so",
"unsupported foo",
]
with mozunit.MockedOpen({"manifest": "\n".join(manifest)}):
with self.assertRaises(AccumulatedErrors):
with errors.accumulate():
list(parse_manifest(os.curdir, "manifest"))
out = self.get_output()
# Expecting 2 errors
self.assertEqual(len(out), 2)
path = os.path.abspath("manifest")
# First on line 1
self.assertTrue(out[0].startswith("Error: %s:1: " % path))
# Second on line 4
self.assertTrue(out[1].startswith("Error: %s:4: " % path))
def test_parse_manifest_errors(self):
manifest = [
'skin global classic/1.0 content/skin/classic/ platform',
'',
'binary-component bar.so',
'unsupported foo',
]
with mozunit.MockedOpen({'manifest': '\n'.join(manifest)}):
with self.assertRaises(AccumulatedErrors):
with errors.accumulate():
list(parse_manifest(os.curdir, 'manifest'))
out = self.get_output()
# Expecting 2 errors
self.assertEqual(len(out), 2)
path = os.path.abspath('manifest')
# First on line 1
self.assertTrue(out[0].startswith('Error: %s:1: ' % path))
# Second on line 4
self.assertTrue(out[1].startswith('Error: %s:4: ' % path))
def main():
parser = argparse.ArgumentParser(
description="Merge two crashreporter symbols directories."
)
parser.add_argument("dir1", help="Directory")
parser.add_argument("dir2", help="Directory to merge")
options = parser.parse_args()
dir1_finder = FileFinder(options.dir1)
dir2_finder = FileFinder(options.dir2)
finder = UnifiedSymbolsFinder(dir1_finder, dir2_finder)
copier = FileCopier()
with errors.accumulate():
for p, f in finder:
copier.add(p, f)
copier.copy(options.dir1, skip_if_older=False)
def test_parse_manifest_errors(self):
manifest = [
'skin global classic/1.0 content/skin/classic/ platform',
'',
'binary-component bar.so',
'unsupported foo',
]
with mozunit.MockedOpen({'manifest': '\n'.join(manifest)}):
with self.assertRaises(AccumulatedErrors):
with errors.accumulate():
list(parse_manifest(os.curdir, 'manifest'))
out = self.get_output()
# Expecting 2 errors
self.assertEqual(len(out), 2)
path = os.path.abspath('manifest')
# First on line 1
self.assertTrue(out[0].startswith('Error: %s:1: ' % path))
# Second on line 4
self.assertTrue(out[1].startswith('Error: %s:4: ' % path))
def test_errors_context(self):
with self.assertRaises(AccumulatedErrors):
with errors.accumulate():
self.assertEqual(errors.get_context(), None)
with errors.context("foo", 1):
self.assertEqual(errors.get_context(), ("foo", 1))
errors.error("a")
with errors.context("bar", 2):
self.assertEqual(errors.get_context(), ("bar", 2))
errors.error("b")
self.assertEqual(errors.get_context(), ("foo", 1))
errors.error("c")
self.assertEqual(
self.get_output(),
[
"Error: foo:1: a",
"Error: bar:2: b",
"Error: foo:1: c",
],
)
def main(args):
parser = argparse.ArgumentParser()
parser.add_argument("-C", metavar='DIR', default=".",
help="Change to given directory before considering "
"other paths")
parser.add_argument("--strip", action='store_true',
help="Strip executables")
parser.add_argument("-x", metavar='EXCLUDE', default=[], action='append',
help="Exclude files that match the pattern")
parser.add_argument("zip", help="Path to zip file to write")
parser.add_argument("input", nargs="+",
help="Path to files to add to zip")
args = parser.parse_args(args)
jarrer = Jarrer(optimize=False)
with errors.accumulate():
finder = FileFinder(args.C, find_executables=args.strip)
for path in args.input:
for p, f in finder.find(path):
if not any([match(p, exclude) for exclude in args.x]):
jarrer.add(p, f)
jarrer.copy(mozpath.join(args.C, args.zip))
def test_unified_build_finder(self):
self.create_both('chrome.manifest', 'a\nb\nc\n')
self.create_one('a', 'chrome/chrome.manifest', 'a\nb\nc\n')
self.create_one('b', 'chrome/chrome.manifest', 'b\nc\na\n')
self.create_one(
'a', 'chrome/browser/foo/buildconfig.html', '\n'.join([
'<html>',
'<body>',
'<h1>about:buildconfig</h1>',
'<div>foo</div>',
'</body>',
'</html>',
]))
self.create_one(
'b', 'chrome/browser/foo/buildconfig.html', '\n'.join([
'<html>',
'<body>',
'<h1>about:buildconfig</h1>',
'<div>bar</div>',
'</body>',
'</html>',
]))
finder = UnifiedBuildFinder(FileFinder(self.tmppath('a')),
FileFinder(self.tmppath('b')))
self.assertEqual(
sorted([(f, c.open().read())
for f, c in finder.find('**/chrome.manifest')]),
[('chrome.manifest', 'a\nb\nc\n'),
('chrome/chrome.manifest', 'a\nb\nc\n')])
self.assertEqual(
sorted([(f, c.open().read())
for f, c in finder.find('**/buildconfig.html')]),
[('chrome/browser/foo/buildconfig.html', '\n'.join([
'<html>',
'<body>',
'<h1>about:buildconfig</h1>',
'<div>foo</div>',
'<hr> </hr>',
'<div>bar</div>',
'</body>',
'</html>',
]))])
xpi = MockDest()
with JarWriter(fileobj=xpi, compress=True) as jar:
jar.add('foo', 'foo')
jar.add('bar', 'bar')
foo_xpi = xpi.read()
self.create_both('foo.xpi', foo_xpi)
with JarWriter(fileobj=xpi, compress=True) as jar:
jar.add('foo', 'bar')
self.create_one('a', 'bar.xpi', foo_xpi)
self.create_one('b', 'bar.xpi', xpi.read())
errors.out = StringIO()
with self.assertRaises(AccumulatedErrors), errors.accumulate():
self.assertEqual([(f, c.open().read())
for f, c in finder.find('*.xpi')],
[('foo.xpi', foo_xpi)])
errors.out = sys.stderr
def main():
parser = ArgumentParser()
parser.add_argument('-D',
dest='defines',
action='append',
metavar="VAR[=VAL]",
help='Define a variable')
parser.add_argument('--format',
default='omni',
help='Choose the chrome format for packaging ' +
'(omni, jar or flat ; default: %(default)s)')
parser.add_argument('--removals',
default=None,
help='removed-files source file')
parser.add_argument('--ignore-errors',
action='store_true',
default=False,
help='Transform errors into warnings.')
parser.add_argument('--minify',
action='store_true',
default=False,
help='Make some files more compact while packaging')
parser.add_argument('--minify-js',
action='store_true',
help='Minify JavaScript files while packaging.')
parser.add_argument('--js-binary',
help='Path to js binary. This is used to verify '
'minified JavaScript. If this is not defined, '
'minification verification will not be performed.')
parser.add_argument('--jarlog',
default='',
help='File containing jar ' + 'access logs')
parser.add_argument('--optimizejars',
action='store_true',
default=False,
help='Enable jar optimizations')
parser.add_argument('--unify',
default='',
help='Base directory of another build to unify with')
parser.add_argument('manifest',
default=None,
nargs='?',
help='Manifest file name')
parser.add_argument('source', help='Source directory')
parser.add_argument('destination', help='Destination directory')
parser.add_argument('--non-resource',
nargs='+',
metavar='PATTERN',
default=[],
help='Extra files not to be considered as resources')
args = parser.parse_args()
defines = dict(buildconfig.defines)
if args.ignore_errors:
errors.ignore_errors()
if args.defines:
for name, value in [split_define(d) for d in args.defines]:
defines[name] = value
copier = FileCopier()
if args.format == 'flat':
formatter = FlatFormatter(copier)
elif args.format == 'jar':
formatter = JarFormatter(copier, optimize=args.optimizejars)
elif args.format == 'omni':
formatter = OmniJarFormatter(copier,
buildconfig.substs['OMNIJAR_NAME'],
optimize=args.optimizejars,
non_resources=args.non_resource)
else:
errors.fatal('Unknown format: %s' % args.format)
# Adjust defines according to the requested format.
if isinstance(formatter, OmniJarFormatter):
defines['MOZ_OMNIJAR'] = 1
elif 'MOZ_OMNIJAR' in defines:
del defines['MOZ_OMNIJAR']
binpath = ''
if 'BINPATH' in defines:
binpath = SimpleManifestSink.normalize_path(defines['BINPATH'])
while binpath.startswith('/'):
binpath = binpath[1:]
if args.unify:
def is_native(path):
path = os.path.abspath(path)
return platform.machine() in mozpack.path.split(path)
# Invert args.unify and args.source if args.unify points to the
# native architecture.
args.source, args.unify = sorted([args.source, args.unify],
key=is_native,
reverse=True)
if is_native(args.source):
launcher.tooldir = args.source
elif not buildconfig.substs['CROSS_COMPILE']:
launcher.tooldir = buildconfig.substs['LIBXUL_DIST']
with errors.accumulate():
finder_args = dict(
minify=args.minify,
minify_js=args.minify_js,
)
if args.js_binary:
finder_args['minify_js_verify_command'] = [
args.js_binary,
os.path.join(os.path.abspath(os.path.dirname(__file__)),
'js-compare-ast.js')
]
if args.unify:
finder = UnifiedBuildFinder(FileFinder(args.source),
FileFinder(args.unify), **finder_args)
else:
finder = FileFinder(args.source, **finder_args)
if 'NO_PKG_FILES' in os.environ:
sinkformatter = NoPkgFilesRemover(formatter, args.manifest
is not None)
else:
sinkformatter = formatter
sink = SimpleManifestSink(finder, sinkformatter)
if args.manifest:
preprocess_manifest(sink, args.manifest, defines)
else:
sink.add(Component(''), 'bin/*')
sink.close(args.manifest is not None)
if args.removals:
lines = [l.lstrip() for l in open(args.removals).readlines()]
removals_in = StringIO(''.join(lines))
removals_in.name = args.removals
removals = RemovedFiles(copier)
preprocess(removals_in, removals, defines)
copier.add(mozpack.path.join(binpath, 'removed-files'), removals)
# shlibsign libraries
if launcher.can_launch():
for lib in SIGN_LIBS:
libbase = mozpack.path.join(binpath, '%s%s') \
% (buildconfig.substs['DLL_PREFIX'], lib)
libname = '%s%s' % (libbase, buildconfig.substs['DLL_SUFFIX'])
if copier.contains(libname):
copier.add(
libbase + '.chk',
LibSignFile(os.path.join(args.destination, libname)))
# Setup preloading
if args.jarlog and os.path.exists(args.jarlog):
from mozpack.mozjar import JarLog
log = JarLog(args.jarlog)
for p, f in copier:
if not isinstance(f, Jarrer):
continue
key = JarLog.canonicalize(os.path.join(args.destination, p))
if key in log:
f.preload(log[key])
# Fill startup cache
if isinstance(formatter, OmniJarFormatter) and launcher.can_launch() \
and buildconfig.substs['MOZ_DISABLE_STARTUPCACHE'] != '1':
if buildconfig.substs['LIBXUL_SDK']:
gre_path = mozpack.path.join(buildconfig.substs['LIBXUL_DIST'],
'bin')
else:
gre_path = None
for base in sorted([[
p for p in [mozpack.path.join('bin', b), b]
if os.path.exists(os.path.join(args.source, p))
][0] for b in sink.packager.get_bases()]):
if not gre_path:
gre_path = base
base_path = sink.normalize_path(base)
if base_path in formatter.omnijars:
precompile_cache(formatter.omnijars[base_path], args.source,
gre_path, base)
copier.copy(args.destination)
generate_precomplete(
os.path.normpath(os.path.join(args.destination, binpath)))
def main():
parser = ArgumentParser()
parser.add_argument(
"-D",
dest="defines",
action="append",
metavar="VAR[=VAL]",
help="Define a variable",
)
parser.add_argument(
"--format",
default="omni",
help="Choose the chrome format for packaging " +
"(omni, jar or flat ; default: %(default)s)",
)
parser.add_argument("--removals",
default=None,
help="removed-files source file")
parser.add_argument(
"--ignore-errors",
action="store_true",
default=False,
help="Transform errors into warnings.",
)
parser.add_argument(
"--ignore-broken-symlinks",
action="store_true",
default=False,
help="Do not fail when processing broken symlinks.",
)
parser.add_argument(
"--minify",
action="store_true",
default=False,
help="Make some files more compact while packaging",
)
parser.add_argument(
"--minify-js",
action="store_true",
help="Minify JavaScript files while packaging.",
)
parser.add_argument(
"--js-binary",
help="Path to js binary. This is used to verify "
"minified JavaScript. If this is not defined, "
"minification verification will not be performed.",
)
parser.add_argument("--jarlog",
default="",
help="File containing jar " + "access logs")
parser.add_argument(
"--compress",
choices=("none", "deflate"),
default="deflate",
help="Use given jar compression (default: deflate)",
)
parser.add_argument("manifest",
default=None,
nargs="?",
help="Manifest file name")
parser.add_argument("source", help="Source directory")
parser.add_argument("destination", help="Destination directory")
parser.add_argument(
"--non-resource",
nargs="+",
metavar="PATTERN",
default=[],
help="Extra files not to be considered as resources",
)
args = parser.parse_args()
defines = dict(buildconfig.defines["ALLDEFINES"])
if args.ignore_errors:
errors.ignore_errors()
if args.defines:
for name, value in [split_define(d) for d in args.defines]:
defines[name] = value
compress = {
"none": False,
"deflate": True,
}[args.compress]
copier = FileCopier()
if args.format == "flat":
formatter = FlatFormatter(copier)
elif args.format == "jar":
formatter = JarFormatter(copier, compress=compress)
elif args.format == "omni":
formatter = OmniJarFormatter(
copier,
buildconfig.substs["OMNIJAR_NAME"],
compress=compress,
non_resources=args.non_resource,
)
else:
errors.fatal("Unknown format: %s" % args.format)
# Adjust defines according to the requested format.
if isinstance(formatter, OmniJarFormatter):
defines["MOZ_OMNIJAR"] = 1
elif "MOZ_OMNIJAR" in defines:
del defines["MOZ_OMNIJAR"]
respath = ""
if "RESPATH" in defines:
respath = SimpleManifestSink.normalize_path(defines["RESPATH"])
while respath.startswith("/"):
respath = respath[1:]
with errors.accumulate():
finder_args = dict(
minify=args.minify,
minify_js=args.minify_js,
ignore_broken_symlinks=args.ignore_broken_symlinks,
)
if args.js_binary:
finder_args["minify_js_verify_command"] = [
args.js_binary,
os.path.join(os.path.abspath(os.path.dirname(__file__)),
"js-compare-ast.js"),
]
finder = PackagerFileFinder(args.source,
find_executables=True,
**finder_args)
if "NO_PKG_FILES" in os.environ:
sinkformatter = NoPkgFilesRemover(formatter, args.manifest
is not None)
else:
sinkformatter = formatter
sink = SimpleManifestSink(finder, sinkformatter)
if args.manifest:
preprocess_manifest(sink, args.manifest, defines)
else:
sink.add(Component(""), "bin/*")
sink.close(args.manifest is not None)
if args.removals:
removals_in = StringIO(open(args.removals).read())
removals_in.name = args.removals
removals = RemovedFiles(copier)
preprocess(removals_in, removals, defines)
copier.add(mozpath.join(respath, "removed-files"), removals)
# If a pdb file is present and we were instructed to copy it, include it.
# Run on all OSes to capture MinGW builds
if buildconfig.substs.get("MOZ_COPY_PDBS"):
# We want to mutate the copier while we're iterating through it, so copy
# the items to a list first.
copier_items = [(p, f) for p, f in copier]
for p, f in copier_items:
if isinstance(f, ExecutableFile):
pdbname = os.path.splitext(f.inputs()[0])[0] + ".pdb"
if os.path.exists(pdbname):
copier.add(os.path.basename(pdbname), File(pdbname))
# Setup preloading
if args.jarlog:
if not os.path.exists(args.jarlog):
raise Exception("Cannot find jar log: %s" % args.jarlog)
omnijars = []
if isinstance(formatter, OmniJarFormatter):
omnijars = [
mozpath.join(base, buildconfig.substs["OMNIJAR_NAME"])
for base in sink.packager.get_bases(addons=False)
]
from mozpack.mozjar import JarLog
log = JarLog(args.jarlog)
for p, f in copier:
if not isinstance(f, Jarrer):
continue
if respath:
p = mozpath.relpath(p, respath)
if p in log:
f.preload(log[p])
elif p in omnijars:
raise Exception("No jar log data for %s" % p)
copier.copy(args.destination)
generate_precomplete(
os.path.normpath(os.path.join(args.destination, respath)))
def test_unified_build_finder(self):
finder = UnifiedBuildFinder(FileFinder(self.tmppath('a')),
FileFinder(self.tmppath('b')))
# Test chrome.manifest unification
self.create_both('chrome.manifest', 'a\nb\nc\n')
self.create_one('a', 'chrome/chrome.manifest', 'a\nb\nc\n')
self.create_one('b', 'chrome/chrome.manifest', 'b\nc\na\n')
self.assertEqual(
sorted([(f, c.open().read())
for f, c in finder.find('**/chrome.manifest')]),
[('chrome.manifest', 'a\nb\nc\n'),
('chrome/chrome.manifest', 'a\nb\nc\n')])
# Test buildconfig.html unification
self.create_one(
'a', 'chrome/browser/foo/buildconfig.html', '\n'.join([
'<html>',
'<body>',
'<h1>about:buildconfig</h1>',
'<div>foo</div>',
'</body>',
'</html>',
]))
self.create_one(
'b', 'chrome/browser/foo/buildconfig.html', '\n'.join([
'<html>',
'<body>',
'<h1>about:buildconfig</h1>',
'<div>bar</div>',
'</body>',
'</html>',
]))
self.assertEqual(
sorted([(f, c.open().read())
for f, c in finder.find('**/buildconfig.html')]),
[('chrome/browser/foo/buildconfig.html', '\n'.join([
'<html>',
'<body>',
'<h1>about:buildconfig</h1>',
'<div>foo</div>',
'<hr> </hr>',
'<div>bar</div>',
'</body>',
'</html>',
]))])
# Test xpi file unification
xpi = MockDest()
with JarWriter(fileobj=xpi, compress=True) as jar:
jar.add('foo', 'foo')
jar.add('bar', 'bar')
foo_xpi = xpi.read()
self.create_both('foo.xpi', foo_xpi)
with JarWriter(fileobj=xpi, compress=True) as jar:
jar.add('foo', 'bar')
self.create_one('a', 'bar.xpi', foo_xpi)
self.create_one('b', 'bar.xpi', xpi.read())
errors.out = StringIO()
with self.assertRaises(AccumulatedErrors), errors.accumulate():
self.assertEqual([(f, c.open().read())
for f, c in finder.find('*.xpi')],
[('foo.xpi', foo_xpi)])
errors.out = sys.stderr
# Test install.rdf unification
x86_64 = 'Darwin_x86_64-gcc3'
x86 = 'Darwin_x86-gcc3'
target_tag = '<{em}targetPlatform>{platform}</{em}targetPlatform>'
target_attr = '{em}targetPlatform="{platform}" '
rdf_tag = ''.join([
'<{RDF}Description {em}bar="bar" {em}qux="qux">',
'<{em}foo>foo</{em}foo>', '{targets}', '<{em}baz>baz</{em}baz>',
'</{RDF}Description>'
])
rdf_attr = ''.join([
'<{RDF}Description {em}bar="bar" {attr}{em}qux="qux">',
'{targets}', '<{em}foo>foo</{em}foo><{em}baz>baz</{em}baz>',
'</{RDF}Description>'
])
for descr_ns, target_ns in (('RDF:', ''), ('', 'em:'), ('RDF:',
'em:')):
# First we need to infuse the above strings with our namespaces and
# platform values.
ns = {'RDF': descr_ns, 'em': target_ns}
target_tag_x86_64 = target_tag.format(platform=x86_64, **ns)
target_tag_x86 = target_tag.format(platform=x86, **ns)
target_attr_x86_64 = target_attr.format(platform=x86_64, **ns)
target_attr_x86 = target_attr.format(platform=x86, **ns)
tag_x86_64 = rdf_tag.format(targets=target_tag_x86_64, **ns)
tag_x86 = rdf_tag.format(targets=target_tag_x86, **ns)
tag_merged = rdf_tag.format(targets=target_tag_x86_64 +
target_tag_x86,
**ns)
tag_empty = rdf_tag.format(targets="", **ns)
attr_x86_64 = rdf_attr.format(attr=target_attr_x86_64,
targets="",
**ns)
attr_x86 = rdf_attr.format(attr=target_attr_x86, targets="", **ns)
attr_merged = rdf_attr.format(attr="",
targets=target_tag_x86_64 +
target_tag_x86,
**ns)
# This table defines the test cases, columns "a" and "b" being the
# contents of the install.rdf of the respective platform and
# "result" the exepected merged content after unification.
testcases = (
#_____a_____ _____b_____ ___result___#
(tag_x86_64, tag_x86, tag_merged),
(tag_x86_64, tag_empty, tag_empty),
(tag_empty, tag_x86, tag_empty),
(tag_empty, tag_empty, tag_empty),
(attr_x86_64, attr_x86, attr_merged),
(tag_x86_64, attr_x86, tag_merged),
(attr_x86_64, tag_x86, attr_merged),
(attr_x86_64, tag_empty, tag_empty),
(tag_empty, attr_x86, tag_empty))
# Now create the files from the above table and compare
results = []
for emid, (rdf_a, rdf_b, result) in enumerate(testcases):
filename = 'ext/id{0}/install.rdf'.format(emid)
self.create_one('a', filename, rdf_a)
self.create_one('b', filename, rdf_b)
results.append((filename, result))
self.assertEqual(
sorted([(f, c.open().read())
for f, c in finder.find('**/install.rdf')]), results)
def test_unified_build_finder(self):
finder = UnifiedBuildFinder(FileFinder(self.tmppath("a")),
FileFinder(self.tmppath("b")))
# Test chrome.manifest unification
self.create_both("chrome.manifest", "a\nb\nc\n")
self.create_one("a", "chrome/chrome.manifest", "a\nb\nc\n")
self.create_one("b", "chrome/chrome.manifest", "b\nc\na\n")
self.assertEqual(
sorted([(f, c.open().read().decode("utf-8"))
for f, c in finder.find("**/chrome.manifest")]),
[("chrome.manifest", "a\nb\nc\n"),
("chrome/chrome.manifest", "a\nb\nc\n")],
)
# Test buildconfig.html unification
self.create_one(
"a",
"chrome/browser/foo/buildconfig.html",
"\n".join([
"<html>",
" <body>",
" <div>",
" <h1>Build Configuration</h1>",
" <div>foo</div>",
" </div>",
" </body>",
"</html>",
]),
)
self.create_one(
"b",
"chrome/browser/foo/buildconfig.html",
"\n".join([
"<html>",
" <body>",
" <div>",
" <h1>Build Configuration</h1>",
" <div>bar</div>",
" </div>",
" </body>",
"</html>",
]),
)
self.assertEqual(
sorted([(f, c.open().read().decode("utf-8"))
for f, c in finder.find("**/buildconfig.html")]),
[(
"chrome/browser/foo/buildconfig.html",
"\n".join([
"<html>",
" <body>",
" <div>",
" <h1>Build Configuration</h1>",
" <div>foo</div>",
" <hr> </hr>",
" <div>bar</div>",
" </div>",
" </body>",
"</html>",
]),
)],
)
# Test xpi file unification
xpi = MockDest()
with JarWriter(fileobj=xpi, compress=True) as jar:
jar.add("foo", "foo")
jar.add("bar", "bar")
foo_xpi = xpi.read()
self.create_both("foo.xpi", foo_xpi)
with JarWriter(fileobj=xpi, compress=True) as jar:
jar.add("foo", "bar")
self.create_one("a", "bar.xpi", foo_xpi)
self.create_one("b", "bar.xpi", xpi.read())
errors.out = StringIO()
with self.assertRaises(AccumulatedErrors), errors.accumulate():
self.assertEqual(
[(f, c.open().read()) for f, c in finder.find("*.xpi")],
[("foo.xpi", foo_xpi)],
)
errors.out = sys.stderr
# Test install.rdf unification
x86_64 = "Darwin_x86_64-gcc3"
x86 = "Darwin_x86-gcc3"
target_tag = "<{em}targetPlatform>{platform}</{em}targetPlatform>"
target_attr = '{em}targetPlatform="{platform}" '
rdf_tag = "".join([
'<{RDF}Description {em}bar="bar" {em}qux="qux">',
"<{em}foo>foo</{em}foo>",
"{targets}",
"<{em}baz>baz</{em}baz>",
"</{RDF}Description>",
])
rdf_attr = "".join([
'<{RDF}Description {em}bar="bar" {attr}{em}qux="qux">',
"{targets}",
"<{em}foo>foo</{em}foo><{em}baz>baz</{em}baz>",
"</{RDF}Description>",
])
for descr_ns, target_ns in (("RDF:", ""), ("", "em:"), ("RDF:",
"em:")):
# First we need to infuse the above strings with our namespaces and
# platform values.
ns = {"RDF": descr_ns, "em": target_ns}
target_tag_x86_64 = target_tag.format(platform=x86_64, **ns)
target_tag_x86 = target_tag.format(platform=x86, **ns)
target_attr_x86_64 = target_attr.format(platform=x86_64, **ns)
target_attr_x86 = target_attr.format(platform=x86, **ns)
tag_x86_64 = rdf_tag.format(targets=target_tag_x86_64, **ns)
tag_x86 = rdf_tag.format(targets=target_tag_x86, **ns)
tag_merged = rdf_tag.format(targets=target_tag_x86_64 +
target_tag_x86,
**ns)
tag_empty = rdf_tag.format(targets="", **ns)
attr_x86_64 = rdf_attr.format(attr=target_attr_x86_64,
targets="",
**ns)
attr_x86 = rdf_attr.format(attr=target_attr_x86, targets="", **ns)
attr_merged = rdf_attr.format(attr="",
targets=target_tag_x86_64 +
target_tag_x86,
**ns)
# This table defines the test cases, columns "a" and "b" being the
# contents of the install.rdf of the respective platform and
# "result" the exepected merged content after unification.
testcases = (
# _____a_____ _____b_____ ___result___#
(tag_x86_64, tag_x86, tag_merged),
(tag_x86_64, tag_empty, tag_empty),
(tag_empty, tag_x86, tag_empty),
(tag_empty, tag_empty, tag_empty),
(attr_x86_64, attr_x86, attr_merged),
(tag_x86_64, attr_x86, tag_merged),
(attr_x86_64, tag_x86, attr_merged),
(attr_x86_64, tag_empty, tag_empty),
(tag_empty, attr_x86, tag_empty),
)
# Now create the files from the above table and compare
results = []
for emid, (rdf_a, rdf_b, result) in enumerate(testcases):
filename = "ext/id{0}/install.rdf".format(emid)
self.create_one("a", filename, rdf_a)
self.create_one("b", filename, rdf_b)
results.append((filename, result))
self.assertEqual(
sorted([(f, c.open().read().decode("utf-8"))
for f, c in finder.find("**/install.rdf")]),
results,
)
def test_unified_build_finder(self):
finder = UnifiedBuildFinder(FileFinder(self.tmppath('a')),
FileFinder(self.tmppath('b')))
# Test chrome.manifest unification
self.create_both('chrome.manifest', 'a\nb\nc\n')
self.create_one('a', 'chrome/chrome.manifest', 'a\nb\nc\n')
self.create_one('b', 'chrome/chrome.manifest', 'b\nc\na\n')
self.assertEqual(sorted([(f, c.open().read()) for f, c in
finder.find('**/chrome.manifest')]),
[('chrome.manifest', 'a\nb\nc\n'),
('chrome/chrome.manifest', 'a\nb\nc\n')])
# Test buildconfig.html unification
self.create_one('a', 'chrome/browser/foo/buildconfig.html',
'\n'.join([
'<html>',
'<body>',
'<h1>about:buildconfig</h1>',
'<div>foo</div>',
'</body>',
'</html>',
]))
self.create_one('b', 'chrome/browser/foo/buildconfig.html',
'\n'.join([
'<html>',
'<body>',
'<h1>about:buildconfig</h1>',
'<div>bar</div>',
'</body>',
'</html>',
]))
self.assertEqual(sorted([(f, c.open().read()) for f, c in
finder.find('**/buildconfig.html')]),
[('chrome/browser/foo/buildconfig.html', '\n'.join([
'<html>',
'<body>',
'<h1>about:buildconfig</h1>',
'<div>foo</div>',
'<hr> </hr>',
'<div>bar</div>',
'</body>',
'</html>',
]))])
# Test xpi file unification
xpi = MockDest()
with JarWriter(fileobj=xpi, compress=True) as jar:
jar.add('foo', 'foo')
jar.add('bar', 'bar')
foo_xpi = xpi.read()
self.create_both('foo.xpi', foo_xpi)
with JarWriter(fileobj=xpi, compress=True) as jar:
jar.add('foo', 'bar')
self.create_one('a', 'bar.xpi', foo_xpi)
self.create_one('b', 'bar.xpi', xpi.read())
errors.out = StringIO()
with self.assertRaises(AccumulatedErrors), errors.accumulate():
self.assertEqual([(f, c.open().read()) for f, c in
finder.find('*.xpi')],
[('foo.xpi', foo_xpi)])
errors.out = sys.stderr
# Test install.rdf unification
x86_64 = 'Darwin_x86_64-gcc3'
x86 = 'Darwin_x86-gcc3'
target_tag = '<{em}targetPlatform>{platform}</{em}targetPlatform>'
target_attr = '{em}targetPlatform="{platform}" '
rdf_tag = ''.join([
'<{RDF}Description {em}bar="bar" {em}qux="qux">',
'<{em}foo>foo</{em}foo>',
'{targets}',
'<{em}baz>baz</{em}baz>',
'</{RDF}Description>'
])
rdf_attr = ''.join([
'<{RDF}Description {em}bar="bar" {attr}{em}qux="qux">',
'{targets}',
'<{em}foo>foo</{em}foo><{em}baz>baz</{em}baz>',
'</{RDF}Description>'
])
for descr_ns, target_ns in (('RDF:', ''), ('', 'em:'), ('RDF:', 'em:')):
# First we need to infuse the above strings with our namespaces and
# platform values.
ns = { 'RDF': descr_ns, 'em': target_ns }
target_tag_x86_64 = target_tag.format(platform=x86_64, **ns)
target_tag_x86 = target_tag.format(platform=x86, **ns)
target_attr_x86_64 = target_attr.format(platform=x86_64, **ns)
target_attr_x86 = target_attr.format(platform=x86, **ns)
tag_x86_64 = rdf_tag.format(targets=target_tag_x86_64, **ns)
tag_x86 = rdf_tag.format(targets=target_tag_x86, **ns)
tag_merged = rdf_tag.format(targets=target_tag_x86_64 + target_tag_x86, **ns)
tag_empty = rdf_tag.format(targets="", **ns)
attr_x86_64 = rdf_attr.format(attr=target_attr_x86_64, targets="", **ns)
attr_x86 = rdf_attr.format(attr=target_attr_x86, targets="", **ns)
attr_merged = rdf_attr.format(attr="", targets=target_tag_x86_64 + target_tag_x86, **ns)
# This table defines the test cases, columns "a" and "b" being the
# contents of the install.rdf of the respective platform and
# "result" the exepected merged content after unification.
testcases = (
#_____a_____ _____b_____ ___result___#
(tag_x86_64, tag_x86, tag_merged ),
(tag_x86_64, tag_empty, tag_empty ),
(tag_empty, tag_x86, tag_empty ),
(tag_empty, tag_empty, tag_empty ),
(attr_x86_64, attr_x86, attr_merged ),
(tag_x86_64, attr_x86, tag_merged ),
(attr_x86_64, tag_x86, attr_merged ),
(attr_x86_64, tag_empty, tag_empty ),
(tag_empty, attr_x86, tag_empty )
)
# Now create the files from the above table and compare
results = []
for emid, (rdf_a, rdf_b, result) in enumerate(testcases):
filename = 'ext/id{0}/install.rdf'.format(emid)
self.create_one('a', filename, rdf_a)
self.create_one('b', filename, rdf_b)
results.append((filename, result))
self.assertEqual(sorted([(f, c.open().read()) for f, c in
finder.find('**/install.rdf')]), results)
def test_simple_error(self):
with self.assertRaises(AccumulatedErrors):
with errors.accumulate():
errors.error('1')
self.assertEquals(self.get_output(), ['Error: 1'])
def test_no_error(self):
with errors.accumulate():
errors.warn('1')
def main():
parser = ArgumentParser()
parser.add_argument('-D', dest='defines', action='append',
metavar="VAR[=VAL]", help='Define a variable')
parser.add_argument('--format', default='omni',
help='Choose the chrome format for packaging ' +
'(omni, jar or flat ; default: %(default)s)')
parser.add_argument('--removals', default=None,
help='removed-files source file')
parser.add_argument('--ignore-errors', action='store_true', default=False,
help='Transform errors into warnings.')
parser.add_argument('--minify', action='store_true', default=False,
help='Make some files more compact while packaging')
parser.add_argument('--jarlog', default='', help='File containing jar ' +
'access logs')
parser.add_argument('--optimizejars', action='store_true', default=False,
help='Enable jar optimizations')
parser.add_argument('--unify', default='',
help='Base directory of another build to unify with')
parser.add_argument('manifest', default=None, nargs='?',
help='Manifest file name')
parser.add_argument('source', help='Source directory')
parser.add_argument('destination', help='Destination directory')
parser.add_argument('--non-resource', nargs='+', metavar='PATTERN',
default=[],
help='Extra files not to be considered as resources')
args = parser.parse_args()
defines = dict(buildconfig.defines)
if args.ignore_errors:
errors.ignore_errors()
if args.defines:
for name, value in [split_define(d) for d in args.defines]:
defines[name] = value
copier = FileCopier()
if args.format == 'flat':
formatter = FlatFormatter(copier)
elif args.format == 'jar':
formatter = JarFormatter(copier, optimize=args.optimizejars)
elif args.format == 'omni':
formatter = OmniJarFormatter(copier,
buildconfig.substs['OMNIJAR_NAME'],
optimize=args.optimizejars,
non_resources=args.non_resource)
else:
errors.fatal('Unknown format: %s' % args.format)
# Adjust defines according to the requested format.
if isinstance(formatter, OmniJarFormatter):
defines['MOZ_OMNIJAR'] = 1
elif 'MOZ_OMNIJAR' in defines:
del defines['MOZ_OMNIJAR']
binpath = ''
if 'BINPATH' in defines:
binpath = SimpleManifestSink.normalize_path(defines['BINPATH'])
while binpath.startswith('/'):
binpath = binpath[1:]
if args.unify:
def is_native(path):
path = os.path.abspath(path)
return platform.machine() in mozpack.path.split(path)
# Invert args.unify and args.source if args.unify points to the
# native architecture.
args.source, args.unify = sorted([args.source, args.unify],
key=is_native, reverse=True)
if is_native(args.source):
launcher.tooldir = args.source
elif not buildconfig.substs['CROSS_COMPILE']:
launcher.tooldir = buildconfig.substs['LIBXUL_DIST']
with errors.accumulate():
if args.unify:
finder = UnifiedBuildFinder(FileFinder(args.source),
FileFinder(args.unify),
minify=args.minify)
else:
finder = FileFinder(args.source, minify=args.minify)
if 'NO_PKG_FILES' in os.environ:
sinkformatter = NoPkgFilesRemover(formatter,
args.manifest is not None)
else:
sinkformatter = formatter
sink = SimpleManifestSink(finder, sinkformatter)
if args.manifest:
preprocess_manifest(sink, args.manifest, defines)
else:
sink.add(Component(''), 'bin/*')
sink.close(args.manifest is not None)
if args.removals:
lines = [l.lstrip() for l in open(args.removals).readlines()]
removals_in = StringIO(''.join(lines))
removals_in.name = args.removals
removals = RemovedFiles(copier)
preprocess(removals_in, removals, defines)
copier.add(mozpack.path.join(binpath, 'removed-files'), removals)
# shlibsign libraries
if launcher.can_launch():
for lib in SIGN_LIBS:
libbase = mozpack.path.join(binpath, '%s%s') \
% (buildconfig.substs['DLL_PREFIX'], lib)
libname = '%s%s' % (libbase, buildconfig.substs['DLL_SUFFIX'])
if copier.contains(libname):
copier.add(libbase + '.chk',
LibSignFile(os.path.join(args.destination,
libname)))
# Setup preloading
if args.jarlog and os.path.exists(args.jarlog):
from mozpack.mozjar import JarLog
log = JarLog(args.jarlog)
for p, f in copier:
if not isinstance(f, Jarrer):
continue
key = JarLog.canonicalize(os.path.join(args.destination, p))
if key in log:
f.preload(log[key])
# Fill startup cache on Windows and Linux only
# (this currently causes build failure on BSD, so skip on that platfom)
if sys.platform == 'win32' or sys.platform.startswith ('linux'):
if isinstance(formatter, OmniJarFormatter) and launcher.can_launch():
if buildconfig.substs['LIBXUL_SDK']:
gre_path = mozpack.path.join(buildconfig.substs['LIBXUL_DIST'],
'bin')
else:
gre_path = None
for base in sorted([[p for p in [mozpack.path.join('bin', b), b]
if os.path.exists(os.path.join(args.source, p))][0]
for b in sink.packager.get_bases()]):
if not gre_path:
gre_path = base
base_path = sink.normalize_path(base)
if base_path in formatter.omnijars:
precompile_cache(formatter.omnijars[base_path],
args.source, gre_path, base)
copier.copy(args.destination)
generate_precomplete(os.path.normpath(os.path.join(args.destination,
binpath)))
def main():
parser = ArgumentParser()
parser.add_argument('-D', dest='defines', action='append',
metavar="VAR[=VAL]", help='Define a variable')
parser.add_argument('--format', default='omni',
help='Choose the chrome format for packaging ' +
'(omni, jar or flat ; default: %(default)s)')
parser.add_argument('--removals', default=None,
help='removed-files source file')
parser.add_argument('--ignore-errors', action='store_true', default=False,
help='Transform errors into warnings.')
parser.add_argument('--ignore-broken-symlinks', action='store_true', default=False,
help='Do not fail when processing broken symlinks.')
parser.add_argument('--minify', action='store_true', default=False,
help='Make some files more compact while packaging')
parser.add_argument('--minify-js', action='store_true',
help='Minify JavaScript files while packaging.')
parser.add_argument('--js-binary',
help='Path to js binary. This is used to verify '
'minified JavaScript. If this is not defined, '
'minification verification will not be performed.')
parser.add_argument('--jarlog', default='', help='File containing jar ' +
'access logs')
parser.add_argument('--compress', choices=('none', 'deflate', 'brotli'),
default='deflate',
help='Use given jar compression (default: deflate)')
parser.add_argument('manifest', default=None, nargs='?',
help='Manifest file name')
parser.add_argument('source', help='Source directory')
parser.add_argument('destination', help='Destination directory')
parser.add_argument('--non-resource', nargs='+', metavar='PATTERN',
default=[],
help='Extra files not to be considered as resources')
args = parser.parse_args()
defines = dict(buildconfig.defines['ALLDEFINES'])
if args.ignore_errors:
errors.ignore_errors()
if args.defines:
for name, value in [split_define(d) for d in args.defines]:
defines[name] = value
compress = {
'none': False,
'deflate': True,
'brotli': JAR_BROTLI,
}[args.compress]
copier = FileCopier()
if args.format == 'flat':
formatter = FlatFormatter(copier)
elif args.format == 'jar':
formatter = JarFormatter(copier, compress=compress)
elif args.format == 'omni':
formatter = OmniJarFormatter(copier,
buildconfig.substs['OMNIJAR_NAME'],
compress=compress,
non_resources=args.non_resource)
else:
errors.fatal('Unknown format: %s' % args.format)
# Adjust defines according to the requested format.
if isinstance(formatter, OmniJarFormatter):
defines['MOZ_OMNIJAR'] = 1
elif 'MOZ_OMNIJAR' in defines:
del defines['MOZ_OMNIJAR']
respath = ''
if 'RESPATH' in defines:
respath = SimpleManifestSink.normalize_path(defines['RESPATH'])
while respath.startswith('/'):
respath = respath[1:]
if not buildconfig.substs['CROSS_COMPILE']:
launcher.tooldir = mozpath.join(buildconfig.topobjdir, 'dist')
with errors.accumulate():
finder_args = dict(
minify=args.minify,
minify_js=args.minify_js,
ignore_broken_symlinks=args.ignore_broken_symlinks,
)
if args.js_binary:
finder_args['minify_js_verify_command'] = [
args.js_binary,
os.path.join(os.path.abspath(os.path.dirname(__file__)),
'js-compare-ast.js')
]
finder = FileFinder(args.source, find_executables=True,
**finder_args)
if 'NO_PKG_FILES' in os.environ:
sinkformatter = NoPkgFilesRemover(formatter,
args.manifest is not None)
else:
sinkformatter = formatter
sink = SimpleManifestSink(finder, sinkformatter)
if args.manifest:
preprocess_manifest(sink, args.manifest, defines)
else:
sink.add(Component(''), 'bin/*')
sink.close(args.manifest is not None)
if args.removals:
removals_in = StringIO(open(args.removals).read())
removals_in.name = args.removals
removals = RemovedFiles(copier)
preprocess(removals_in, removals, defines)
copier.add(mozpath.join(respath, 'removed-files'), removals)
# shlibsign libraries
if launcher.can_launch():
if not mozinfo.isMac and buildconfig.substs.get('COMPILE_ENVIRONMENT'):
for lib in SIGN_LIBS:
libbase = mozpath.join(respath, '%s%s') \
% (buildconfig.substs['DLL_PREFIX'], lib)
libname = '%s%s' % (libbase, buildconfig.substs['DLL_SUFFIX'])
if copier.contains(libname):
copier.add(libbase + '.chk',
LibSignFile(os.path.join(args.destination,
libname)))
# If a pdb file is present and we were instructed to copy it, include it.
# Run on all OSes to capture MinGW builds
if buildconfig.substs.get('MOZ_COPY_PDBS'):
for p, f in copier:
if isinstance(f, ExecutableFile):
pdbname = os.path.splitext(f.inputs()[0])[0] + '.pdb'
if os.path.exists(pdbname):
copier.add(os.path.basename(pdbname), File(pdbname))
# Setup preloading
if args.jarlog:
if not os.path.exists(args.jarlog):
raise Exception('Cannot find jar log: %s' % args.jarlog)
omnijars = []
if isinstance(formatter, OmniJarFormatter):
omnijars = [mozpath.join(base, buildconfig.substs['OMNIJAR_NAME'])
for base in sink.packager.get_bases(addons=False)]
from mozpack.mozjar import JarLog
log = JarLog(args.jarlog)
for p, f in copier:
if not isinstance(f, Jarrer):
continue
if respath:
p = mozpath.relpath(p, respath)
if p in log:
f.preload(log[p])
elif p in omnijars:
raise Exception('No jar log data for %s' % p)
copier.copy(args.destination)
generate_precomplete(os.path.normpath(os.path.join(args.destination,
respath)))
def main():
parser = ArgumentParser()
parser.add_argument('-D', dest='defines', action='append',
metavar="VAR[=VAL]", help='Define a variable')
parser.add_argument('--format', default='omni',
help='Choose the chrome format for packaging ' +
'(omni, jar or flat ; default: %(default)s)')
parser.add_argument('--removals', default=None,
help='removed-files source file')
parser.add_argument('--ignore-errors', action='store_true', default=False,
help='Transform errors into warnings.')
parser.add_argument('--minify', action='store_true', default=False,
help='Make some files more compact while packaging')
parser.add_argument('--minify-js', action='store_true',
help='Minify JavaScript files while packaging.')
parser.add_argument('--js-binary',
help='Path to js binary. This is used to verify '
'minified JavaScript. If this is not defined, '
'minification verification will not be performed.')
parser.add_argument('--jarlog', default='', help='File containing jar ' +
'access logs')
parser.add_argument('--optimizejars', action='store_true', default=False,
help='Enable jar optimizations')
parser.add_argument('--disable-compression', action='store_false',
dest='compress', default=True,
help='Disable jar compression')
parser.add_argument('manifest', default=None, nargs='?',
help='Manifest file name')
parser.add_argument('source', help='Source directory')
parser.add_argument('destination', help='Destination directory')
parser.add_argument('--non-resource', nargs='+', metavar='PATTERN',
default=[],
help='Extra files not to be considered as resources')
args = parser.parse_args()
defines = dict(buildconfig.defines)
if args.ignore_errors:
errors.ignore_errors()
if args.defines:
for name, value in [split_define(d) for d in args.defines]:
defines[name] = value
copier = FileCopier()
if args.format == 'flat':
formatter = FlatFormatter(copier)
elif args.format == 'jar':
formatter = JarFormatter(copier, compress=args.compress, optimize=args.optimizejars)
elif args.format == 'omni':
formatter = OmniJarFormatter(copier,
buildconfig.substs['OMNIJAR_NAME'],
compress=args.compress,
optimize=args.optimizejars,
non_resources=args.non_resource)
else:
errors.fatal('Unknown format: %s' % args.format)
# Adjust defines according to the requested format.
if isinstance(formatter, OmniJarFormatter):
defines['MOZ_OMNIJAR'] = 1
elif 'MOZ_OMNIJAR' in defines:
del defines['MOZ_OMNIJAR']
respath = ''
if 'RESPATH' in defines:
respath = SimpleManifestSink.normalize_path(defines['RESPATH'])
while respath.startswith('/'):
respath = respath[1:]
if not buildconfig.substs['CROSS_COMPILE']:
launcher.tooldir = mozpath.join(buildconfig.topobjdir, 'dist')
with errors.accumulate():
finder_args = dict(
minify=args.minify,
minify_js=args.minify_js,
)
if args.js_binary:
finder_args['minify_js_verify_command'] = [
args.js_binary,
os.path.join(os.path.abspath(os.path.dirname(__file__)),
'js-compare-ast.js')
]
finder = FileFinder(args.source, find_executables=True,
**finder_args)
if 'NO_PKG_FILES' in os.environ:
sinkformatter = NoPkgFilesRemover(formatter,
args.manifest is not None)
else:
sinkformatter = formatter
sink = SimpleManifestSink(finder, sinkformatter)
if args.manifest:
preprocess_manifest(sink, args.manifest, defines)
else:
sink.add(Component(''), 'bin/*')
sink.close(args.manifest is not None)
if args.removals:
removals_in = StringIO(open(args.removals).read())
removals_in.name = args.removals
removals = RemovedFiles(copier)
preprocess(removals_in, removals, defines)
copier.add(mozpath.join(respath, 'removed-files'), removals)
# shlibsign libraries
if launcher.can_launch():
if not mozinfo.isMac and buildconfig.substs.get('COMPILE_ENVIRONMENT'):
for lib in SIGN_LIBS:
libbase = mozpath.join(respath, '%s%s') \
% (buildconfig.substs['DLL_PREFIX'], lib)
libname = '%s%s' % (libbase, buildconfig.substs['DLL_SUFFIX'])
if copier.contains(libname):
copier.add(libbase + '.chk',
LibSignFile(os.path.join(args.destination,
libname)))
# Setup preloading
if args.jarlog and os.path.exists(args.jarlog):
from mozpack.mozjar import JarLog
log = JarLog(args.jarlog)
for p, f in copier:
if not isinstance(f, Jarrer):
continue
key = JarLog.canonicalize(os.path.join(args.destination, p))
if key in log:
f.preload(log[key])
copier.copy(args.destination)
generate_precomplete(os.path.normpath(os.path.join(args.destination,
respath)))
def main():
parser = ArgumentParser()
parser.add_argument('-D', dest='defines', action='append',
metavar="VAR[=VAL]", help='Define a variable')
parser.add_argument('--format', default='omni',
help='Choose the chrome format for packaging ' +
'(omni, jar or flat ; default: %(default)s)')
parser.add_argument('--removals', default=None,
help='removed-files source file')
parser.add_argument('--ignore-errors', action='store_true', default=False,
help='Transform errors into warnings.')
parser.add_argument('--minify', action='store_true', default=False,
help='Make some files more compact while packaging')
parser.add_argument('--minify-js', action='store_true',
help='Minify JavaScript files while packaging.')
parser.add_argument('--js-binary',
help='Path to js binary. This is used to verify '
'minified JavaScript. If this is not defined, '
'minification verification will not be performed.')
parser.add_argument('--jarlog', default='', help='File containing jar ' +
'access logs')
parser.add_argument('--optimizejars', action='store_true', default=False,
help='Enable jar optimizations')
parser.add_argument('--unify', default='',
help='Base directory of another build to unify with')
parser.add_argument('manifest', default=None, nargs='?',
help='Manifest file name')
parser.add_argument('source', help='Source directory')
parser.add_argument('destination', help='Destination directory')
parser.add_argument('--non-resource', nargs='+', metavar='PATTERN',
default=[],
help='Extra files not to be considered as resources')
args = parser.parse_args()
defines = dict(buildconfig.defines)
if args.ignore_errors:
errors.ignore_errors()
if args.defines:
for name, value in [split_define(d) for d in args.defines]:
defines[name] = value
copier = FileCopier()
if args.format == 'flat':
formatter = FlatFormatter(copier)
elif args.format == 'jar':
formatter = JarFormatter(copier, optimize=args.optimizejars)
elif args.format == 'omni':
formatter = OmniJarFormatter(copier,
buildconfig.substs['OMNIJAR_NAME'],
optimize=args.optimizejars,
non_resources=args.non_resource)
else:
errors.fatal('Unknown format: %s' % args.format)
# Adjust defines according to the requested format.
if isinstance(formatter, OmniJarFormatter):
defines['MOZ_OMNIJAR'] = 1
elif 'MOZ_OMNIJAR' in defines:
del defines['MOZ_OMNIJAR']
respath = ''
if 'RESPATH' in defines:
respath = SimpleManifestSink.normalize_path(defines['RESPATH'])
while respath.startswith('/'):
respath = respath[1:]
if args.unify:
def is_native(path):
path = os.path.abspath(path)
return platform.machine() in mozpath.split(path)
# Invert args.unify and args.source if args.unify points to the
# native architecture.
args.source, args.unify = sorted([args.source, args.unify],
key=is_native, reverse=True)
if is_native(args.source):
launcher.tooldir = args.source
elif not buildconfig.substs['CROSS_COMPILE']:
launcher.tooldir = buildconfig.substs['LIBXUL_DIST']
with errors.accumulate():
finder_args = dict(
minify=args.minify,
minify_js=args.minify_js,
)
if args.js_binary:
finder_args['minify_js_verify_command'] = [
args.js_binary,
os.path.join(os.path.abspath(os.path.dirname(__file__)),
'js-compare-ast.js')
]
if args.unify:
finder = UnifiedBuildFinder(FileFinder(args.source),
FileFinder(args.unify),
**finder_args)
else:
finder = FileFinder(args.source, **finder_args)
if 'NO_PKG_FILES' in os.environ:
sinkformatter = NoPkgFilesRemover(formatter,
args.manifest is not None)
else:
sinkformatter = formatter
sink = SimpleManifestSink(finder, sinkformatter)
if args.manifest:
preprocess_manifest(sink, args.manifest, defines)
else:
sink.add(Component(''), 'bin/*')
sink.close(args.manifest is not None)
if args.removals:
removals_in = StringIO(open(args.removals).read())
removals_in.name = args.removals
removals = RemovedFiles(copier)
preprocess(removals_in, removals, defines)
copier.add(mozpath.join(respath, 'removed-files'), removals)
# shlibsign libraries
if launcher.can_launch():
if not mozinfo.isMac:
for lib in SIGN_LIBS:
libbase = mozpath.join(respath, '%s%s') \
% (buildconfig.substs['DLL_PREFIX'], lib)
libname = '%s%s' % (libbase, buildconfig.substs['DLL_SUFFIX'])
if copier.contains(libname):
copier.add(libbase + '.chk',
LibSignFile(os.path.join(args.destination,
libname)))
# Setup preloading
if args.jarlog and os.path.exists(args.jarlog):
from mozpack.mozjar import JarLog
log = JarLog(args.jarlog)
for p, f in copier:
if not isinstance(f, Jarrer):
continue
key = JarLog.canonicalize(os.path.join(args.destination, p))
if key in log:
f.preload(log[key])
# Fill startup cache
if isinstance(formatter, OmniJarFormatter) and launcher.can_launch() \
and buildconfig.substs['MOZ_DISABLE_STARTUPCACHE'] != '1':
gre_path = None
def get_bases():
for b in sink.packager.get_bases(addons=False):
for p in (mozpath.join('bin', b), b):
if os.path.exists(os.path.join(args.source, p)):
yield p
break
for base in sorted(get_bases()):
if not gre_path:
gre_path = base
base_path = sink.normalize_path(base)
if base_path in formatter.omnijars:
precompile_cache(formatter.omnijars[base_path],
args.source, gre_path, base)
copier.copy(args.destination)
def main():
parser = ArgumentParser()
parser.add_argument("-D", dest="defines", action="append", metavar="VAR[=VAL]", help="Define a variable")
parser.add_argument(
"--format",
default="omni",
help="Choose the chrome format for packaging " + "(omni, jar or flat ; default: %(default)s)",
)
parser.add_argument("--removals", default=None, help="removed-files source file")
parser.add_argument("--ignore-errors", action="store_true", default=False, help="Transform errors into warnings.")
parser.add_argument(
"--minify", action="store_true", default=False, help="Make some files more compact while packaging"
)
parser.add_argument("--minify-js", action="store_true", help="Minify JavaScript files while packaging.")
parser.add_argument(
"--js-binary",
help="Path to js binary. This is used to verify "
"minified JavaScript. If this is not defined, "
"minification verification will not be performed.",
)
parser.add_argument("--jarlog", default="", help="File containing jar " + "access logs")
parser.add_argument("--optimizejars", action="store_true", default=False, help="Enable jar optimizations")
parser.add_argument("--unify", default="", help="Base directory of another build to unify with")
parser.add_argument(
"--disable-compression", action="store_false", dest="compress", default=True, help="Disable jar compression"
)
parser.add_argument("manifest", default=None, nargs="?", help="Manifest file name")
parser.add_argument("source", help="Source directory")
parser.add_argument("destination", help="Destination directory")
parser.add_argument(
"--non-resource", nargs="+", metavar="PATTERN", default=[], help="Extra files not to be considered as resources"
)
args = parser.parse_args()
defines = dict(buildconfig.defines)
if args.ignore_errors:
errors.ignore_errors()
if args.defines:
for name, value in [split_define(d) for d in args.defines]:
defines[name] = value
copier = FileCopier()
if args.format == "flat":
formatter = FlatFormatter(copier)
elif args.format == "jar":
formatter = JarFormatter(copier, compress=args.compress, optimize=args.optimizejars)
elif args.format == "omni":
formatter = OmniJarFormatter(
copier,
buildconfig.substs["OMNIJAR_NAME"],
compress=args.compress,
optimize=args.optimizejars,
non_resources=args.non_resource,
)
else:
errors.fatal("Unknown format: %s" % args.format)
# Adjust defines according to the requested format.
if isinstance(formatter, OmniJarFormatter):
defines["MOZ_OMNIJAR"] = 1
elif "MOZ_OMNIJAR" in defines:
del defines["MOZ_OMNIJAR"]
respath = ""
if "RESPATH" in defines:
respath = SimpleManifestSink.normalize_path(defines["RESPATH"])
while respath.startswith("/"):
respath = respath[1:]
if args.unify:
def is_native(path):
path = os.path.abspath(path)
return platform.machine() in mozpath.split(path)
# Invert args.unify and args.source if args.unify points to the
# native architecture.
args.source, args.unify = sorted([args.source, args.unify], key=is_native, reverse=True)
if is_native(args.source):
launcher.tooldir = args.source
elif not buildconfig.substs["CROSS_COMPILE"]:
launcher.tooldir = mozpath.join(buildconfig.topobjdir, "dist")
with errors.accumulate():
finder_args = dict(minify=args.minify, minify_js=args.minify_js)
if args.js_binary:
finder_args["minify_js_verify_command"] = [
args.js_binary,
os.path.join(os.path.abspath(os.path.dirname(__file__)), "js-compare-ast.js"),
]
if args.unify:
finder = UnifiedBuildFinder(FileFinder(args.source), FileFinder(args.unify), **finder_args)
else:
finder = FileFinder(args.source, **finder_args)
if "NO_PKG_FILES" in os.environ:
sinkformatter = NoPkgFilesRemover(formatter, args.manifest is not None)
else:
sinkformatter = formatter
sink = SimpleManifestSink(finder, sinkformatter)
if args.manifest:
preprocess_manifest(sink, args.manifest, defines)
else:
sink.add(Component(""), "bin/*")
sink.close(args.manifest is not None)
if args.removals:
removals_in = StringIO(open(args.removals).read())
removals_in.name = args.removals
removals = RemovedFiles(copier)
preprocess(removals_in, removals, defines)
copier.add(mozpath.join(respath, "removed-files"), removals)
# shlibsign libraries
if launcher.can_launch():
if not mozinfo.isMac and buildconfig.substs.get("COMPILE_ENVIRONMENT"):
for lib in SIGN_LIBS:
libbase = mozpath.join(respath, "%s%s") % (buildconfig.substs["DLL_PREFIX"], lib)
libname = "%s%s" % (libbase, buildconfig.substs["DLL_SUFFIX"])
if copier.contains(libname):
copier.add(libbase + ".chk", LibSignFile(os.path.join(args.destination, libname)))
# Setup preloading
if args.jarlog and os.path.exists(args.jarlog):
from mozpack.mozjar import JarLog
log = JarLog(args.jarlog)
for p, f in copier:
if not isinstance(f, Jarrer):
continue
key = JarLog.canonicalize(os.path.join(args.destination, p))
if key in log:
f.preload(log[key])
# Fill startup cache
if (
isinstance(formatter, OmniJarFormatter)
and launcher.can_launch()
and buildconfig.substs["MOZ_DISABLE_STARTUPCACHE"] != "1"
):
gre_path = None
def get_bases():
for b in sink.packager.get_bases(addons=False):
for p in (mozpath.join("bin", b), b):
if os.path.exists(os.path.join(args.source, p)):
yield p
break
for base in sorted(get_bases()):
if not gre_path:
gre_path = base
omnijar_path = mozpath.join(sink.normalize_path(base), buildconfig.substs["OMNIJAR_NAME"])
if formatter.contains(omnijar_path):
precompile_cache(formatter.copier[omnijar_path], args.source, gre_path, base)
copier.copy(args.destination)