def test_bases(self):
formatter = FlatFormatter(FileRegistry())
formatter.add_base('')
formatter.add_base('browser')
formatter.add_base('webapprt')
self.assertEqual(formatter._get_base('platform.ini'), '')
self.assertEqual(formatter._get_base('browser/application.ini'),
'browser')
self.assertEqual(formatter._get_base('webapprt/webapprt.ini'),
'webapprt')
def test_bases(self):
formatter = FlatFormatter(FileRegistry())
formatter.add_base('')
formatter.add_base('browser')
formatter.add_base('addon0', addon=True)
self.assertEqual(formatter._get_base('platform.ini'),
('', 'platform.ini'))
self.assertEqual(formatter._get_base('browser/application.ini'),
('browser', 'application.ini'))
self.assertEqual(formatter._get_base('addon0/install.rdf'),
('addon0', 'install.rdf'))
def test_flat_formatter(self):
registry = FileRegistry()
formatter = FlatFormatter(registry)
fill_formatter(formatter, CONTENTS)
self.assertEqual(get_contents(registry), RESULT_FLAT)
self.do_test_contents(formatter, CONTENTS)
def test_flat_formatter_with_base(self):
registry = FileRegistry()
formatter = FlatFormatter(registry)
fill_formatter(formatter, CONTENTS_WITH_BASE)
self.assertEqual(get_contents(registry), RESULT_FLAT_WITH_BASE)
self.do_test_contents(formatter, CONTENTS_WITH_BASE)
def unpack_to_registry(source, registry, omnijar_name=None):
"""
Transform a jar chrome or omnijar packaged directory into a flat package.
The given registry is filled with the flat package.
"""
finder = UnpackFinder(source, omnijar_name)
packager = SimplePackager(FlatFormatter(registry))
for p, f in finder.find("*"):
packager.add(p, f)
packager.close()
def unpack_to_registry(source, registry):
'''
Transform a jar chrome or omnijar packaged directory into a flat package.
The given registry is filled with the flat package.
'''
finder = UnpackFinder(source)
packager = SimplePackager(FlatFormatter(registry))
for p, f in finder.find('*'):
packager.add(p, f)
packager.close()
def unpack(source):
'''
Transform a jar chrome or omnijar packaged directory into a flat package.
'''
copier = FileCopier()
finder = UnpackFinder(source)
packager = SimplePackager(FlatFormatter(copier))
for p, f in finder.find('*'):
if mozpack.path.split(p)[0] not in STARTUP_CACHE_PATHS:
packager.add(p, f)
packager.close()
copier.copy(source, skip_if_older=False)
def unpack_to_registry(source, registry):
'''
Transform a jar chrome or omnijar packaged directory into a flat package.
The given registry is filled with the flat package.
'''
finder = UnpackFinder(source)
packager = SimplePackager(FlatFormatter(registry))
for p, f in finder.find('*'):
if mozpath.split(p)[0] not in STARTUP_CACHE_PATHS:
packager.add(p, f)
packager.close()
def main():
parser = argparse.ArgumentParser(
description=
"Merge two builds of a Gecko-based application into a Universal build")
parser.add_argument("app1", help="Directory containing the application")
parser.add_argument("app2",
help="Directory containing the application to merge")
parser.add_argument(
"--non-resource",
nargs="+",
metavar="PATTERN",
default=[],
help="Extra files not to be considered as resources",
)
options = parser.parse_args()
buildconfig.substs["OS_ARCH"] = "Darwin"
buildconfig.substs["LIPO"] = os.environ.get("LIPO")
app1_finder = UnpackFinder(FileFinder(options.app1, find_executables=True))
app2_finder = UnpackFinder(FileFinder(options.app2, find_executables=True))
app_finder = UnifiedBuildFinderWasmHack(app1_finder, app2_finder)
copier = FileCopier()
compress = min(app1_finder.compressed, JAR_DEFLATED)
if app1_finder.kind == "flat":
formatter = FlatFormatter(copier)
elif app1_finder.kind == "jar":
formatter = JarFormatter(copier, compress=compress)
elif app1_finder.kind == "omni":
formatter = OmniJarFormatter(
copier,
app1_finder.omnijar,
compress=compress,
non_resources=options.non_resource,
)
with errors.accumulate():
packager = SimplePackager(formatter)
for p, f in app_finder:
packager.add(p, f)
packager.close()
# Transplant jar preloading information.
for path, log in six.iteritems(app1_finder.jarlogs):
assert isinstance(copier[path], Jarrer)
copier[path].preload(log)
copier.copy(options.app1, skip_if_older=False)
def repack(source, l10n, extra_l10n={}, non_resources=[], non_chrome=set()):
'''
Replace localized data from the `source` directory with localized data
from `l10n` and `extra_l10n`.
The `source` argument points to a directory containing a packaged
application (in omnijar, jar or flat form).
The `l10n` argument points to a directory containing the main localized
data (usually in the form of a language pack addon) to use to replace
in the packaged application.
The `extra_l10n` argument contains a dict associating relative paths in
the source to separate directories containing localized data for them.
This can be used to point at different language pack addons for different
parts of the package application.
The `non_resources` argument gives a list of relative paths in the source
that should not be added in an omnijar in case the packaged application
is in that format.
The `non_chrome` argument gives a list of file/directory patterns for
localized files that are not listed in a chrome.manifest.
'''
app_finder = UnpackFinder(source)
l10n_finder = UnpackFinder(l10n)
if extra_l10n:
finders = {
'': l10n_finder,
}
for base, path in extra_l10n.iteritems():
finders[base] = UnpackFinder(path)
l10n_finder = ComposedFinder(finders)
copier = FileCopier()
compress = min(app_finder.compressed, JAR_DEFLATED)
if app_finder.kind == 'flat':
formatter = FlatFormatter(copier)
elif app_finder.kind == 'jar':
formatter = JarFormatter(copier,
optimize=app_finder.optimizedjars,
compress=compress)
elif app_finder.kind == 'omni':
formatter = OmniJarFormatter(copier,
app_finder.omnijar,
optimize=app_finder.optimizedjars,
compress=compress,
non_resources=non_resources)
with errors.accumulate():
_repack(app_finder, l10n_finder, copier, formatter, non_chrome)
copier.copy(source, skip_if_older=False)
generate_precomplete(source)
def repack(source, l10n, non_resources=[], non_chrome=set()):
app_finder = UnpackFinder(source)
l10n_finder = UnpackFinder(l10n)
copier = FileCopier()
if app_finder.kind == 'flat':
formatter = FlatFormatter(copier)
elif app_finder.kind == 'jar':
formatter = JarFormatter(copier, optimize=app_finder.optimizedjars)
elif app_finder.kind == 'omni':
formatter = OmniJarFormatter(copier, app_finder.omnijar,
optimize=app_finder.optimizedjars,
non_resources=non_resources)
with errors.accumulate():
_repack(app_finder, l10n_finder, copier, formatter, non_chrome)
copier.copy(source, skip_if_older=False)
generate_precomplete(source)
def main():
parser = ArgumentParser()
parser.add_argument('-D',
dest='defines',
action='append',
metavar="VAR[=VAL]",
help='Define a variable')
parser.add_argument('--format',
default='omni',
help='Choose the chrome format for packaging ' +
'(omni, jar or flat ; default: %(default)s)')
parser.add_argument('--removals',
default=None,
help='removed-files source file')
parser.add_argument('--ignore-errors',
action='store_true',
default=False,
help='Transform errors into warnings.')
parser.add_argument('--minify',
action='store_true',
default=False,
help='Make some files more compact while packaging')
parser.add_argument('--minify-js',
action='store_true',
help='Minify JavaScript files while packaging.')
parser.add_argument('--js-binary',
help='Path to js binary. This is used to verify '
'minified JavaScript. If this is not defined, '
'minification verification will not be performed.')
parser.add_argument('--jarlog',
default='',
help='File containing jar ' + 'access logs')
parser.add_argument('--optimizejars',
action='store_true',
default=False,
help='Enable jar optimizations')
parser.add_argument('--unify',
default='',
help='Base directory of another build to unify with')
parser.add_argument('manifest',
default=None,
nargs='?',
help='Manifest file name')
parser.add_argument('source', help='Source directory')
parser.add_argument('destination', help='Destination directory')
parser.add_argument('--non-resource',
nargs='+',
metavar='PATTERN',
default=[],
help='Extra files not to be considered as resources')
args = parser.parse_args()
defines = dict(buildconfig.defines)
if args.ignore_errors:
errors.ignore_errors()
if args.defines:
for name, value in [split_define(d) for d in args.defines]:
defines[name] = value
copier = FileCopier()
if args.format == 'flat':
formatter = FlatFormatter(copier)
elif args.format == 'jar':
formatter = JarFormatter(copier, optimize=args.optimizejars)
elif args.format == 'omni':
formatter = OmniJarFormatter(copier,
buildconfig.substs['OMNIJAR_NAME'],
optimize=args.optimizejars,
non_resources=args.non_resource)
else:
errors.fatal('Unknown format: %s' % args.format)
# Adjust defines according to the requested format.
if isinstance(formatter, OmniJarFormatter):
defines['MOZ_OMNIJAR'] = 1
elif 'MOZ_OMNIJAR' in defines:
del defines['MOZ_OMNIJAR']
binpath = ''
if 'BINPATH' in defines:
binpath = SimpleManifestSink.normalize_path(defines['BINPATH'])
while binpath.startswith('/'):
binpath = binpath[1:]
if args.unify:
def is_native(path):
path = os.path.abspath(path)
return platform.machine() in mozpack.path.split(path)
# Invert args.unify and args.source if args.unify points to the
# native architecture.
args.source, args.unify = sorted([args.source, args.unify],
key=is_native,
reverse=True)
if is_native(args.source):
launcher.tooldir = args.source
elif not buildconfig.substs['CROSS_COMPILE']:
launcher.tooldir = buildconfig.substs['LIBXUL_DIST']
with errors.accumulate():
finder_args = dict(
minify=args.minify,
minify_js=args.minify_js,
)
if args.js_binary:
finder_args['minify_js_verify_command'] = [
args.js_binary,
os.path.join(os.path.abspath(os.path.dirname(__file__)),
'js-compare-ast.js')
]
if args.unify:
finder = UnifiedBuildFinder(FileFinder(args.source),
FileFinder(args.unify), **finder_args)
else:
finder = FileFinder(args.source, **finder_args)
if 'NO_PKG_FILES' in os.environ:
sinkformatter = NoPkgFilesRemover(formatter, args.manifest
is not None)
else:
sinkformatter = formatter
sink = SimpleManifestSink(finder, sinkformatter)
if args.manifest:
preprocess_manifest(sink, args.manifest, defines)
else:
sink.add(Component(''), 'bin/*')
sink.close(args.manifest is not None)
if args.removals:
lines = [l.lstrip() for l in open(args.removals).readlines()]
removals_in = StringIO(''.join(lines))
removals_in.name = args.removals
removals = RemovedFiles(copier)
preprocess(removals_in, removals, defines)
copier.add(mozpack.path.join(binpath, 'removed-files'), removals)
# shlibsign libraries
if launcher.can_launch():
for lib in SIGN_LIBS:
libbase = mozpack.path.join(binpath, '%s%s') \
% (buildconfig.substs['DLL_PREFIX'], lib)
libname = '%s%s' % (libbase, buildconfig.substs['DLL_SUFFIX'])
if copier.contains(libname):
copier.add(
libbase + '.chk',
LibSignFile(os.path.join(args.destination, libname)))
# Setup preloading
if args.jarlog and os.path.exists(args.jarlog):
from mozpack.mozjar import JarLog
log = JarLog(args.jarlog)
for p, f in copier:
if not isinstance(f, Jarrer):
continue
key = JarLog.canonicalize(os.path.join(args.destination, p))
if key in log:
f.preload(log[key])
# Fill startup cache
if isinstance(formatter, OmniJarFormatter) and launcher.can_launch() \
and buildconfig.substs['MOZ_DISABLE_STARTUPCACHE'] != '1':
if buildconfig.substs['LIBXUL_SDK']:
gre_path = mozpack.path.join(buildconfig.substs['LIBXUL_DIST'],
'bin')
else:
gre_path = None
for base in sorted([[
p for p in [mozpack.path.join('bin', b), b]
if os.path.exists(os.path.join(args.source, p))
][0] for b in sink.packager.get_bases()]):
if not gre_path:
gre_path = base
base_path = sink.normalize_path(base)
if base_path in formatter.omnijars:
precompile_cache(formatter.omnijars[base_path], args.source,
gre_path, base)
copier.copy(args.destination)
generate_precomplete(
os.path.normpath(os.path.join(args.destination, binpath)))
def test_bases(self):
formatter = FlatFormatter(FileRegistry())
formatter.add_base('')
formatter.add_base('browser')
formatter.add_base('webapprt')
self.assertEqual(formatter._get_base('platform.ini'), '')
self.assertEqual(formatter._get_base('browser/application.ini'),
'browser')
self.assertEqual(formatter._get_base('webapprt/webapprt.ini'),
'webapprt')
def main():
parser = ArgumentParser()
parser.add_argument(
"-D",
dest="defines",
action="append",
metavar="VAR[=VAL]",
help="Define a variable",
)
parser.add_argument(
"--format",
default="omni",
help="Choose the chrome format for packaging " +
"(omni, jar or flat ; default: %(default)s)",
)
parser.add_argument("--removals",
default=None,
help="removed-files source file")
parser.add_argument(
"--ignore-errors",
action="store_true",
default=False,
help="Transform errors into warnings.",
)
parser.add_argument(
"--ignore-broken-symlinks",
action="store_true",
default=False,
help="Do not fail when processing broken symlinks.",
)
parser.add_argument(
"--minify",
action="store_true",
default=False,
help="Make some files more compact while packaging",
)
parser.add_argument(
"--minify-js",
action="store_true",
help="Minify JavaScript files while packaging.",
)
parser.add_argument(
"--js-binary",
help="Path to js binary. This is used to verify "
"minified JavaScript. If this is not defined, "
"minification verification will not be performed.",
)
parser.add_argument("--jarlog",
default="",
help="File containing jar " + "access logs")
parser.add_argument(
"--compress",
choices=("none", "deflate"),
default="deflate",
help="Use given jar compression (default: deflate)",
)
parser.add_argument("manifest",
default=None,
nargs="?",
help="Manifest file name")
parser.add_argument("source", help="Source directory")
parser.add_argument("destination", help="Destination directory")
parser.add_argument(
"--non-resource",
nargs="+",
metavar="PATTERN",
default=[],
help="Extra files not to be considered as resources",
)
args = parser.parse_args()
defines = dict(buildconfig.defines["ALLDEFINES"])
if args.ignore_errors:
errors.ignore_errors()
if args.defines:
for name, value in [split_define(d) for d in args.defines]:
defines[name] = value
compress = {
"none": False,
"deflate": True,
}[args.compress]
copier = FileCopier()
if args.format == "flat":
formatter = FlatFormatter(copier)
elif args.format == "jar":
formatter = JarFormatter(copier, compress=compress)
elif args.format == "omni":
formatter = OmniJarFormatter(
copier,
buildconfig.substs["OMNIJAR_NAME"],
compress=compress,
non_resources=args.non_resource,
)
else:
errors.fatal("Unknown format: %s" % args.format)
# Adjust defines according to the requested format.
if isinstance(formatter, OmniJarFormatter):
defines["MOZ_OMNIJAR"] = 1
elif "MOZ_OMNIJAR" in defines:
del defines["MOZ_OMNIJAR"]
respath = ""
if "RESPATH" in defines:
respath = SimpleManifestSink.normalize_path(defines["RESPATH"])
while respath.startswith("/"):
respath = respath[1:]
with errors.accumulate():
finder_args = dict(
minify=args.minify,
minify_js=args.minify_js,
ignore_broken_symlinks=args.ignore_broken_symlinks,
)
if args.js_binary:
finder_args["minify_js_verify_command"] = [
args.js_binary,
os.path.join(os.path.abspath(os.path.dirname(__file__)),
"js-compare-ast.js"),
]
finder = PackagerFileFinder(args.source,
find_executables=True,
**finder_args)
if "NO_PKG_FILES" in os.environ:
sinkformatter = NoPkgFilesRemover(formatter, args.manifest
is not None)
else:
sinkformatter = formatter
sink = SimpleManifestSink(finder, sinkformatter)
if args.manifest:
preprocess_manifest(sink, args.manifest, defines)
else:
sink.add(Component(""), "bin/*")
sink.close(args.manifest is not None)
if args.removals:
removals_in = StringIO(open(args.removals).read())
removals_in.name = args.removals
removals = RemovedFiles(copier)
preprocess(removals_in, removals, defines)
copier.add(mozpath.join(respath, "removed-files"), removals)
# If a pdb file is present and we were instructed to copy it, include it.
# Run on all OSes to capture MinGW builds
if buildconfig.substs.get("MOZ_COPY_PDBS"):
# We want to mutate the copier while we're iterating through it, so copy
# the items to a list first.
copier_items = [(p, f) for p, f in copier]
for p, f in copier_items:
if isinstance(f, ExecutableFile):
pdbname = os.path.splitext(f.inputs()[0])[0] + ".pdb"
if os.path.exists(pdbname):
copier.add(os.path.basename(pdbname), File(pdbname))
# Setup preloading
if args.jarlog:
if not os.path.exists(args.jarlog):
raise Exception("Cannot find jar log: %s" % args.jarlog)
omnijars = []
if isinstance(formatter, OmniJarFormatter):
omnijars = [
mozpath.join(base, buildconfig.substs["OMNIJAR_NAME"])
for base in sink.packager.get_bases(addons=False)
]
from mozpack.mozjar import JarLog
log = JarLog(args.jarlog)
for p, f in copier:
if not isinstance(f, Jarrer):
continue
if respath:
p = mozpath.relpath(p, respath)
if p in log:
f.preload(log[p])
elif p in omnijars:
raise Exception("No jar log data for %s" % p)
copier.copy(args.destination)
generate_precomplete(
os.path.normpath(os.path.join(args.destination, respath)))
def main():
parser = ArgumentParser()
parser.add_argument('-D', dest='defines', action='append',
metavar="VAR[=VAL]", help='Define a variable')
parser.add_argument('--format', default='omni',
help='Choose the chrome format for packaging ' +
'(omni, jar or flat ; default: %(default)s)')
parser.add_argument('--removals', default=None,
help='removed-files source file')
parser.add_argument('--ignore-errors', action='store_true', default=False,
help='Transform errors into warnings.')
parser.add_argument('--minify', action='store_true', default=False,
help='Make some files more compact while packaging')
parser.add_argument('--minify-js', action='store_true',
help='Minify JavaScript files while packaging.')
parser.add_argument('--js-binary',
help='Path to js binary. This is used to verify '
'minified JavaScript. If this is not defined, '
'minification verification will not be performed.')
parser.add_argument('--jarlog', default='', help='File containing jar ' +
'access logs')
parser.add_argument('--optimizejars', action='store_true', default=False,
help='Enable jar optimizations')
parser.add_argument('--unify', default='',
help='Base directory of another build to unify with')
parser.add_argument('manifest', default=None, nargs='?',
help='Manifest file name')
parser.add_argument('source', help='Source directory')
parser.add_argument('destination', help='Destination directory')
parser.add_argument('--non-resource', nargs='+', metavar='PATTERN',
default=[],
help='Extra files not to be considered as resources')
args = parser.parse_args()
defines = dict(buildconfig.defines)
if args.ignore_errors:
errors.ignore_errors()
if args.defines:
for name, value in [split_define(d) for d in args.defines]:
defines[name] = value
copier = FileCopier()
if args.format == 'flat':
formatter = FlatFormatter(copier)
elif args.format == 'jar':
formatter = JarFormatter(copier, optimize=args.optimizejars)
elif args.format == 'omni':
formatter = OmniJarFormatter(copier,
buildconfig.substs['OMNIJAR_NAME'],
optimize=args.optimizejars,
non_resources=args.non_resource)
else:
errors.fatal('Unknown format: %s' % args.format)
# Adjust defines according to the requested format.
if isinstance(formatter, OmniJarFormatter):
defines['MOZ_OMNIJAR'] = 1
elif 'MOZ_OMNIJAR' in defines:
del defines['MOZ_OMNIJAR']
respath = ''
if 'RESPATH' in defines:
respath = SimpleManifestSink.normalize_path(defines['RESPATH'])
while respath.startswith('/'):
respath = respath[1:]
if args.unify:
def is_native(path):
path = os.path.abspath(path)
return platform.machine() in mozpath.split(path)
# Invert args.unify and args.source if args.unify points to the
# native architecture.
args.source, args.unify = sorted([args.source, args.unify],
key=is_native, reverse=True)
if is_native(args.source):
launcher.tooldir = args.source
elif not buildconfig.substs['CROSS_COMPILE']:
launcher.tooldir = mozpath.join(buildconfig.topobjdir, 'dist')
with errors.accumulate():
finder_args = dict(
minify=args.minify,
minify_js=args.minify_js,
)
if args.js_binary:
finder_args['minify_js_verify_command'] = [
args.js_binary,
os.path.join(os.path.abspath(os.path.dirname(__file__)),
'js-compare-ast.js')
]
if args.unify:
finder = UnifiedBuildFinder(FileFinder(args.source),
FileFinder(args.unify),
**finder_args)
else:
finder = FileFinder(args.source, **finder_args)
if 'NO_PKG_FILES' in os.environ:
sinkformatter = NoPkgFilesRemover(formatter,
args.manifest is not None)
else:
sinkformatter = formatter
sink = SimpleManifestSink(finder, sinkformatter)
if args.manifest:
preprocess_manifest(sink, args.manifest, defines)
else:
sink.add(Component(''), 'bin/*')
sink.close(args.manifest is not None)
if args.removals:
removals_in = StringIO(open(args.removals).read())
removals_in.name = args.removals
removals = RemovedFiles(copier)
preprocess(removals_in, removals, defines)
copier.add(mozpath.join(respath, 'removed-files'), removals)
# shlibsign libraries
if launcher.can_launch():
if not mozinfo.isMac:
for lib in SIGN_LIBS:
libbase = mozpath.join(respath, '%s%s') \
% (buildconfig.substs['DLL_PREFIX'], lib)
libname = '%s%s' % (libbase, buildconfig.substs['DLL_SUFFIX'])
if copier.contains(libname):
copier.add(libbase + '.chk',
LibSignFile(os.path.join(args.destination,
libname)))
# Setup preloading
if args.jarlog and os.path.exists(args.jarlog):
from mozpack.mozjar import JarLog
log = JarLog(args.jarlog)
for p, f in copier:
if not isinstance(f, Jarrer):
continue
key = JarLog.canonicalize(os.path.join(args.destination, p))
if key in log:
f.preload(log[key])
# Fill startup cache
if isinstance(formatter, OmniJarFormatter) and launcher.can_launch() \
and buildconfig.substs['MOZ_DISABLE_STARTUPCACHE'] != '1':
gre_path = None
def get_bases():
for b in sink.packager.get_bases(addons=False):
for p in (mozpath.join('bin', b), b):
if os.path.exists(os.path.join(args.source, p)):
yield p
break
for base in sorted(get_bases()):
if not gre_path:
gre_path = base
omnijar_path = mozpath.join(sink.normalize_path(base),
buildconfig.substs['OMNIJAR_NAME'])
if formatter.contains(omnijar_path):
precompile_cache(formatter.copier[omnijar_path],
args.source, gre_path, base)
copier.copy(args.destination)
def main():
parser = ArgumentParser()
parser.add_argument('-D', dest='defines', action='append',
metavar="VAR[=VAL]", help='Define a variable')
parser.add_argument('--format', default='omni',
help='Choose the chrome format for packaging ' +
'(omni, jar or flat ; default: %(default)s)')
parser.add_argument('--removals', default=None,
help='removed-files source file')
parser.add_argument('--ignore-errors', action='store_true', default=False,
help='Transform errors into warnings.')
parser.add_argument('--ignore-broken-symlinks', action='store_true', default=False,
help='Do not fail when processing broken symlinks.')
parser.add_argument('--minify', action='store_true', default=False,
help='Make some files more compact while packaging')
parser.add_argument('--minify-js', action='store_true',
help='Minify JavaScript files while packaging.')
parser.add_argument('--js-binary',
help='Path to js binary. This is used to verify '
'minified JavaScript. If this is not defined, '
'minification verification will not be performed.')
parser.add_argument('--jarlog', default='', help='File containing jar ' +
'access logs')
parser.add_argument('--compress', choices=('none', 'deflate', 'brotli'),
default='deflate',
help='Use given jar compression (default: deflate)')
parser.add_argument('manifest', default=None, nargs='?',
help='Manifest file name')
parser.add_argument('source', help='Source directory')
parser.add_argument('destination', help='Destination directory')
parser.add_argument('--non-resource', nargs='+', metavar='PATTERN',
default=[],
help='Extra files not to be considered as resources')
args = parser.parse_args()
defines = dict(buildconfig.defines['ALLDEFINES'])
if args.ignore_errors:
errors.ignore_errors()
if args.defines:
for name, value in [split_define(d) for d in args.defines]:
defines[name] = value
compress = {
'none': False,
'deflate': True,
'brotli': JAR_BROTLI,
}[args.compress]
copier = FileCopier()
if args.format == 'flat':
formatter = FlatFormatter(copier)
elif args.format == 'jar':
formatter = JarFormatter(copier, compress=compress)
elif args.format == 'omni':
formatter = OmniJarFormatter(copier,
buildconfig.substs['OMNIJAR_NAME'],
compress=compress,
non_resources=args.non_resource)
else:
errors.fatal('Unknown format: %s' % args.format)
# Adjust defines according to the requested format.
if isinstance(formatter, OmniJarFormatter):
defines['MOZ_OMNIJAR'] = 1
elif 'MOZ_OMNIJAR' in defines:
del defines['MOZ_OMNIJAR']
respath = ''
if 'RESPATH' in defines:
respath = SimpleManifestSink.normalize_path(defines['RESPATH'])
while respath.startswith('/'):
respath = respath[1:]
if not buildconfig.substs['CROSS_COMPILE']:
launcher.tooldir = mozpath.join(buildconfig.topobjdir, 'dist')
with errors.accumulate():
finder_args = dict(
minify=args.minify,
minify_js=args.minify_js,
ignore_broken_symlinks=args.ignore_broken_symlinks,
)
if args.js_binary:
finder_args['minify_js_verify_command'] = [
args.js_binary,
os.path.join(os.path.abspath(os.path.dirname(__file__)),
'js-compare-ast.js')
]
finder = FileFinder(args.source, find_executables=True,
**finder_args)
if 'NO_PKG_FILES' in os.environ:
sinkformatter = NoPkgFilesRemover(formatter,
args.manifest is not None)
else:
sinkformatter = formatter
sink = SimpleManifestSink(finder, sinkformatter)
if args.manifest:
preprocess_manifest(sink, args.manifest, defines)
else:
sink.add(Component(''), 'bin/*')
sink.close(args.manifest is not None)
if args.removals:
removals_in = StringIO(open(args.removals).read())
removals_in.name = args.removals
removals = RemovedFiles(copier)
preprocess(removals_in, removals, defines)
copier.add(mozpath.join(respath, 'removed-files'), removals)
# shlibsign libraries
if launcher.can_launch():
if not mozinfo.isMac and buildconfig.substs.get('COMPILE_ENVIRONMENT'):
for lib in SIGN_LIBS:
libbase = mozpath.join(respath, '%s%s') \
% (buildconfig.substs['DLL_PREFIX'], lib)
libname = '%s%s' % (libbase, buildconfig.substs['DLL_SUFFIX'])
if copier.contains(libname):
copier.add(libbase + '.chk',
LibSignFile(os.path.join(args.destination,
libname)))
# If a pdb file is present and we were instructed to copy it, include it.
# Run on all OSes to capture MinGW builds
if buildconfig.substs.get('MOZ_COPY_PDBS'):
for p, f in copier:
if isinstance(f, ExecutableFile):
pdbname = os.path.splitext(f.inputs()[0])[0] + '.pdb'
if os.path.exists(pdbname):
copier.add(os.path.basename(pdbname), File(pdbname))
# Setup preloading
if args.jarlog:
if not os.path.exists(args.jarlog):
raise Exception('Cannot find jar log: %s' % args.jarlog)
omnijars = []
if isinstance(formatter, OmniJarFormatter):
omnijars = [mozpath.join(base, buildconfig.substs['OMNIJAR_NAME'])
for base in sink.packager.get_bases(addons=False)]
from mozpack.mozjar import JarLog
log = JarLog(args.jarlog)
for p, f in copier:
if not isinstance(f, Jarrer):
continue
if respath:
p = mozpath.relpath(p, respath)
if p in log:
f.preload(log[p])
elif p in omnijars:
raise Exception('No jar log data for %s' % p)
copier.copy(args.destination)
generate_precomplete(os.path.normpath(os.path.join(args.destination,
respath)))
def test_chrome_override(self):
registry = FileRegistry()
f = FlatFormatter(registry)
f.add_base('')
f.add_manifest(ManifestContent('chrome', 'foo', 'foo/unix'))
# A more specific entry for a given chrome name can override a more
# generic one.
f.add_manifest(ManifestContent('chrome', 'foo', 'foo/win', 'os=WINNT'))
f.add_manifest(ManifestContent('chrome', 'foo', 'foo/osx', 'os=Darwin'))
# Chrome with the same name overrides the previous registration.
with self.assertRaises(ErrorMessage) as e:
f.add_manifest(ManifestContent('chrome', 'foo', 'foo/'))
self.assertEqual(e.exception.message,
'Error: "content foo foo/" overrides '
'"content foo foo/unix"')
# Chrome with the same name and same flags overrides the previous
# registration.
with self.assertRaises(ErrorMessage) as e:
f.add_manifest(ManifestContent('chrome', 'foo', 'foo/', 'os=WINNT'))
self.assertEqual(e.exception.message,
'Error: "content foo foo/ os=WINNT" overrides '
'"content foo foo/win os=WINNT"')
# We may start with the more specific entry first
f.add_manifest(ManifestContent('chrome', 'bar', 'bar/win', 'os=WINNT'))
# Then adding a more generic one overrides it.
with self.assertRaises(ErrorMessage) as e:
f.add_manifest(ManifestContent('chrome', 'bar', 'bar/unix'))
self.assertEqual(e.exception.message,
'Error: "content bar bar/unix" overrides '
'"content bar bar/win os=WINNT"')
# Adding something more specific still works.
f.add_manifest(ManifestContent('chrome', 'bar', 'bar/win',
'os=WINNT osversion>=7.0'))
# Variations of skin/locales are allowed.
f.add_manifest(ManifestSkin('chrome', 'foo', 'classic/1.0',
'foo/skin/classic/'))
f.add_manifest(ManifestSkin('chrome', 'foo', 'modern/1.0',
'foo/skin/modern/'))
f.add_manifest(ManifestLocale('chrome', 'foo', 'en-US',
'foo/locale/en-US/'))
f.add_manifest(ManifestLocale('chrome', 'foo', 'ja-JP',
'foo/locale/ja-JP/'))
# But same-skin/locale still error out.
with self.assertRaises(ErrorMessage) as e:
f.add_manifest(ManifestSkin('chrome', 'foo', 'classic/1.0',
'foo/skin/classic/foo'))
self.assertEqual(e.exception.message,
'Error: "skin foo classic/1.0 foo/skin/classic/foo" overrides '
'"skin foo classic/1.0 foo/skin/classic/"')
with self.assertRaises(ErrorMessage) as e:
f.add_manifest(ManifestLocale('chrome', 'foo', 'en-US',
'foo/locale/en-US/foo'))
self.assertEqual(e.exception.message,
'Error: "locale foo en-US foo/locale/en-US/foo" overrides '
'"locale foo en-US foo/locale/en-US/"')
# Duplicating existing manifest entries is not an error.
f.add_manifest(ManifestContent('chrome', 'foo', 'foo/unix'))
self.assertEqual(self.get_output(), [
'Warning: "content foo foo/unix" is duplicated. Skipping.',
])
def repack(source, l10n):
finder = UnpackFinder(source)
l10n_finder = UnpackFinder(l10n)
copier = FileCopier()
if finder.kind == 'flat':
formatter = FlatFormatter(copier)
elif finder.kind == 'jar':
formatter = JarFormatter(copier, optimize=finder.optimizedjars)
elif finder.kind == 'omni':
formatter = OmniJarFormatter(copier, finder.omnijar,
optimize=finder.optimizedjars)
# Read all manifest entries from the packaged directory.
manifests = dict((p, m) for p, m in finder.find('**/*.manifest')
if is_manifest(p))
assert all(isinstance(m, ManifestFile) for m in manifests.itervalues())
entries = [e for m in manifests.itervalues() for e in m if e.localized]
# Find unique locales used in these manifest entries.
locales = list(set(e.id for e in entries if isinstance(e, ManifestLocale)))
# Find all paths whose manifest are included by no other manifest.
includes = set(mozpack.path.join(e.base, e.relpath)
for m in manifests.itervalues()
for e in m if isinstance(e, Manifest))
bases = [mozpack.path.dirname(p) for p in set(manifests.keys()) - includes]
# Read all manifest entries from the langpack directory.
manifests = [m for p, m in l10n_finder.find('**/*.manifest')
if is_manifest(p)]
assert all(isinstance(m, ManifestFile) for m in manifests)
l10n_entries = [e for m in manifests for e in m if e.localized]
# Find unique locales used in these manifest entries.
l10n_locales = list(set(e.id for e in l10n_entries
if isinstance(e, ManifestLocale)))
# The code further below assumes there's only one locale replaced with
# another one.
if len(locales) > 1 or len(l10n_locales) > 1:
errors.fatal("Multiple locales aren't supported")
locale = locales[0]
l10n_locale = l10n_locales[0]
# For each base directory, store what path a locale chrome package name
# corresponds to.
# e.g., for the following entry under app/chrome:
# locale foo en-US path/to/files
# keep track that the locale path for foo in app is
# app/chrome/path/to/files.
l10n_paths = {}
for e in l10n_entries:
if isinstance(e, ManifestChrome):
base = mozpack.path.basedir(e.path, bases)
if not base in l10n_paths:
l10n_paths[base] = {}
l10n_paths[base][e.name] = e.path
# For chrome and non chrome files or directories, store what langpack path
# corresponds to a package path.
paths = dict((e.path,
l10n_paths[mozpack.path.basedir(e.path, bases)][e.name])
for e in entries if isinstance(e, ManifestEntryWithRelPath))
for path in NON_CHROME:
for p, f in l10n_finder.find(path):
paths[p] = p
# Create a new package, with non localized bits coming from the original
# package, and localized bits coming from the langpack.
packager = SimplePackager(formatter)
for p, f in finder:
if is_manifest(p):
# Remove localized manifest entries.
for e in [e for e in f if e.localized]:
f.remove(e)
base = mozpack.path.basedir(p, paths.keys())
if base:
# If the path is one that needs a locale replacement, use the
# corresponding file from the langpack.
subpath = mozpack.path.relpath(p, base)
path = mozpack.path.normpath(mozpack.path.join(paths[base],
subpath))
files = [f for p, f in l10n_finder.find(path)]
if len(files) == 0 and base in NON_CHROME:
path = path.replace(locale, l10n_locale)
files = [f for p, f in l10n_finder.find(path)]
if len(files) == 0:
if not base in NON_CHROME:
errors.error("Missing file: %s" % os.path.join(l10n, path))
else:
packager.add(path, files[0])
else:
packager.add(p, f)
# Add localized manifest entries from the langpack.
l10n_manifests = []
for base in set(e.base for e in l10n_entries):
m = ManifestFile(base)
for e in l10n_entries:
if e.base == base:
m.add(e)
path = mozpack.path.join(base, 'chrome.%s.manifest' % l10n_locale)
l10n_manifests.append((path, m))
bases = packager.get_bases()
for path, m in l10n_manifests:
base = mozpack.path.basedir(path, bases)
packager.add(path, m)
# Add a "manifest $path" entry in the top manifest under that base.
m = ManifestFile(base)
m.add(Manifest(base, mozpack.path.relpath(path, base)))
packager.add(mozpack.path.join(base, 'chrome.manifest'), m)
packager.close()
# Add any remaining non chrome files.
for base in NON_CHROME:
for p, f in l10n_finder.find(base):
if not formatter.contains(p):
formatter.add(p, f)
# Transplant jar preloading information.
for path, log in finder.jarlogs.iteritems():
assert isinstance(copier[path], Jarrer)
copier[path].preload([l.replace(locale, l10n_locale) for l in log])
copier.copy(source)
generate_precomplete(source)
def test_flat_formatter(self):
registry = FileRegistry()
formatter = FlatFormatter(registry)
formatter.add_base('app')
formatter.add('f/oo/bar', GeneratedFile('foobar'))
formatter.add('f/oo/baz', GeneratedFile('foobaz'))
formatter.add('f/oo/qux', GeneratedFile('fooqux'))
formatter.add_manifest(ManifestContent('f/oo', 'bar', 'bar'))
formatter.add_manifest(ManifestContent('f/oo', 'qux', 'qux'))
self.assertEqual(registry.paths(),
['f/oo/bar', 'f/oo/baz', 'f/oo/qux',
'chrome.manifest', 'f/f.manifest',
'f/oo/oo.manifest'])
self.assertEqual(registry['chrome.manifest'].open().read(),
'manifest f/f.manifest\n')
self.assertEqual(registry['f/f.manifest'].open().read(),
'manifest oo/oo.manifest\n')
self.assertEqual(registry['f/oo/oo.manifest'].open().read(), ''.join([
'content bar bar\n',
'content qux qux\n',
]))
formatter.add_interfaces('components/foo.xpt', foo_xpt)
formatter.add_interfaces('components/bar.xpt', bar_xpt)
self.assertEqual(registry.paths(),
['f/oo/bar', 'f/oo/baz', 'f/oo/qux',
'chrome.manifest', 'f/f.manifest',
'f/oo/oo.manifest', 'components/components.manifest',
'components/interfaces.xpt'])
self.assertEqual(registry['chrome.manifest'].open().read(), ''.join([
'manifest f/f.manifest\n',
'manifest components/components.manifest\n',
]))
self.assertEqual(
registry['components/components.manifest'].open().read(),
'interfaces interfaces.xpt\n'
)
registry['components/interfaces.xpt'] \
.copy(self.tmppath('interfaces.xpt'))
linked = read_interfaces(self.tmppath('interfaces.xpt'))
foo = read_interfaces(foo_xpt.open())
bar = read_interfaces(bar_xpt.open())
self.assertEqual(foo['foo'], linked['foo'])
self.assertEqual(bar['bar'], linked['bar'])
formatter.add_manifest(ManifestContent('app/chrome', 'content',
'foo/'))
self.assertEqual(registry['chrome.manifest'].open().read(), ''.join([
'manifest f/f.manifest\n',
'manifest components/components.manifest\n',
]))
self.assertEqual(registry['app/chrome.manifest'].open().read(),
'manifest chrome/chrome.manifest\n')
self.assertEqual(registry['app/chrome/chrome.manifest'].open().read(),
'content content foo/\n')
def test_chrome_override(self):
registry = FileRegistry()
f = FlatFormatter(registry)
f.add_base('')
f.add_manifest(ManifestContent('chrome', 'foo', 'foo/unix'))
# A more specific entry for a given chrome name can override a more
# generic one.
f.add_manifest(ManifestContent('chrome', 'foo', 'foo/win', 'os=WINNT'))
f.add_manifest(ManifestContent('chrome', 'foo', 'foo/osx', 'os=Darwin'))
# Chrome with the same name overrides the previous registration.
with self.assertRaises(ErrorMessage) as e:
f.add_manifest(ManifestContent('chrome', 'foo', 'foo/'))
self.assertEqual(e.exception.message,
'Error: "content foo foo/" overrides '
'"content foo foo/unix"')
# Chrome with the same name and same flags overrides the previous
# registration.
with self.assertRaises(ErrorMessage) as e:
f.add_manifest(ManifestContent('chrome', 'foo', 'foo/', 'os=WINNT'))
self.assertEqual(e.exception.message,
'Error: "content foo foo/ os=WINNT" overrides '
'"content foo foo/win os=WINNT"')
# We may start with the more specific entry first
f.add_manifest(ManifestContent('chrome', 'bar', 'bar/win', 'os=WINNT'))
# Then adding a more generic one overrides it.
with self.assertRaises(ErrorMessage) as e:
f.add_manifest(ManifestContent('chrome', 'bar', 'bar/unix'))
self.assertEqual(e.exception.message,
'Error: "content bar bar/unix" overrides '
'"content bar bar/win os=WINNT"')
# Adding something more specific still works.
f.add_manifest(ManifestContent('chrome', 'bar', 'bar/win',
'os=WINNT osversion>=7.0'))
def test_flat_formatter(self):
registry = FileRegistry()
formatter = FlatFormatter(registry)
fill_formatter(formatter, CONTENTS)
RESULT = {
'chrome.manifest': [
'manifest chrome/chrome.manifest',
'manifest components/components.manifest',
],
'chrome/chrome.manifest': [
'manifest f/f.manifest',
],
'chrome/f/f.manifest': [
'content oo oo/',
'content bar oo/bar/',
'resource foo resource://bar/',
],
'chrome/f/oo/bar/baz':
CONTENTS['chrome/f/oo/bar/baz'],
'chrome/f/oo/baz':
CONTENTS['chrome/f/oo/baz'],
'chrome/f/oo/qux':
CONTENTS['chrome/f/oo/qux'],
'components/components.manifest': [
'binary-component foo.so',
'interfaces interfaces.xpt',
],
'components/foo.so':
CONTENTS['components/foo.so'],
'components/interfaces.xpt': {
'foo': read_interfaces(foo_xpt.open())['foo'],
'bar': read_interfaces(bar_xpt.open())['bar'],
},
'foo':
CONTENTS['foo'],
'app/chrome.manifest': [
'manifest chrome/chrome.manifest',
'manifest components/components.manifest',
],
'app/chrome/chrome.manifest': [
'content content foo/',
],
'app/chrome/foo/foo':
CONTENTS['app/chrome/foo/foo'],
'app/components/components.manifest': [
'component {foo-id} foo.js',
],
'app/components/foo.js':
CONTENTS['app/components/foo.js'],
'addon0/chrome.manifest': [
'manifest chrome/chrome.manifest',
'manifest components/components.manifest',
],
'addon0/chrome/chrome.manifest': [
'content content foo/bar/',
],
'addon0/chrome/foo/bar/baz':
CONTENTS['addon0/chrome/foo/bar/baz'],
'addon0/components/components.manifest': [
'interfaces interfaces.xpt',
],
'addon0/components/interfaces.xpt': {
'foo': read_interfaces(foo2_xpt.open())['foo'],
'bar': read_interfaces(bar_xpt.open())['bar'],
},
}
self.assertEqual(get_contents(registry), RESULT)
def test_bases(self):
formatter = FlatFormatter(FileRegistry())
formatter.add_base("")
formatter.add_base("browser")
formatter.add_base("webapprt")
formatter.add_base("addon0", addon=True)
self.assertEqual(formatter._get_base("platform.ini"), "")
self.assertEqual(formatter._get_base("browser/application.ini"), "browser")
self.assertEqual(formatter._get_base("webapprt/webapprt.ini"), "webapprt")
self.assertEqual(formatter._get_base("addon0/install.rdf"), "addon0")
def test_chrome_override(self):
registry = FileRegistry()
f = FlatFormatter(registry)
f.add_base('')
f.add_manifest(ManifestContent('chrome', 'foo', 'foo/unix'))
# A more specific entry for a given chrome name can override a more
# generic one.
f.add_manifest(ManifestContent('chrome', 'foo', 'foo/win', 'os=WINNT'))
f.add_manifest(ManifestContent('chrome', 'foo', 'foo/osx', 'os=Darwin'))
# Chrome with the same name overrides the previous registration.
with self.assertRaises(ErrorMessage) as e:
f.add_manifest(ManifestContent('chrome', 'foo', 'foo/'))
self.assertEqual(e.exception.message,
'Error: "content foo foo/" overrides '
'"content foo foo/unix"')
# Chrome with the same name and same flags overrides the previous
# registration.
with self.assertRaises(ErrorMessage) as e:
f.add_manifest(ManifestContent('chrome', 'foo', 'foo/', 'os=WINNT'))
self.assertEqual(e.exception.message,
'Error: "content foo foo/ os=WINNT" overrides '
'"content foo foo/win os=WINNT"')
# We may start with the more specific entry first
f.add_manifest(ManifestContent('chrome', 'bar', 'bar/win', 'os=WINNT'))
# Then adding a more generic one overrides it.
with self.assertRaises(ErrorMessage) as e:
f.add_manifest(ManifestContent('chrome', 'bar', 'bar/unix'))
self.assertEqual(e.exception.message,
'Error: "content bar bar/unix" overrides '
'"content bar bar/win os=WINNT"')
# Adding something more specific still works.
f.add_manifest(ManifestContent('chrome', 'bar', 'bar/win',
'os=WINNT osversion>=7.0'))
# Variations of skin/locales are allowed.
f.add_manifest(ManifestSkin('chrome', 'foo', 'classic/1.0',
'foo/skin/classic/'))
f.add_manifest(ManifestSkin('chrome', 'foo', 'modern/1.0',
'foo/skin/modern/'))
f.add_manifest(ManifestLocale('chrome', 'foo', 'en-US',
'foo/locale/en-US/'))
f.add_manifest(ManifestLocale('chrome', 'foo', 'ja-JP',
'foo/locale/ja-JP/'))
# But same-skin/locale still error out.
with self.assertRaises(ErrorMessage) as e:
f.add_manifest(ManifestSkin('chrome', 'foo', 'classic/1.0',
'foo/skin/classic/foo'))
self.assertEqual(e.exception.message,
'Error: "skin foo classic/1.0 foo/skin/classic/foo" overrides '
'"skin foo classic/1.0 foo/skin/classic/"')
with self.assertRaises(ErrorMessage) as e:
f.add_manifest(ManifestLocale('chrome', 'foo', 'en-US',
'foo/locale/en-US/foo'))
self.assertEqual(e.exception.message,
'Error: "locale foo en-US foo/locale/en-US/foo" overrides '
'"locale foo en-US foo/locale/en-US/"')
# Duplicating existing manifest entries is not an error.
f.add_manifest(ManifestContent('chrome', 'foo', 'foo/unix'))
self.assertEqual(self.get_output(), [
'Warning: "content foo foo/unix" is duplicated. Skipping.',
])
def test_l10n_repack(self):
foo = GeneratedFile('foo')
foobar = GeneratedFile('foobar')
qux = GeneratedFile('qux')
bar = GeneratedFile('bar')
baz = GeneratedFile('baz')
dict_aa = GeneratedFile('dict_aa')
dict_bb = GeneratedFile('dict_bb')
dict_cc = GeneratedFile('dict_cc')
barbaz = GeneratedFile('barbaz')
lst = GeneratedFile('foo\nbar')
app_finder = MockFinder({
'bar/foo':
foo,
'chrome/foo/foobar':
foobar,
'chrome/qux/qux.properties':
qux,
'chrome/qux/baz/baz.properties':
baz,
'chrome/chrome.manifest':
ManifestFile('chrome', [
ManifestContent('chrome', 'foo', 'foo/'),
ManifestLocale('chrome', 'qux', 'en-US', 'qux/'),
]),
'chrome.manifest':
ManifestFile('', [Manifest('', 'chrome/chrome.manifest')]),
'dict/aa':
dict_aa,
'app/chrome/bar/barbaz.dtd':
barbaz,
'app/chrome/chrome.manifest':
ManifestFile(
'app/chrome',
[ManifestLocale('app/chrome', 'bar', 'en-US', 'bar/')]),
'app/chrome.manifest':
ManifestFile('app', [Manifest('app', 'chrome/chrome.manifest')]),
'app/dict/bb':
dict_bb,
'app/dict/cc':
dict_cc,
'app/chrome/bar/search/foo.xml':
foo,
'app/chrome/bar/search/bar.xml':
bar,
'app/chrome/bar/search/lst.txt':
lst,
})
app_finder.jarlogs = {}
app_finder.base = 'app'
foo_l10n = GeneratedFile('foo_l10n')
qux_l10n = GeneratedFile('qux_l10n')
baz_l10n = GeneratedFile('baz_l10n')
barbaz_l10n = GeneratedFile('barbaz_l10n')
lst_l10n = GeneratedFile('foo\nqux')
l10n_finder = MockFinder({
'chrome/qux-l10n/qux.properties':
qux_l10n,
'chrome/qux-l10n/baz/baz.properties':
baz_l10n,
'chrome/chrome.manifest':
ManifestFile('chrome', [
ManifestLocale('chrome', 'qux', 'x-test', 'qux-l10n/'),
]),
'chrome.manifest':
ManifestFile('', [Manifest('', 'chrome/chrome.manifest')]),
'dict/bb':
dict_bb,
'dict/cc':
dict_cc,
'app/chrome/bar-l10n/barbaz.dtd':
barbaz_l10n,
'app/chrome/chrome.manifest':
ManifestFile(
'app/chrome',
[ManifestLocale('app/chrome', 'bar', 'x-test', 'bar-l10n/')]),
'app/chrome.manifest':
ManifestFile('app', [Manifest('app', 'chrome/chrome.manifest')]),
'app/dict/aa':
dict_aa,
'app/chrome/bar-l10n/search/foo.xml':
foo_l10n,
'app/chrome/bar-l10n/search/qux.xml':
qux_l10n,
'app/chrome/bar-l10n/search/lst.txt':
lst_l10n,
})
l10n_finder.base = 'l10n'
copier = FileRegistry()
formatter = FlatFormatter(copier)
l10n._repack(app_finder, l10n_finder, copier, formatter,
['dict', 'chrome/**/search/*.xml'])
self.maxDiff = None
repacked = {
'bar/foo':
foo,
'chrome/foo/foobar':
foobar,
'chrome/qux-l10n/qux.properties':
qux_l10n,
'chrome/qux-l10n/baz/baz.properties':
baz_l10n,
'chrome/chrome.manifest':
ManifestFile('chrome', [
ManifestContent('chrome', 'foo', 'foo/'),
ManifestLocale('chrome', 'qux', 'x-test', 'qux-l10n/'),
]),
'chrome.manifest':
ManifestFile('', [Manifest('', 'chrome/chrome.manifest')]),
'dict/bb':
dict_bb,
'dict/cc':
dict_cc,
'app/chrome/bar-l10n/barbaz.dtd':
barbaz_l10n,
'app/chrome/chrome.manifest':
ManifestFile(
'app/chrome',
[ManifestLocale('app/chrome', 'bar', 'x-test', 'bar-l10n/')]),
'app/chrome.manifest':
ManifestFile('app', [Manifest('app', 'chrome/chrome.manifest')]),
'app/dict/aa':
dict_aa,
'app/chrome/bar-l10n/search/foo.xml':
foo_l10n,
'app/chrome/bar-l10n/search/qux.xml':
qux_l10n,
'app/chrome/bar-l10n/search/lst.txt':
lst_l10n,
}
self.assertEqual(
dict((p, f.open().read()) for p, f in copier),
dict((p, f.open().read()) for p, f in repacked.iteritems()))
def test_flat_formatter(self):
registry = FileRegistry()
formatter = FlatFormatter(registry)
formatter.add_base('app')
formatter.add('f/oo/bar', GeneratedFile('foobar'))
formatter.add('f/oo/baz', GeneratedFile('foobaz'))
formatter.add('f/oo/qux', GeneratedFile('fooqux'))
formatter.add_manifest(ManifestContent('f/oo', 'bar', 'bar'))
formatter.add_manifest(ManifestContent('f/oo', 'qux', 'qux'))
self.assertEqual(registry.paths(), [
'f/oo/bar', 'f/oo/baz', 'f/oo/qux', 'chrome.manifest',
'f/f.manifest', 'f/oo/oo.manifest'
])
self.assertEqual(registry['chrome.manifest'].open().read(),
'manifest f/f.manifest\n')
self.assertEqual(registry['f/f.manifest'].open().read(),
'manifest oo/oo.manifest\n')
self.assertEqual(registry['f/oo/oo.manifest'].open().read(), ''.join([
'content bar bar\n',
'content qux qux\n',
]))
formatter.add_interfaces('components/foo.xpt', foo_xpt)
formatter.add_interfaces('components/bar.xpt', bar_xpt)
self.assertEqual(registry.paths(), [
'f/oo/bar', 'f/oo/baz', 'f/oo/qux', 'chrome.manifest',
'f/f.manifest', 'f/oo/oo.manifest',
'components/components.manifest', 'components/interfaces.xpt'
])
self.assertEqual(
registry['chrome.manifest'].open().read(), ''.join([
'manifest f/f.manifest\n',
'manifest components/components.manifest\n',
]))
self.assertEqual(
registry['components/components.manifest'].open().read(),
'interfaces interfaces.xpt\n')
registry['components/interfaces.xpt'] \
.copy(self.tmppath('interfaces.xpt'))
linked = read_interfaces(self.tmppath('interfaces.xpt'))
foo = read_interfaces(foo_xpt.open())
bar = read_interfaces(bar_xpt.open())
self.assertEqual(foo['foo'], linked['foo'])
self.assertEqual(bar['bar'], linked['bar'])
formatter.add_manifest(ManifestContent('app/chrome', 'content',
'foo/'))
self.assertEqual(
registry['chrome.manifest'].open().read(), ''.join([
'manifest f/f.manifest\n',
'manifest components/components.manifest\n',
]))
self.assertEqual(registry['app/chrome.manifest'].open().read(),
'manifest chrome/chrome.manifest\n')
self.assertEqual(registry['app/chrome/chrome.manifest'].open().read(),
'content content foo/\n')
def test_l10n_repack(self):
foo = GeneratedFile(b"foo")
foobar = GeneratedFile(b"foobar")
qux = GeneratedFile(b"qux")
bar = GeneratedFile(b"bar")
baz = GeneratedFile(b"baz")
dict_aa = GeneratedFile(b"dict_aa")
dict_bb = GeneratedFile(b"dict_bb")
dict_cc = GeneratedFile(b"dict_cc")
barbaz = GeneratedFile(b"barbaz")
lst = GeneratedFile(b"foo\nbar")
app_finder = MockFinder({
"bar/foo":
foo,
"chrome/foo/foobar":
foobar,
"chrome/qux/qux.properties":
qux,
"chrome/qux/baz/baz.properties":
baz,
"chrome/chrome.manifest":
ManifestFile(
"chrome",
[
ManifestContent("chrome", "foo", "foo/"),
ManifestLocale("chrome", "qux", "en-US", "qux/"),
],
),
"chrome.manifest":
ManifestFile("", [Manifest("", "chrome/chrome.manifest")]),
"dict/aa":
dict_aa,
"app/chrome/bar/barbaz.dtd":
barbaz,
"app/chrome/chrome.manifest":
ManifestFile(
"app/chrome",
[ManifestLocale("app/chrome", "bar", "en-US", "bar/")]),
"app/chrome.manifest":
ManifestFile("app", [Manifest("app", "chrome/chrome.manifest")]),
"app/dict/bb":
dict_bb,
"app/dict/cc":
dict_cc,
"app/chrome/bar/search/foo.xml":
foo,
"app/chrome/bar/search/bar.xml":
bar,
"app/chrome/bar/search/lst.txt":
lst,
})
app_finder.jarlogs = {}
app_finder.base = "app"
foo_l10n = GeneratedFile(b"foo_l10n")
qux_l10n = GeneratedFile(b"qux_l10n")
baz_l10n = GeneratedFile(b"baz_l10n")
barbaz_l10n = GeneratedFile(b"barbaz_l10n")
lst_l10n = GeneratedFile(b"foo\nqux")
l10n_finder = MockFinder({
"chrome/qux-l10n/qux.properties":
qux_l10n,
"chrome/qux-l10n/baz/baz.properties":
baz_l10n,
"chrome/chrome.manifest":
ManifestFile(
"chrome",
[
ManifestLocale("chrome", "qux", "x-test", "qux-l10n/"),
],
),
"chrome.manifest":
ManifestFile("", [Manifest("", "chrome/chrome.manifest")]),
"dict/bb":
dict_bb,
"dict/cc":
dict_cc,
"app/chrome/bar-l10n/barbaz.dtd":
barbaz_l10n,
"app/chrome/chrome.manifest":
ManifestFile(
"app/chrome",
[ManifestLocale("app/chrome", "bar", "x-test", "bar-l10n/")],
),
"app/chrome.manifest":
ManifestFile("app", [Manifest("app", "chrome/chrome.manifest")]),
"app/dict/aa":
dict_aa,
"app/chrome/bar-l10n/search/foo.xml":
foo_l10n,
"app/chrome/bar-l10n/search/qux.xml":
qux_l10n,
"app/chrome/bar-l10n/search/lst.txt":
lst_l10n,
})
l10n_finder.base = "l10n"
copier = FileRegistry()
formatter = FlatFormatter(copier)
l10n._repack(
app_finder,
l10n_finder,
copier,
formatter,
["dict", "chrome/**/search/*.xml"],
)
self.maxDiff = None
repacked = {
"bar/foo":
foo,
"chrome/foo/foobar":
foobar,
"chrome/qux-l10n/qux.properties":
qux_l10n,
"chrome/qux-l10n/baz/baz.properties":
baz_l10n,
"chrome/chrome.manifest":
ManifestFile(
"chrome",
[
ManifestContent("chrome", "foo", "foo/"),
ManifestLocale("chrome", "qux", "x-test", "qux-l10n/"),
],
),
"chrome.manifest":
ManifestFile("", [Manifest("", "chrome/chrome.manifest")]),
"dict/bb":
dict_bb,
"dict/cc":
dict_cc,
"app/chrome/bar-l10n/barbaz.dtd":
barbaz_l10n,
"app/chrome/chrome.manifest":
ManifestFile(
"app/chrome",
[ManifestLocale("app/chrome", "bar", "x-test", "bar-l10n/")],
),
"app/chrome.manifest":
ManifestFile("app", [Manifest("app", "chrome/chrome.manifest")]),
"app/dict/aa":
dict_aa,
"app/chrome/bar-l10n/search/foo.xml":
foo_l10n,
"app/chrome/bar-l10n/search/qux.xml":
qux_l10n,
"app/chrome/bar-l10n/search/lst.txt":
lst_l10n,
}
self.assertEqual(
dict((p, f.open().read()) for p, f in copier),
dict((p, f.open().read()) for p, f in six.iteritems(repacked)),
)
def test_chrome_override(self):
registry = FileRegistry()
f = FlatFormatter(registry)
f.add_base('')
f.add_manifest(ManifestContent('chrome', 'foo', 'foo/unix'))
# A more specific entry for a given chrome name can override a more
# generic one.
f.add_manifest(ManifestContent('chrome', 'foo', 'foo/win', 'os=WINNT'))
f.add_manifest(ManifestContent('chrome', 'foo', 'foo/osx',
'os=Darwin'))
# Chrome with the same name overrides the previous registration.
with self.assertRaises(ErrorMessage) as e:
f.add_manifest(ManifestContent('chrome', 'foo', 'foo/'))
self.assertEqual(
e.exception.message, 'Error: "content foo foo/" overrides '
'"content foo foo/unix"')
# Chrome with the same name and same flags overrides the previous
# registration.
with self.assertRaises(ErrorMessage) as e:
f.add_manifest(ManifestContent('chrome', 'foo', 'foo/',
'os=WINNT'))
self.assertEqual(
e.exception.message,
'Error: "content foo foo/ os=WINNT" overrides '
'"content foo foo/win os=WINNT"')
# We may start with the more specific entry first
f.add_manifest(ManifestContent('chrome', 'bar', 'bar/win', 'os=WINNT'))
# Then adding a more generic one overrides it.
with self.assertRaises(ErrorMessage) as e:
f.add_manifest(ManifestContent('chrome', 'bar', 'bar/unix'))
self.assertEqual(
e.exception.message, 'Error: "content bar bar/unix" overrides '
'"content bar bar/win os=WINNT"')
# Adding something more specific still works.
f.add_manifest(
ManifestContent('chrome', 'bar', 'bar/win',
'os=WINNT osversion>=7.0'))
def test_bases(self):
formatter = FlatFormatter(FileRegistry())
formatter.add_base('')
formatter.add_base('browser')
formatter.add_base('addon0', addon=True)
self.assertEqual(formatter._get_base('platform.ini'),
('', 'platform.ini'))
self.assertEqual(formatter._get_base('browser/application.ini'),
('browser', 'application.ini'))
self.assertEqual(formatter._get_base('addon0/install.rdf'),
('addon0', 'install.rdf'))
def main():
parser = ArgumentParser()
parser.add_argument('-D', dest='defines', action='append',
metavar="VAR[=VAL]", help='Define a variable')
parser.add_argument('--format', default='omni',
help='Choose the chrome format for packaging ' +
'(omni, jar or flat ; default: %(default)s)')
parser.add_argument('--removals', default=None,
help='removed-files source file')
parser.add_argument('--ignore-errors', action='store_true', default=False,
help='Transform errors into warnings.')
parser.add_argument('--minify', action='store_true', default=False,
help='Make some files more compact while packaging')
parser.add_argument('--minify-js', action='store_true',
help='Minify JavaScript files while packaging.')
parser.add_argument('--js-binary',
help='Path to js binary. This is used to verify '
'minified JavaScript. If this is not defined, '
'minification verification will not be performed.')
parser.add_argument('--jarlog', default='', help='File containing jar ' +
'access logs')
parser.add_argument('--optimizejars', action='store_true', default=False,
help='Enable jar optimizations')
parser.add_argument('--disable-compression', action='store_false',
dest='compress', default=True,
help='Disable jar compression')
parser.add_argument('manifest', default=None, nargs='?',
help='Manifest file name')
parser.add_argument('source', help='Source directory')
parser.add_argument('destination', help='Destination directory')
parser.add_argument('--non-resource', nargs='+', metavar='PATTERN',
default=[],
help='Extra files not to be considered as resources')
args = parser.parse_args()
defines = dict(buildconfig.defines)
if args.ignore_errors:
errors.ignore_errors()
if args.defines:
for name, value in [split_define(d) for d in args.defines]:
defines[name] = value
copier = FileCopier()
if args.format == 'flat':
formatter = FlatFormatter(copier)
elif args.format == 'jar':
formatter = JarFormatter(copier, compress=args.compress, optimize=args.optimizejars)
elif args.format == 'omni':
formatter = OmniJarFormatter(copier,
buildconfig.substs['OMNIJAR_NAME'],
compress=args.compress,
optimize=args.optimizejars,
non_resources=args.non_resource)
else:
errors.fatal('Unknown format: %s' % args.format)
# Adjust defines according to the requested format.
if isinstance(formatter, OmniJarFormatter):
defines['MOZ_OMNIJAR'] = 1
elif 'MOZ_OMNIJAR' in defines:
del defines['MOZ_OMNIJAR']
respath = ''
if 'RESPATH' in defines:
respath = SimpleManifestSink.normalize_path(defines['RESPATH'])
while respath.startswith('/'):
respath = respath[1:]
if not buildconfig.substs['CROSS_COMPILE']:
launcher.tooldir = mozpath.join(buildconfig.topobjdir, 'dist')
with errors.accumulate():
finder_args = dict(
minify=args.minify,
minify_js=args.minify_js,
)
if args.js_binary:
finder_args['minify_js_verify_command'] = [
args.js_binary,
os.path.join(os.path.abspath(os.path.dirname(__file__)),
'js-compare-ast.js')
]
finder = FileFinder(args.source, find_executables=True,
**finder_args)
if 'NO_PKG_FILES' in os.environ:
sinkformatter = NoPkgFilesRemover(formatter,
args.manifest is not None)
else:
sinkformatter = formatter
sink = SimpleManifestSink(finder, sinkformatter)
if args.manifest:
preprocess_manifest(sink, args.manifest, defines)
else:
sink.add(Component(''), 'bin/*')
sink.close(args.manifest is not None)
if args.removals:
removals_in = StringIO(open(args.removals).read())
removals_in.name = args.removals
removals = RemovedFiles(copier)
preprocess(removals_in, removals, defines)
copier.add(mozpath.join(respath, 'removed-files'), removals)
# shlibsign libraries
if launcher.can_launch():
if not mozinfo.isMac and buildconfig.substs.get('COMPILE_ENVIRONMENT'):
for lib in SIGN_LIBS:
libbase = mozpath.join(respath, '%s%s') \
% (buildconfig.substs['DLL_PREFIX'], lib)
libname = '%s%s' % (libbase, buildconfig.substs['DLL_SUFFIX'])
if copier.contains(libname):
copier.add(libbase + '.chk',
LibSignFile(os.path.join(args.destination,
libname)))
# Setup preloading
if args.jarlog and os.path.exists(args.jarlog):
from mozpack.mozjar import JarLog
log = JarLog(args.jarlog)
for p, f in copier:
if not isinstance(f, Jarrer):
continue
key = JarLog.canonicalize(os.path.join(args.destination, p))
if key in log:
f.preload(log[key])
copier.copy(args.destination)
generate_precomplete(os.path.normpath(os.path.join(args.destination,
respath)))
def main():
parser = ArgumentParser()
parser.add_argument("-D", dest="defines", action="append", metavar="VAR[=VAL]", help="Define a variable")
parser.add_argument(
"--format",
default="omni",
help="Choose the chrome format for packaging " + "(omni, jar or flat ; default: %(default)s)",
)
parser.add_argument("--removals", default=None, help="removed-files source file")
parser.add_argument("--ignore-errors", action="store_true", default=False, help="Transform errors into warnings.")
parser.add_argument(
"--minify", action="store_true", default=False, help="Make some files more compact while packaging"
)
parser.add_argument("--minify-js", action="store_true", help="Minify JavaScript files while packaging.")
parser.add_argument(
"--js-binary",
help="Path to js binary. This is used to verify "
"minified JavaScript. If this is not defined, "
"minification verification will not be performed.",
)
parser.add_argument("--jarlog", default="", help="File containing jar " + "access logs")
parser.add_argument("--optimizejars", action="store_true", default=False, help="Enable jar optimizations")
parser.add_argument("--unify", default="", help="Base directory of another build to unify with")
parser.add_argument(
"--disable-compression", action="store_false", dest="compress", default=True, help="Disable jar compression"
)
parser.add_argument("manifest", default=None, nargs="?", help="Manifest file name")
parser.add_argument("source", help="Source directory")
parser.add_argument("destination", help="Destination directory")
parser.add_argument(
"--non-resource", nargs="+", metavar="PATTERN", default=[], help="Extra files not to be considered as resources"
)
args = parser.parse_args()
defines = dict(buildconfig.defines)
if args.ignore_errors:
errors.ignore_errors()
if args.defines:
for name, value in [split_define(d) for d in args.defines]:
defines[name] = value
copier = FileCopier()
if args.format == "flat":
formatter = FlatFormatter(copier)
elif args.format == "jar":
formatter = JarFormatter(copier, compress=args.compress, optimize=args.optimizejars)
elif args.format == "omni":
formatter = OmniJarFormatter(
copier,
buildconfig.substs["OMNIJAR_NAME"],
compress=args.compress,
optimize=args.optimizejars,
non_resources=args.non_resource,
)
else:
errors.fatal("Unknown format: %s" % args.format)
# Adjust defines according to the requested format.
if isinstance(formatter, OmniJarFormatter):
defines["MOZ_OMNIJAR"] = 1
elif "MOZ_OMNIJAR" in defines:
del defines["MOZ_OMNIJAR"]
respath = ""
if "RESPATH" in defines:
respath = SimpleManifestSink.normalize_path(defines["RESPATH"])
while respath.startswith("/"):
respath = respath[1:]
if args.unify:
def is_native(path):
path = os.path.abspath(path)
return platform.machine() in mozpath.split(path)
# Invert args.unify and args.source if args.unify points to the
# native architecture.
args.source, args.unify = sorted([args.source, args.unify], key=is_native, reverse=True)
if is_native(args.source):
launcher.tooldir = args.source
elif not buildconfig.substs["CROSS_COMPILE"]:
launcher.tooldir = mozpath.join(buildconfig.topobjdir, "dist")
with errors.accumulate():
finder_args = dict(minify=args.minify, minify_js=args.minify_js)
if args.js_binary:
finder_args["minify_js_verify_command"] = [
args.js_binary,
os.path.join(os.path.abspath(os.path.dirname(__file__)), "js-compare-ast.js"),
]
if args.unify:
finder = UnifiedBuildFinder(FileFinder(args.source), FileFinder(args.unify), **finder_args)
else:
finder = FileFinder(args.source, **finder_args)
if "NO_PKG_FILES" in os.environ:
sinkformatter = NoPkgFilesRemover(formatter, args.manifest is not None)
else:
sinkformatter = formatter
sink = SimpleManifestSink(finder, sinkformatter)
if args.manifest:
preprocess_manifest(sink, args.manifest, defines)
else:
sink.add(Component(""), "bin/*")
sink.close(args.manifest is not None)
if args.removals:
removals_in = StringIO(open(args.removals).read())
removals_in.name = args.removals
removals = RemovedFiles(copier)
preprocess(removals_in, removals, defines)
copier.add(mozpath.join(respath, "removed-files"), removals)
# shlibsign libraries
if launcher.can_launch():
if not mozinfo.isMac and buildconfig.substs.get("COMPILE_ENVIRONMENT"):
for lib in SIGN_LIBS:
libbase = mozpath.join(respath, "%s%s") % (buildconfig.substs["DLL_PREFIX"], lib)
libname = "%s%s" % (libbase, buildconfig.substs["DLL_SUFFIX"])
if copier.contains(libname):
copier.add(libbase + ".chk", LibSignFile(os.path.join(args.destination, libname)))
# Setup preloading
if args.jarlog and os.path.exists(args.jarlog):
from mozpack.mozjar import JarLog
log = JarLog(args.jarlog)
for p, f in copier:
if not isinstance(f, Jarrer):
continue
key = JarLog.canonicalize(os.path.join(args.destination, p))
if key in log:
f.preload(log[key])
# Fill startup cache
if (
isinstance(formatter, OmniJarFormatter)
and launcher.can_launch()
and buildconfig.substs["MOZ_DISABLE_STARTUPCACHE"] != "1"
):
gre_path = None
def get_bases():
for b in sink.packager.get_bases(addons=False):
for p in (mozpath.join("bin", b), b):
if os.path.exists(os.path.join(args.source, p)):
yield p
break
for base in sorted(get_bases()):
if not gre_path:
gre_path = base
omnijar_path = mozpath.join(sink.normalize_path(base), buildconfig.substs["OMNIJAR_NAME"])
if formatter.contains(omnijar_path):
precompile_cache(formatter.copier[omnijar_path], args.source, gre_path, base)
copier.copy(args.destination)
def test_bases(self):
formatter = FlatFormatter(FileRegistry())
formatter.add_base("")
formatter.add_base("addon0", addon=True)
formatter.add_base("browser")
self.assertEqual(formatter._get_base("platform.ini"),
("", "platform.ini"))
self.assertEqual(
formatter._get_base("browser/application.ini"),
("browser", "application.ini"),
)
self.assertEqual(formatter._get_base("addon0/install.rdf"),
("addon0", "install.rdf"))