def test_jarlog(self):
s = six.moves.cStringIO('\n'.join([
'bar/baz.jar first',
'bar/baz.jar second',
'bar/baz.jar third',
'bar/baz.jar second',
'bar/baz.jar second',
'omni.ja stuff',
'bar/baz.jar first',
'omni.ja other/stuff',
'omni.ja stuff',
'bar/baz.jar third',
]))
log = JarLog(fileobj=s)
self.assertEqual(set(log.keys()), set([
'bar/baz.jar',
'omni.ja',
]))
self.assertEqual(log['bar/baz.jar'], [
'first',
'second',
'third',
])
self.assertEqual(log['omni.ja'], [
'stuff',
'other/stuff',
])
def test_jarlog(self):
base = 'file:' + pathname2url(os.path.abspath(os.curdir))
s = StringIO('\n'.join([
base + '/bar/baz.jar first',
base + '/bar/baz.jar second',
base + '/bar/baz.jar third',
base + '/bar/baz.jar second',
base + '/bar/baz.jar second',
'jar:' + base + '/qux.zip!/omni.ja stuff',
base + '/bar/baz.jar first',
'jar:' + base + '/qux.zip!/omni.ja other/stuff',
'jar:' + base + '/qux.zip!/omni.ja stuff',
base + '/bar/baz.jar third',
'jar:jar:' + base + '/qux.zip!/baz/baz.jar!/omni.ja nested/stuff',
'jar:jar:jar:' + base + '/qux.zip!/baz/baz.jar!/foo.zip!/omni.ja' +
' deeply/nested/stuff',
]))
log = JarLog(fileobj=s)
def canonicalize(p):
return mozpath.normsep(os.path.normcase(os.path.realpath(p)))
baz_jar = canonicalize('bar/baz.jar')
qux_zip = canonicalize('qux.zip')
self.assertEqual(set(log.keys()), set([
baz_jar,
(qux_zip, 'omni.ja'),
(qux_zip, 'baz/baz.jar', 'omni.ja'),
(qux_zip, 'baz/baz.jar', 'foo.zip', 'omni.ja'),
]))
self.assertEqual(log[baz_jar], [
'first',
'second',
'third',
])
self.assertEqual(log[(qux_zip, 'omni.ja')], [
'stuff',
'other/stuff',
])
self.assertEqual(log[(qux_zip, 'baz/baz.jar', 'omni.ja')],
['nested/stuff'])
self.assertEqual(log[(qux_zip, 'baz/baz.jar', 'foo.zip',
'omni.ja')], ['deeply/nested/stuff'])
# The above tests also indirectly check the value returned by
# JarLog.canonicalize for various jar: and file: urls, but
# JarLog.canonicalize also supports plain paths.
self.assertEqual(JarLog.canonicalize(os.path.abspath('bar/baz.jar')),
baz_jar)
self.assertEqual(JarLog.canonicalize('bar/baz.jar'), baz_jar)
def test_jarlog(self):
base = 'file:' + pathname2url(os.path.abspath(os.curdir))
s = StringIO('\n'.join([
base + '/bar/baz.jar first',
base + '/bar/baz.jar second',
base + '/bar/baz.jar third',
base + '/bar/baz.jar second',
base + '/bar/baz.jar second',
'jar:' + base + '/qux.zip!/omni.ja stuff',
base + '/bar/baz.jar first',
'jar:' + base + '/qux.zip!/omni.ja other/stuff',
'jar:' + base + '/qux.zip!/omni.ja stuff',
base + '/bar/baz.jar third',
'jar:jar:' + base + '/qux.zip!/baz/baz.jar!/omni.ja nested/stuff',
'jar:jar:jar:' + base + '/qux.zip!/baz/baz.jar!/foo.zip!/omni.ja' +
' deeply/nested/stuff',
]))
log = JarLog(fileobj=s)
canonicalize = lambda p: \
mozpack.path.normsep(os.path.normcase(os.path.realpath(p)))
baz_jar = canonicalize('bar/baz.jar')
qux_zip = canonicalize('qux.zip')
self.assertEqual(set(log.keys()), set([
baz_jar,
(qux_zip, 'omni.ja'),
(qux_zip, 'baz/baz.jar', 'omni.ja'),
(qux_zip, 'baz/baz.jar', 'foo.zip', 'omni.ja'),
]))
self.assertEqual(log[baz_jar], [
'first',
'second',
'third',
])
self.assertEqual(log[(qux_zip, 'omni.ja')], [
'stuff',
'other/stuff',
])
self.assertEqual(log[(qux_zip, 'baz/baz.jar', 'omni.ja')],
['nested/stuff'])
self.assertEqual(log[(qux_zip, 'baz/baz.jar', 'foo.zip',
'omni.ja')], ['deeply/nested/stuff'])
# The above tests also indirectly check the value returned by
# JarLog.canonicalize for various jar: and file: urls, but
# JarLog.canonicalize also supports plain paths.
self.assertEqual(JarLog.canonicalize(os.path.abspath('bar/baz.jar')),
baz_jar)
self.assertEqual(JarLog.canonicalize('bar/baz.jar'), baz_jar)
def test_jarlog(self):
s = six.moves.cStringIO("\n".join([
"bar/baz.jar first",
"bar/baz.jar second",
"bar/baz.jar third",
"bar/baz.jar second",
"bar/baz.jar second",
"omni.ja stuff",
"bar/baz.jar first",
"omni.ja other/stuff",
"omni.ja stuff",
"bar/baz.jar third",
]))
log = JarLog(fileobj=s)
self.assertEqual(
set(log.keys()),
set([
"bar/baz.jar",
"omni.ja",
]),
)
self.assertEqual(
log["bar/baz.jar"],
[
"first",
"second",
"third",
],
)
self.assertEqual(
log["omni.ja"],
[
"stuff",
"other/stuff",
],
)
def main():
parser = ArgumentParser()
parser.add_argument('-D', dest='defines', action='append',
metavar="VAR[=VAL]", help='Define a variable')
parser.add_argument('--format', default='omni',
help='Choose the chrome format for packaging ' +
'(omni, jar or flat ; default: %(default)s)')
parser.add_argument('--removals', default=None,
help='removed-files source file')
parser.add_argument('--ignore-errors', action='store_true', default=False,
help='Transform errors into warnings.')
parser.add_argument('--minify', action='store_true', default=False,
help='Make some files more compact while packaging')
parser.add_argument('--minify-js', action='store_true',
help='Minify JavaScript files while packaging.')
parser.add_argument('--js-binary',
help='Path to js binary. This is used to verify '
'minified JavaScript. If this is not defined, '
'minification verification will not be performed.')
parser.add_argument('--jarlog', default='', help='File containing jar ' +
'access logs')
parser.add_argument('--optimizejars', action='store_true', default=False,
help='Enable jar optimizations')
parser.add_argument('--unify', default='',
help='Base directory of another build to unify with')
parser.add_argument('manifest', default=None, nargs='?',
help='Manifest file name')
parser.add_argument('source', help='Source directory')
parser.add_argument('destination', help='Destination directory')
parser.add_argument('--non-resource', nargs='+', metavar='PATTERN',
default=[],
help='Extra files not to be considered as resources')
args = parser.parse_args()
defines = dict(buildconfig.defines)
if args.ignore_errors:
errors.ignore_errors()
if args.defines:
for name, value in [split_define(d) for d in args.defines]:
defines[name] = value
copier = FileCopier()
if args.format == 'flat':
formatter = FlatFormatter(copier)
elif args.format == 'jar':
formatter = JarFormatter(copier, optimize=args.optimizejars)
elif args.format == 'omni':
formatter = OmniJarFormatter(copier,
buildconfig.substs['OMNIJAR_NAME'],
optimize=args.optimizejars,
non_resources=args.non_resource)
else:
errors.fatal('Unknown format: %s' % args.format)
# Adjust defines according to the requested format.
if isinstance(formatter, OmniJarFormatter):
defines['MOZ_OMNIJAR'] = 1
elif 'MOZ_OMNIJAR' in defines:
del defines['MOZ_OMNIJAR']
respath = ''
if 'RESPATH' in defines:
respath = SimpleManifestSink.normalize_path(defines['RESPATH'])
while respath.startswith('/'):
respath = respath[1:]
if args.unify:
def is_native(path):
path = os.path.abspath(path)
return platform.machine() in mozpath.split(path)
# Invert args.unify and args.source if args.unify points to the
# native architecture.
args.source, args.unify = sorted([args.source, args.unify],
key=is_native, reverse=True)
if is_native(args.source):
launcher.tooldir = args.source
elif not buildconfig.substs['CROSS_COMPILE']:
launcher.tooldir = buildconfig.substs['LIBXUL_DIST']
with errors.accumulate():
finder_args = dict(
minify=args.minify,
minify_js=args.minify_js,
)
if args.js_binary:
finder_args['minify_js_verify_command'] = [
args.js_binary,
os.path.join(os.path.abspath(os.path.dirname(__file__)),
'js-compare-ast.js')
]
if args.unify:
finder = UnifiedBuildFinder(FileFinder(args.source),
FileFinder(args.unify),
**finder_args)
else:
finder = FileFinder(args.source, **finder_args)
if 'NO_PKG_FILES' in os.environ:
sinkformatter = NoPkgFilesRemover(formatter,
args.manifest is not None)
else:
sinkformatter = formatter
sink = SimpleManifestSink(finder, sinkformatter)
if args.manifest:
preprocess_manifest(sink, args.manifest, defines)
else:
sink.add(Component(''), 'bin/*')
sink.close(args.manifest is not None)
if args.removals:
removals_in = StringIO(open(args.removals).read())
removals_in.name = args.removals
removals = RemovedFiles(copier)
preprocess(removals_in, removals, defines)
copier.add(mozpath.join(respath, 'removed-files'), removals)
# shlibsign libraries
if launcher.can_launch():
if not mozinfo.isMac:
for lib in SIGN_LIBS:
libbase = mozpath.join(respath, '%s%s') \
% (buildconfig.substs['DLL_PREFIX'], lib)
libname = '%s%s' % (libbase, buildconfig.substs['DLL_SUFFIX'])
if copier.contains(libname):
copier.add(libbase + '.chk',
LibSignFile(os.path.join(args.destination,
libname)))
# Setup preloading
if args.jarlog and os.path.exists(args.jarlog):
from mozpack.mozjar import JarLog
log = JarLog(args.jarlog)
for p, f in copier:
if not isinstance(f, Jarrer):
continue
key = JarLog.canonicalize(os.path.join(args.destination, p))
if key in log:
f.preload(log[key])
# Fill startup cache
if isinstance(formatter, OmniJarFormatter) and launcher.can_launch() \
and buildconfig.substs['MOZ_DISABLE_STARTUPCACHE'] != '1':
gre_path = None
def get_bases():
for b in sink.packager.get_bases(addons=False):
for p in (mozpath.join('bin', b), b):
if os.path.exists(os.path.join(args.source, p)):
yield p
break
for base in sorted(get_bases()):
if not gre_path:
gre_path = base
base_path = sink.normalize_path(base)
if base_path in formatter.omnijars:
precompile_cache(formatter.omnijars[base_path],
args.source, gre_path, base)
copier.copy(args.destination)
def main():
parser = ArgumentParser()
parser.add_argument('-D',
dest='defines',
action='append',
metavar="VAR[=VAL]",
help='Define a variable')
parser.add_argument('--format',
default='omni',
help='Choose the chrome format for packaging ' +
'(omni, jar or flat ; default: %(default)s)')
parser.add_argument('--removals',
default=None,
help='removed-files source file')
parser.add_argument('--ignore-errors',
action='store_true',
default=False,
help='Transform errors into warnings.')
parser.add_argument('--minify',
action='store_true',
default=False,
help='Make some files more compact while packaging')
parser.add_argument('--minify-js',
action='store_true',
help='Minify JavaScript files while packaging.')
parser.add_argument('--js-binary',
help='Path to js binary. This is used to verify '
'minified JavaScript. If this is not defined, '
'minification verification will not be performed.')
parser.add_argument('--jarlog',
default='',
help='File containing jar ' + 'access logs')
parser.add_argument('--optimizejars',
action='store_true',
default=False,
help='Enable jar optimizations')
parser.add_argument('--unify',
default='',
help='Base directory of another build to unify with')
parser.add_argument('manifest',
default=None,
nargs='?',
help='Manifest file name')
parser.add_argument('source', help='Source directory')
parser.add_argument('destination', help='Destination directory')
parser.add_argument('--non-resource',
nargs='+',
metavar='PATTERN',
default=[],
help='Extra files not to be considered as resources')
args = parser.parse_args()
defines = dict(buildconfig.defines)
if args.ignore_errors:
errors.ignore_errors()
if args.defines:
for name, value in [split_define(d) for d in args.defines]:
defines[name] = value
copier = FileCopier()
if args.format == 'flat':
formatter = FlatFormatter(copier)
elif args.format == 'jar':
formatter = JarFormatter(copier, optimize=args.optimizejars)
elif args.format == 'omni':
formatter = OmniJarFormatter(copier,
buildconfig.substs['OMNIJAR_NAME'],
optimize=args.optimizejars,
non_resources=args.non_resource)
else:
errors.fatal('Unknown format: %s' % args.format)
# Adjust defines according to the requested format.
if isinstance(formatter, OmniJarFormatter):
defines['MOZ_OMNIJAR'] = 1
elif 'MOZ_OMNIJAR' in defines:
del defines['MOZ_OMNIJAR']
binpath = ''
if 'BINPATH' in defines:
binpath = SimpleManifestSink.normalize_path(defines['BINPATH'])
while binpath.startswith('/'):
binpath = binpath[1:]
if args.unify:
def is_native(path):
path = os.path.abspath(path)
return platform.machine() in mozpack.path.split(path)
# Invert args.unify and args.source if args.unify points to the
# native architecture.
args.source, args.unify = sorted([args.source, args.unify],
key=is_native,
reverse=True)
if is_native(args.source):
launcher.tooldir = args.source
elif not buildconfig.substs['CROSS_COMPILE']:
launcher.tooldir = buildconfig.substs['LIBXUL_DIST']
with errors.accumulate():
finder_args = dict(
minify=args.minify,
minify_js=args.minify_js,
)
if args.js_binary:
finder_args['minify_js_verify_command'] = [
args.js_binary,
os.path.join(os.path.abspath(os.path.dirname(__file__)),
'js-compare-ast.js')
]
if args.unify:
finder = UnifiedBuildFinder(FileFinder(args.source),
FileFinder(args.unify), **finder_args)
else:
finder = FileFinder(args.source, **finder_args)
if 'NO_PKG_FILES' in os.environ:
sinkformatter = NoPkgFilesRemover(formatter, args.manifest
is not None)
else:
sinkformatter = formatter
sink = SimpleManifestSink(finder, sinkformatter)
if args.manifest:
preprocess_manifest(sink, args.manifest, defines)
else:
sink.add(Component(''), 'bin/*')
sink.close(args.manifest is not None)
if args.removals:
lines = [l.lstrip() for l in open(args.removals).readlines()]
removals_in = StringIO(''.join(lines))
removals_in.name = args.removals
removals = RemovedFiles(copier)
preprocess(removals_in, removals, defines)
copier.add(mozpack.path.join(binpath, 'removed-files'), removals)
# shlibsign libraries
if launcher.can_launch():
for lib in SIGN_LIBS:
libbase = mozpack.path.join(binpath, '%s%s') \
% (buildconfig.substs['DLL_PREFIX'], lib)
libname = '%s%s' % (libbase, buildconfig.substs['DLL_SUFFIX'])
if copier.contains(libname):
copier.add(
libbase + '.chk',
LibSignFile(os.path.join(args.destination, libname)))
# Setup preloading
if args.jarlog and os.path.exists(args.jarlog):
from mozpack.mozjar import JarLog
log = JarLog(args.jarlog)
for p, f in copier:
if not isinstance(f, Jarrer):
continue
key = JarLog.canonicalize(os.path.join(args.destination, p))
if key in log:
f.preload(log[key])
# Fill startup cache
if isinstance(formatter, OmniJarFormatter) and launcher.can_launch() \
and buildconfig.substs['MOZ_DISABLE_STARTUPCACHE'] != '1':
if buildconfig.substs['LIBXUL_SDK']:
gre_path = mozpack.path.join(buildconfig.substs['LIBXUL_DIST'],
'bin')
else:
gre_path = None
for base in sorted([[
p for p in [mozpack.path.join('bin', b), b]
if os.path.exists(os.path.join(args.source, p))
][0] for b in sink.packager.get_bases()]):
if not gre_path:
gre_path = base
base_path = sink.normalize_path(base)
if base_path in formatter.omnijars:
precompile_cache(formatter.omnijars[base_path], args.source,
gre_path, base)
copier.copy(args.destination)
generate_precomplete(
os.path.normpath(os.path.join(args.destination, binpath)))
def main():
parser = ArgumentParser()
parser.add_argument(
"-D",
dest="defines",
action="append",
metavar="VAR[=VAL]",
help="Define a variable",
)
parser.add_argument(
"--format",
default="omni",
help="Choose the chrome format for packaging " +
"(omni, jar or flat ; default: %(default)s)",
)
parser.add_argument("--removals",
default=None,
help="removed-files source file")
parser.add_argument(
"--ignore-errors",
action="store_true",
default=False,
help="Transform errors into warnings.",
)
parser.add_argument(
"--ignore-broken-symlinks",
action="store_true",
default=False,
help="Do not fail when processing broken symlinks.",
)
parser.add_argument(
"--minify",
action="store_true",
default=False,
help="Make some files more compact while packaging",
)
parser.add_argument(
"--minify-js",
action="store_true",
help="Minify JavaScript files while packaging.",
)
parser.add_argument(
"--js-binary",
help="Path to js binary. This is used to verify "
"minified JavaScript. If this is not defined, "
"minification verification will not be performed.",
)
parser.add_argument("--jarlog",
default="",
help="File containing jar " + "access logs")
parser.add_argument(
"--compress",
choices=("none", "deflate"),
default="deflate",
help="Use given jar compression (default: deflate)",
)
parser.add_argument("manifest",
default=None,
nargs="?",
help="Manifest file name")
parser.add_argument("source", help="Source directory")
parser.add_argument("destination", help="Destination directory")
parser.add_argument(
"--non-resource",
nargs="+",
metavar="PATTERN",
default=[],
help="Extra files not to be considered as resources",
)
args = parser.parse_args()
defines = dict(buildconfig.defines["ALLDEFINES"])
if args.ignore_errors:
errors.ignore_errors()
if args.defines:
for name, value in [split_define(d) for d in args.defines]:
defines[name] = value
compress = {
"none": False,
"deflate": True,
}[args.compress]
copier = FileCopier()
if args.format == "flat":
formatter = FlatFormatter(copier)
elif args.format == "jar":
formatter = JarFormatter(copier, compress=compress)
elif args.format == "omni":
formatter = OmniJarFormatter(
copier,
buildconfig.substs["OMNIJAR_NAME"],
compress=compress,
non_resources=args.non_resource,
)
else:
errors.fatal("Unknown format: %s" % args.format)
# Adjust defines according to the requested format.
if isinstance(formatter, OmniJarFormatter):
defines["MOZ_OMNIJAR"] = 1
elif "MOZ_OMNIJAR" in defines:
del defines["MOZ_OMNIJAR"]
respath = ""
if "RESPATH" in defines:
respath = SimpleManifestSink.normalize_path(defines["RESPATH"])
while respath.startswith("/"):
respath = respath[1:]
with errors.accumulate():
finder_args = dict(
minify=args.minify,
minify_js=args.minify_js,
ignore_broken_symlinks=args.ignore_broken_symlinks,
)
if args.js_binary:
finder_args["minify_js_verify_command"] = [
args.js_binary,
os.path.join(os.path.abspath(os.path.dirname(__file__)),
"js-compare-ast.js"),
]
finder = PackagerFileFinder(args.source,
find_executables=True,
**finder_args)
if "NO_PKG_FILES" in os.environ:
sinkformatter = NoPkgFilesRemover(formatter, args.manifest
is not None)
else:
sinkformatter = formatter
sink = SimpleManifestSink(finder, sinkformatter)
if args.manifest:
preprocess_manifest(sink, args.manifest, defines)
else:
sink.add(Component(""), "bin/*")
sink.close(args.manifest is not None)
if args.removals:
removals_in = StringIO(open(args.removals).read())
removals_in.name = args.removals
removals = RemovedFiles(copier)
preprocess(removals_in, removals, defines)
copier.add(mozpath.join(respath, "removed-files"), removals)
# If a pdb file is present and we were instructed to copy it, include it.
# Run on all OSes to capture MinGW builds
if buildconfig.substs.get("MOZ_COPY_PDBS"):
# We want to mutate the copier while we're iterating through it, so copy
# the items to a list first.
copier_items = [(p, f) for p, f in copier]
for p, f in copier_items:
if isinstance(f, ExecutableFile):
pdbname = os.path.splitext(f.inputs()[0])[0] + ".pdb"
if os.path.exists(pdbname):
copier.add(os.path.basename(pdbname), File(pdbname))
# Setup preloading
if args.jarlog:
if not os.path.exists(args.jarlog):
raise Exception("Cannot find jar log: %s" % args.jarlog)
omnijars = []
if isinstance(formatter, OmniJarFormatter):
omnijars = [
mozpath.join(base, buildconfig.substs["OMNIJAR_NAME"])
for base in sink.packager.get_bases(addons=False)
]
from mozpack.mozjar import JarLog
log = JarLog(args.jarlog)
for p, f in copier:
if not isinstance(f, Jarrer):
continue
if respath:
p = mozpath.relpath(p, respath)
if p in log:
f.preload(log[p])
elif p in omnijars:
raise Exception("No jar log data for %s" % p)
copier.copy(args.destination)
generate_precomplete(
os.path.normpath(os.path.join(args.destination, respath)))
def main():
parser = ArgumentParser()
parser.add_argument('-D', dest='defines', action='append',
metavar="VAR[=VAL]", help='Define a variable')
parser.add_argument('--format', default='omni',
help='Choose the chrome format for packaging ' +
'(omni, jar or flat ; default: %(default)s)')
parser.add_argument('--removals', default=None,
help='removed-files source file')
parser.add_argument('--ignore-errors', action='store_true', default=False,
help='Transform errors into warnings.')
parser.add_argument('--minify', action='store_true', default=False,
help='Make some files more compact while packaging')
parser.add_argument('--jarlog', default='', help='File containing jar ' +
'access logs')
parser.add_argument('--optimizejars', action='store_true', default=False,
help='Enable jar optimizations')
parser.add_argument('--unify', default='',
help='Base directory of another build to unify with')
parser.add_argument('manifest', default=None, nargs='?',
help='Manifest file name')
parser.add_argument('source', help='Source directory')
parser.add_argument('destination', help='Destination directory')
parser.add_argument('--non-resource', nargs='+', metavar='PATTERN',
default=[],
help='Extra files not to be considered as resources')
args = parser.parse_args()
defines = dict(buildconfig.defines)
if args.ignore_errors:
errors.ignore_errors()
if args.defines:
for name, value in [split_define(d) for d in args.defines]:
defines[name] = value
copier = FileCopier()
if args.format == 'flat':
formatter = FlatFormatter(copier)
elif args.format == 'jar':
formatter = JarFormatter(copier, optimize=args.optimizejars)
elif args.format == 'omni':
formatter = OmniJarFormatter(copier,
buildconfig.substs['OMNIJAR_NAME'],
optimize=args.optimizejars,
non_resources=args.non_resource)
else:
errors.fatal('Unknown format: %s' % args.format)
# Adjust defines according to the requested format.
if isinstance(formatter, OmniJarFormatter):
defines['MOZ_OMNIJAR'] = 1
elif 'MOZ_OMNIJAR' in defines:
del defines['MOZ_OMNIJAR']
binpath = ''
if 'BINPATH' in defines:
binpath = SimpleManifestSink.normalize_path(defines['BINPATH'])
while binpath.startswith('/'):
binpath = binpath[1:]
if args.unify:
def is_native(path):
path = os.path.abspath(path)
return platform.machine() in mozpack.path.split(path)
# Invert args.unify and args.source if args.unify points to the
# native architecture.
args.source, args.unify = sorted([args.source, args.unify],
key=is_native, reverse=True)
if is_native(args.source):
launcher.tooldir = args.source
elif not buildconfig.substs['CROSS_COMPILE']:
launcher.tooldir = buildconfig.substs['LIBXUL_DIST']
with errors.accumulate():
if args.unify:
finder = UnifiedBuildFinder(FileFinder(args.source),
FileFinder(args.unify),
minify=args.minify)
else:
finder = FileFinder(args.source, minify=args.minify)
if 'NO_PKG_FILES' in os.environ:
sinkformatter = NoPkgFilesRemover(formatter,
args.manifest is not None)
else:
sinkformatter = formatter
sink = SimpleManifestSink(finder, sinkformatter)
if args.manifest:
preprocess_manifest(sink, args.manifest, defines)
else:
sink.add(Component(''), 'bin/*')
sink.close(args.manifest is not None)
if args.removals:
lines = [l.lstrip() for l in open(args.removals).readlines()]
removals_in = StringIO(''.join(lines))
removals_in.name = args.removals
removals = RemovedFiles(copier)
preprocess(removals_in, removals, defines)
copier.add(mozpack.path.join(binpath, 'removed-files'), removals)
# shlibsign libraries
if launcher.can_launch():
for lib in SIGN_LIBS:
libbase = mozpack.path.join(binpath, '%s%s') \
% (buildconfig.substs['DLL_PREFIX'], lib)
libname = '%s%s' % (libbase, buildconfig.substs['DLL_SUFFIX'])
if copier.contains(libname):
copier.add(libbase + '.chk',
LibSignFile(os.path.join(args.destination,
libname)))
# Setup preloading
if args.jarlog and os.path.exists(args.jarlog):
from mozpack.mozjar import JarLog
log = JarLog(args.jarlog)
for p, f in copier:
if not isinstance(f, Jarrer):
continue
key = JarLog.canonicalize(os.path.join(args.destination, p))
if key in log:
f.preload(log[key])
# Fill startup cache on Windows and Linux only
# (this currently causes build failure on BSD, so skip on that platfom)
if sys.platform == 'win32' or sys.platform.startswith ('linux'):
if isinstance(formatter, OmniJarFormatter) and launcher.can_launch():
if buildconfig.substs['LIBXUL_SDK']:
gre_path = mozpack.path.join(buildconfig.substs['LIBXUL_DIST'],
'bin')
else:
gre_path = None
for base in sorted([[p for p in [mozpack.path.join('bin', b), b]
if os.path.exists(os.path.join(args.source, p))][0]
for b in sink.packager.get_bases()]):
if not gre_path:
gre_path = base
base_path = sink.normalize_path(base)
if base_path in formatter.omnijars:
precompile_cache(formatter.omnijars[base_path],
args.source, gre_path, base)
copier.copy(args.destination)
generate_precomplete(os.path.normpath(os.path.join(args.destination,
binpath)))
def main():
parser = ArgumentParser()
parser.add_argument('-D', dest='defines', action='append',
metavar="VAR[=VAL]", help='Define a variable')
parser.add_argument('--format', default='omni',
help='Choose the chrome format for packaging ' +
'(omni, jar or flat ; default: %(default)s)')
parser.add_argument('--removals', default=None,
help='removed-files source file')
parser.add_argument('--ignore-errors', action='store_true', default=False,
help='Transform errors into warnings.')
parser.add_argument('--ignore-broken-symlinks', action='store_true', default=False,
help='Do not fail when processing broken symlinks.')
parser.add_argument('--minify', action='store_true', default=False,
help='Make some files more compact while packaging')
parser.add_argument('--minify-js', action='store_true',
help='Minify JavaScript files while packaging.')
parser.add_argument('--js-binary',
help='Path to js binary. This is used to verify '
'minified JavaScript. If this is not defined, '
'minification verification will not be performed.')
parser.add_argument('--jarlog', default='', help='File containing jar ' +
'access logs')
parser.add_argument('--compress', choices=('none', 'deflate', 'brotli'),
default='deflate',
help='Use given jar compression (default: deflate)')
parser.add_argument('manifest', default=None, nargs='?',
help='Manifest file name')
parser.add_argument('source', help='Source directory')
parser.add_argument('destination', help='Destination directory')
parser.add_argument('--non-resource', nargs='+', metavar='PATTERN',
default=[],
help='Extra files not to be considered as resources')
args = parser.parse_args()
defines = dict(buildconfig.defines['ALLDEFINES'])
if args.ignore_errors:
errors.ignore_errors()
if args.defines:
for name, value in [split_define(d) for d in args.defines]:
defines[name] = value
compress = {
'none': False,
'deflate': True,
'brotli': JAR_BROTLI,
}[args.compress]
copier = FileCopier()
if args.format == 'flat':
formatter = FlatFormatter(copier)
elif args.format == 'jar':
formatter = JarFormatter(copier, compress=compress)
elif args.format == 'omni':
formatter = OmniJarFormatter(copier,
buildconfig.substs['OMNIJAR_NAME'],
compress=compress,
non_resources=args.non_resource)
else:
errors.fatal('Unknown format: %s' % args.format)
# Adjust defines according to the requested format.
if isinstance(formatter, OmniJarFormatter):
defines['MOZ_OMNIJAR'] = 1
elif 'MOZ_OMNIJAR' in defines:
del defines['MOZ_OMNIJAR']
respath = ''
if 'RESPATH' in defines:
respath = SimpleManifestSink.normalize_path(defines['RESPATH'])
while respath.startswith('/'):
respath = respath[1:]
if not buildconfig.substs['CROSS_COMPILE']:
launcher.tooldir = mozpath.join(buildconfig.topobjdir, 'dist')
with errors.accumulate():
finder_args = dict(
minify=args.minify,
minify_js=args.minify_js,
ignore_broken_symlinks=args.ignore_broken_symlinks,
)
if args.js_binary:
finder_args['minify_js_verify_command'] = [
args.js_binary,
os.path.join(os.path.abspath(os.path.dirname(__file__)),
'js-compare-ast.js')
]
finder = FileFinder(args.source, find_executables=True,
**finder_args)
if 'NO_PKG_FILES' in os.environ:
sinkformatter = NoPkgFilesRemover(formatter,
args.manifest is not None)
else:
sinkformatter = formatter
sink = SimpleManifestSink(finder, sinkformatter)
if args.manifest:
preprocess_manifest(sink, args.manifest, defines)
else:
sink.add(Component(''), 'bin/*')
sink.close(args.manifest is not None)
if args.removals:
removals_in = StringIO(open(args.removals).read())
removals_in.name = args.removals
removals = RemovedFiles(copier)
preprocess(removals_in, removals, defines)
copier.add(mozpath.join(respath, 'removed-files'), removals)
# shlibsign libraries
if launcher.can_launch():
if not mozinfo.isMac and buildconfig.substs.get('COMPILE_ENVIRONMENT'):
for lib in SIGN_LIBS:
libbase = mozpath.join(respath, '%s%s') \
% (buildconfig.substs['DLL_PREFIX'], lib)
libname = '%s%s' % (libbase, buildconfig.substs['DLL_SUFFIX'])
if copier.contains(libname):
copier.add(libbase + '.chk',
LibSignFile(os.path.join(args.destination,
libname)))
# If a pdb file is present and we were instructed to copy it, include it.
# Run on all OSes to capture MinGW builds
if buildconfig.substs.get('MOZ_COPY_PDBS'):
for p, f in copier:
if isinstance(f, ExecutableFile):
pdbname = os.path.splitext(f.inputs()[0])[0] + '.pdb'
if os.path.exists(pdbname):
copier.add(os.path.basename(pdbname), File(pdbname))
# Setup preloading
if args.jarlog:
if not os.path.exists(args.jarlog):
raise Exception('Cannot find jar log: %s' % args.jarlog)
omnijars = []
if isinstance(formatter, OmniJarFormatter):
omnijars = [mozpath.join(base, buildconfig.substs['OMNIJAR_NAME'])
for base in sink.packager.get_bases(addons=False)]
from mozpack.mozjar import JarLog
log = JarLog(args.jarlog)
for p, f in copier:
if not isinstance(f, Jarrer):
continue
if respath:
p = mozpath.relpath(p, respath)
if p in log:
f.preload(log[p])
elif p in omnijars:
raise Exception('No jar log data for %s' % p)
copier.copy(args.destination)
generate_precomplete(os.path.normpath(os.path.join(args.destination,
respath)))
def main():
parser = ArgumentParser()
parser.add_argument('-D', dest='defines', action='append',
metavar="VAR[=VAL]", help='Define a variable')
parser.add_argument('--format', default='omni',
help='Choose the chrome format for packaging ' +
'(omni, jar or flat ; default: %(default)s)')
parser.add_argument('--removals', default=None,
help='removed-files source file')
parser.add_argument('--ignore-errors', action='store_true', default=False,
help='Transform errors into warnings.')
parser.add_argument('--minify', action='store_true', default=False,
help='Make some files more compact while packaging')
parser.add_argument('--minify-js', action='store_true',
help='Minify JavaScript files while packaging.')
parser.add_argument('--js-binary',
help='Path to js binary. This is used to verify '
'minified JavaScript. If this is not defined, '
'minification verification will not be performed.')
parser.add_argument('--jarlog', default='', help='File containing jar ' +
'access logs')
parser.add_argument('--optimizejars', action='store_true', default=False,
help='Enable jar optimizations')
parser.add_argument('--disable-compression', action='store_false',
dest='compress', default=True,
help='Disable jar compression')
parser.add_argument('manifest', default=None, nargs='?',
help='Manifest file name')
parser.add_argument('source', help='Source directory')
parser.add_argument('destination', help='Destination directory')
parser.add_argument('--non-resource', nargs='+', metavar='PATTERN',
default=[],
help='Extra files not to be considered as resources')
args = parser.parse_args()
defines = dict(buildconfig.defines)
if args.ignore_errors:
errors.ignore_errors()
if args.defines:
for name, value in [split_define(d) for d in args.defines]:
defines[name] = value
copier = FileCopier()
if args.format == 'flat':
formatter = FlatFormatter(copier)
elif args.format == 'jar':
formatter = JarFormatter(copier, compress=args.compress, optimize=args.optimizejars)
elif args.format == 'omni':
formatter = OmniJarFormatter(copier,
buildconfig.substs['OMNIJAR_NAME'],
compress=args.compress,
optimize=args.optimizejars,
non_resources=args.non_resource)
else:
errors.fatal('Unknown format: %s' % args.format)
# Adjust defines according to the requested format.
if isinstance(formatter, OmniJarFormatter):
defines['MOZ_OMNIJAR'] = 1
elif 'MOZ_OMNIJAR' in defines:
del defines['MOZ_OMNIJAR']
respath = ''
if 'RESPATH' in defines:
respath = SimpleManifestSink.normalize_path(defines['RESPATH'])
while respath.startswith('/'):
respath = respath[1:]
if not buildconfig.substs['CROSS_COMPILE']:
launcher.tooldir = mozpath.join(buildconfig.topobjdir, 'dist')
with errors.accumulate():
finder_args = dict(
minify=args.minify,
minify_js=args.minify_js,
)
if args.js_binary:
finder_args['minify_js_verify_command'] = [
args.js_binary,
os.path.join(os.path.abspath(os.path.dirname(__file__)),
'js-compare-ast.js')
]
finder = FileFinder(args.source, find_executables=True,
**finder_args)
if 'NO_PKG_FILES' in os.environ:
sinkformatter = NoPkgFilesRemover(formatter,
args.manifest is not None)
else:
sinkformatter = formatter
sink = SimpleManifestSink(finder, sinkformatter)
if args.manifest:
preprocess_manifest(sink, args.manifest, defines)
else:
sink.add(Component(''), 'bin/*')
sink.close(args.manifest is not None)
if args.removals:
removals_in = StringIO(open(args.removals).read())
removals_in.name = args.removals
removals = RemovedFiles(copier)
preprocess(removals_in, removals, defines)
copier.add(mozpath.join(respath, 'removed-files'), removals)
# shlibsign libraries
if launcher.can_launch():
if not mozinfo.isMac and buildconfig.substs.get('COMPILE_ENVIRONMENT'):
for lib in SIGN_LIBS:
libbase = mozpath.join(respath, '%s%s') \
% (buildconfig.substs['DLL_PREFIX'], lib)
libname = '%s%s' % (libbase, buildconfig.substs['DLL_SUFFIX'])
if copier.contains(libname):
copier.add(libbase + '.chk',
LibSignFile(os.path.join(args.destination,
libname)))
# Setup preloading
if args.jarlog and os.path.exists(args.jarlog):
from mozpack.mozjar import JarLog
log = JarLog(args.jarlog)
for p, f in copier:
if not isinstance(f, Jarrer):
continue
key = JarLog.canonicalize(os.path.join(args.destination, p))
if key in log:
f.preload(log[key])
copier.copy(args.destination)
generate_precomplete(os.path.normpath(os.path.join(args.destination,
respath)))
def main():
parser = ArgumentParser()
parser.add_argument("-D", dest="defines", action="append", metavar="VAR[=VAL]", help="Define a variable")
parser.add_argument(
"--format",
default="omni",
help="Choose the chrome format for packaging " + "(omni, jar or flat ; default: %(default)s)",
)
parser.add_argument("--removals", default=None, help="removed-files source file")
parser.add_argument("--ignore-errors", action="store_true", default=False, help="Transform errors into warnings.")
parser.add_argument(
"--minify", action="store_true", default=False, help="Make some files more compact while packaging"
)
parser.add_argument("--minify-js", action="store_true", help="Minify JavaScript files while packaging.")
parser.add_argument(
"--js-binary",
help="Path to js binary. This is used to verify "
"minified JavaScript. If this is not defined, "
"minification verification will not be performed.",
)
parser.add_argument("--jarlog", default="", help="File containing jar " + "access logs")
parser.add_argument("--optimizejars", action="store_true", default=False, help="Enable jar optimizations")
parser.add_argument("--unify", default="", help="Base directory of another build to unify with")
parser.add_argument(
"--disable-compression", action="store_false", dest="compress", default=True, help="Disable jar compression"
)
parser.add_argument("manifest", default=None, nargs="?", help="Manifest file name")
parser.add_argument("source", help="Source directory")
parser.add_argument("destination", help="Destination directory")
parser.add_argument(
"--non-resource", nargs="+", metavar="PATTERN", default=[], help="Extra files not to be considered as resources"
)
args = parser.parse_args()
defines = dict(buildconfig.defines)
if args.ignore_errors:
errors.ignore_errors()
if args.defines:
for name, value in [split_define(d) for d in args.defines]:
defines[name] = value
copier = FileCopier()
if args.format == "flat":
formatter = FlatFormatter(copier)
elif args.format == "jar":
formatter = JarFormatter(copier, compress=args.compress, optimize=args.optimizejars)
elif args.format == "omni":
formatter = OmniJarFormatter(
copier,
buildconfig.substs["OMNIJAR_NAME"],
compress=args.compress,
optimize=args.optimizejars,
non_resources=args.non_resource,
)
else:
errors.fatal("Unknown format: %s" % args.format)
# Adjust defines according to the requested format.
if isinstance(formatter, OmniJarFormatter):
defines["MOZ_OMNIJAR"] = 1
elif "MOZ_OMNIJAR" in defines:
del defines["MOZ_OMNIJAR"]
respath = ""
if "RESPATH" in defines:
respath = SimpleManifestSink.normalize_path(defines["RESPATH"])
while respath.startswith("/"):
respath = respath[1:]
if args.unify:
def is_native(path):
path = os.path.abspath(path)
return platform.machine() in mozpath.split(path)
# Invert args.unify and args.source if args.unify points to the
# native architecture.
args.source, args.unify = sorted([args.source, args.unify], key=is_native, reverse=True)
if is_native(args.source):
launcher.tooldir = args.source
elif not buildconfig.substs["CROSS_COMPILE"]:
launcher.tooldir = mozpath.join(buildconfig.topobjdir, "dist")
with errors.accumulate():
finder_args = dict(minify=args.minify, minify_js=args.minify_js)
if args.js_binary:
finder_args["minify_js_verify_command"] = [
args.js_binary,
os.path.join(os.path.abspath(os.path.dirname(__file__)), "js-compare-ast.js"),
]
if args.unify:
finder = UnifiedBuildFinder(FileFinder(args.source), FileFinder(args.unify), **finder_args)
else:
finder = FileFinder(args.source, **finder_args)
if "NO_PKG_FILES" in os.environ:
sinkformatter = NoPkgFilesRemover(formatter, args.manifest is not None)
else:
sinkformatter = formatter
sink = SimpleManifestSink(finder, sinkformatter)
if args.manifest:
preprocess_manifest(sink, args.manifest, defines)
else:
sink.add(Component(""), "bin/*")
sink.close(args.manifest is not None)
if args.removals:
removals_in = StringIO(open(args.removals).read())
removals_in.name = args.removals
removals = RemovedFiles(copier)
preprocess(removals_in, removals, defines)
copier.add(mozpath.join(respath, "removed-files"), removals)
# shlibsign libraries
if launcher.can_launch():
if not mozinfo.isMac and buildconfig.substs.get("COMPILE_ENVIRONMENT"):
for lib in SIGN_LIBS:
libbase = mozpath.join(respath, "%s%s") % (buildconfig.substs["DLL_PREFIX"], lib)
libname = "%s%s" % (libbase, buildconfig.substs["DLL_SUFFIX"])
if copier.contains(libname):
copier.add(libbase + ".chk", LibSignFile(os.path.join(args.destination, libname)))
# Setup preloading
if args.jarlog and os.path.exists(args.jarlog):
from mozpack.mozjar import JarLog
log = JarLog(args.jarlog)
for p, f in copier:
if not isinstance(f, Jarrer):
continue
key = JarLog.canonicalize(os.path.join(args.destination, p))
if key in log:
f.preload(log[key])
# Fill startup cache
if (
isinstance(formatter, OmniJarFormatter)
and launcher.can_launch()
and buildconfig.substs["MOZ_DISABLE_STARTUPCACHE"] != "1"
):
gre_path = None
def get_bases():
for b in sink.packager.get_bases(addons=False):
for p in (mozpath.join("bin", b), b):
if os.path.exists(os.path.join(args.source, p)):
yield p
break
for base in sorted(get_bases()):
if not gre_path:
gre_path = base
omnijar_path = mozpath.join(sink.normalize_path(base), buildconfig.substs["OMNIJAR_NAME"])
if formatter.contains(omnijar_path):
precompile_cache(formatter.copier[omnijar_path], args.source, gre_path, base)
copier.copy(args.destination)