def test_optional_exists_creates_unneeded_directory(self): """Demonstrate that a directory not strictly required, but specified as the path to an optional file, will be unnecessarily created. This behaviour is wrong; fixing it is tracked by Bug 972432; and this test exists to guard against unexpected changes in behaviour. """ dest = self.tmppath("dest") copier = FileCopier() copier.add("foo/bar", ExistingFile(required=False)) result = copier.copy(dest) st = os.lstat(self.tmppath("dest/foo")) self.assertFalse(stat.S_ISLNK(st.st_mode)) self.assertTrue(stat.S_ISDIR(st.st_mode)) # What's worse, we have no record that dest was created. self.assertEquals(len(result.updated_files), 0) # But we do have an erroneous record of an optional file # existing when it does not. self.assertIn(self.tmppath("dest/foo/bar"), result.existing_files)
def output_changes(self, verbose=True): ''' Return an iterator of `FasterBuildChange` instances as outputs from the faster build system are updated. ''' for change in self.input_changes(verbose=verbose): now = datetime.datetime.utcnow() for unrecognized in sorted(change.unrecognized): print_line('watch', '! {}'.format(unrecognized), now=now) all_outputs = set() for input in sorted(change.input_to_outputs): outputs = change.input_to_outputs[input] print_line('watch', '< {}'.format(input), now=now) for output in sorted(outputs): print_line('watch', '> {}'.format(output), now=now) all_outputs |= outputs if all_outputs: partial_copier = FileCopier() for output in all_outputs: partial_copier.add(output, self.file_copier[output]) self.incremental_copy(partial_copier, force=True, verbose=verbose) yield change
def test_symlink_directory_replaced(self): """Directory symlinks in destination are replaced if they need to be real directories.""" if not self.symlink_supported: return dest = self.tmppath('dest') copier = FileCopier() copier.add('foo/bar/baz', GeneratedFile('foobarbaz')) os.makedirs(self.tmppath('dest/foo')) dummy = self.tmppath('dummy') os.mkdir(dummy) link = self.tmppath('dest/foo/bar') os.symlink(dummy, link) result = copier.copy(dest) st = os.lstat(link) self.assertFalse(stat.S_ISLNK(st.st_mode)) self.assertTrue(stat.S_ISDIR(st.st_mode)) self.assertEqual(self.all_files(dest), set(copier.paths())) self.assertEqual(result.removed_directories, set()) self.assertEqual(len(result.updated_files), 1)
def test_symlink_directory(self): """Directory symlinks in destination are deleted.""" if not self.symlink_supported: return dest = self.tmppath('dest') copier = FileCopier() copier.add('foo/bar/baz', GeneratedFile('foobarbaz')) os.makedirs(self.tmppath('dest/foo')) dummy = self.tmppath('dummy') os.mkdir(dummy) link = self.tmppath('dest/foo/bar') os.symlink(dummy, link) result = copier.copy(dest) st = os.lstat(link) self.assertFalse(stat.S_ISLNK(st.st_mode)) self.assertTrue(stat.S_ISDIR(st.st_mode)) self.assertEqual(self.all_files(dest), set(copier.paths())) self.assertEqual(result.removed_directories, set()) self.assertEqual(len(result.updated_files), 1)
def main(): parser = argparse.ArgumentParser( description="Merge two directories, creating Universal binaries for " "executables and libraries they contain.") parser.add_argument("dir1", help="Directory") parser.add_argument("dir2", help="Directory to merge") options = parser.parse_args() buildconfig.substs["OS_ARCH"] = "Darwin" buildconfig.substs["LIPO"] = os.environ.get("LIPO") dir1_finder = FileFinder(options.dir1, find_executables=True, find_dotfiles=True) dir2_finder = FileFinder(options.dir2, find_executables=True, find_dotfiles=True) finder = UnifiedTestFinder(dir1_finder, dir2_finder) copier = FileCopier() with errors.accumulate(): for p, f in finder: copier.add(p, f) copier.copy(options.dir1, skip_if_older=False)
def test_symlink_directory_replaced(self): """Directory symlinks in destination are replaced if they need to be real directories.""" if not self.symlink_supported: return dest = self.tmppath("dest") copier = FileCopier() copier.add("foo/bar/baz", GeneratedFile(b"foobarbaz")) os.makedirs(self.tmppath("dest/foo")) dummy = self.tmppath("dummy") os.mkdir(dummy) link = self.tmppath("dest/foo/bar") os.symlink(dummy, link) result = copier.copy(dest) st = os.lstat(link) self.assertFalse(stat.S_ISLNK(st.st_mode)) self.assertTrue(stat.S_ISDIR(st.st_mode)) self.assertEqual(self.all_files(dest), set(copier.paths())) self.assertEqual(result.removed_directories, set()) self.assertEqual(len(result.updated_files), 1)
def test_optional_exists_creates_unneeded_directory(self): """Demonstrate that a directory not strictly required, but specified as the path to an optional file, will be unnecessarily created. This behaviour is wrong; fixing it is tracked by Bug 972432; and this test exists to guard against unexpected changes in behaviour. """ dest = self.tmppath('dest') copier = FileCopier() copier.add('foo/bar', ExistingFile(required=False)) result = copier.copy(dest) st = os.lstat(self.tmppath('dest/foo')) self.assertFalse(stat.S_ISLNK(st.st_mode)) self.assertTrue(stat.S_ISDIR(st.st_mode)) # What's worse, we have no record that dest was created. self.assertEquals(len(result.updated_files), 0) # But we do have an erroneous record of an optional file # existing when it does not. self.assertIn(self.tmppath('dest/foo/bar'), result.existing_files)
def strip(dir): copier = FileCopier() # The FileFinder will give use ExecutableFile instances for files # that can be stripped, and copying ExecutableFiles defaults to # stripping them when buildconfig.substs['PKG_STRIP'] is set. for p, f in FileFinder(dir, find_executables=True): copier.add(p, f) copier.copy(dir)
def strip(dir): copier = FileCopier() # The FileFinder will give use ExecutableFile instances for files # that can be stripped, and copying ExecutableFiles defaults to # stripping them unless buildconfig.substs['PKG_SKIP_STRIP'] is set. for p, f in FileFinder(dir): copier.add(p, f) copier.copy(dir)
def test_no_remove(self): copier = FileCopier() copier.add('foo', GeneratedFile('foo')) with open(self.tmppath('bar'), 'a'): pass os.mkdir(self.tmppath('emptydir')) result = copier.copy(self.tmpdir, remove_unaccounted=False) self.assertEqual(self.all_files(self.tmpdir), set(['foo', 'bar'])) self.assertEqual(result.removed_files, set()) self.assertEqual(result.removed_directories, set([self.tmppath('emptydir')]))
def main(): parser = argparse.ArgumentParser( description="Merge two crashreporter symbols directories." ) parser.add_argument("dir1", help="Directory") parser.add_argument("dir2", help="Directory to merge") options = parser.parse_args() dir1_finder = FileFinder(options.dir1) dir2_finder = FileFinder(options.dir2) finder = UnifiedSymbolsFinder(dir1_finder, dir2_finder) copier = FileCopier() with errors.accumulate(): for p, f in finder: copier.add(p, f) copier.copy(options.dir1, skip_if_older=False)
def test_no_remove_empty_directories(self): copier = FileCopier() copier.add("foo", GeneratedFile("foo")) with open(self.tmppath("bar"), "a"): pass os.mkdir(self.tmppath("emptydir")) d = self.tmppath("populateddir") os.mkdir(d) with open(self.tmppath("populateddir/foo"), "a"): pass result = copier.copy(self.tmpdir, remove_unaccounted=False, remove_empty_directories=False) self.assertEqual(self.all_files(self.tmpdir), set(["foo", "bar", "populateddir/foo"])) self.assertEqual(self.all_dirs(self.tmpdir), set(["emptydir", "populateddir"])) self.assertEqual(result.removed_files, set()) self.assertEqual(result.removed_directories, set())
def test_permissions(self): """Ensure files without write permission can be deleted.""" with open(self.tmppath("dummy"), "a"): pass p = self.tmppath("no_perms") with open(p, "a"): pass # Make file and directory unwritable. Reminder: making a directory # unwritable prevents modifications (including deletes) from the list # of files in that directory. os.chmod(p, 0o400) os.chmod(self.tmpdir, 0o400) copier = FileCopier() copier.add("dummy", GeneratedFile(b"content")) result = copier.copy(self.tmpdir) self.assertEqual(result.removed_files_count, 1) self.assertFalse(os.path.exists(p))
def test_permissions(self): """Ensure files without write permission can be deleted.""" with open(self.tmppath('dummy'), 'a'): pass p = self.tmppath('no_perms') with open(p, 'a'): pass # Make file and directory unwritable. Reminder: making a directory # unwritable prevents modifications (including deletes) from the list # of files in that directory. os.chmod(p, 0400) os.chmod(self.tmpdir, 0400) copier = FileCopier() copier.add('dummy', GeneratedFile('content')) result = copier.copy(self.tmpdir) self.assertEqual(result.removed_files_count, 1) self.assertFalse(os.path.exists(p))
def test_no_remove(self): copier = FileCopier() copier.add("foo", GeneratedFile(b"foo")) with open(self.tmppath("bar"), "a"): pass os.mkdir(self.tmppath("emptydir")) d = self.tmppath("populateddir") os.mkdir(d) with open(self.tmppath("populateddir/foo"), "a"): pass result = copier.copy(self.tmpdir, remove_unaccounted=False) self.assertEqual(self.all_files(self.tmpdir), set(["foo", "bar", "populateddir/foo"])) self.assertEqual(self.all_dirs(self.tmpdir), set(["populateddir"])) self.assertEqual(result.removed_files, set()) self.assertEqual(result.removed_directories, set([self.tmppath("emptydir")]))
def test_remove_unaccounted_file_registry(self): """Test FileCopier.copy(remove_unaccounted=FileRegistry())""" dest = self.tmppath("dest") copier = FileCopier() copier.add("foo/bar/baz", GeneratedFile(b"foobarbaz")) copier.add("foo/bar/qux", GeneratedFile(b"foobarqux")) copier.add("foo/hoge/fuga", GeneratedFile(b"foohogefuga")) copier.add("foo/toto/tata", GeneratedFile(b"footototata")) os.makedirs(os.path.join(dest, "bar")) with open(os.path.join(dest, "bar", "bar"), "w") as fh: fh.write("barbar") os.makedirs(os.path.join(dest, "foo", "toto")) with open(os.path.join(dest, "foo", "toto", "toto"), "w") as fh: fh.write("foototototo") result = copier.copy(dest, remove_unaccounted=False) self.assertEqual(self.all_files(dest), set(copier.paths()) | {"foo/toto/toto", "bar/bar"}) self.assertEqual(self.all_dirs(dest), {"foo/bar", "foo/hoge", "foo/toto", "bar"}) copier2 = FileCopier() copier2.add("foo/hoge/fuga", GeneratedFile(b"foohogefuga")) # We expect only files copied from the first copier to be removed, # not the extra file that was there beforehand. result = copier2.copy(dest, remove_unaccounted=copier) self.assertEqual(self.all_files(dest), set(copier2.paths()) | {"foo/toto/toto", "bar/bar"}) self.assertEqual(self.all_dirs(dest), {"foo/hoge", "foo/toto", "bar"}) self.assertEqual(result.updated_files, {self.tmppath("dest/foo/hoge/fuga")}) self.assertEqual(result.existing_files, set()) self.assertEqual( result.removed_files, { self.tmppath(p) for p in ("dest/foo/bar/baz", "dest/foo/bar/qux", "dest/foo/toto/tata") }, ) self.assertEqual(result.removed_directories, {self.tmppath("dest/foo/bar")})
def test_remove_unaccounted_file_registry(self): """Test FileCopier.copy(remove_unaccounted=FileRegistry())""" dest = self.tmppath('dest') copier = FileCopier() copier.add('foo/bar/baz', GeneratedFile('foobarbaz')) copier.add('foo/bar/qux', GeneratedFile('foobarqux')) copier.add('foo/hoge/fuga', GeneratedFile('foohogefuga')) copier.add('foo/toto/tata', GeneratedFile('footototata')) os.makedirs(os.path.join(dest, 'bar')) with open(os.path.join(dest, 'bar', 'bar'), 'w') as fh: fh.write('barbar') os.makedirs(os.path.join(dest, 'foo', 'toto')) with open(os.path.join(dest, 'foo', 'toto', 'toto'), 'w') as fh: fh.write('foototototo') result = copier.copy(dest, remove_unaccounted=False) self.assertEqual(self.all_files(dest), set(copier.paths()) | {'foo/toto/toto', 'bar/bar'}) self.assertEqual(self.all_dirs(dest), {'foo/bar', 'foo/hoge', 'foo/toto', 'bar'}) copier2 = FileCopier() copier2.add('foo/hoge/fuga', GeneratedFile('foohogefuga')) # We expect only files copied from the first copier to be removed, # not the extra file that was there beforehand. result = copier2.copy(dest, remove_unaccounted=copier) self.assertEqual(self.all_files(dest), set(copier2.paths()) | {'foo/toto/toto', 'bar/bar'}) self.assertEqual(self.all_dirs(dest), {'foo/hoge', 'foo/toto', 'bar'}) self.assertEqual(result.updated_files, {self.tmppath('dest/foo/hoge/fuga')}) self.assertEqual(result.existing_files, set()) self.assertEqual( result.removed_files, { self.tmppath(p) for p in ('dest/foo/bar/baz', 'dest/foo/bar/qux', 'dest/foo/toto/tata') }) self.assertEqual(result.removed_directories, {self.tmppath('dest/foo/bar')})
def test_remove_unaccounted_file_registry(self): """Test FileCopier.copy(remove_unaccounted=FileRegistry())""" dest = self.tmppath('dest') copier = FileCopier() copier.add('foo/bar/baz', GeneratedFile('foobarbaz')) copier.add('foo/bar/qux', GeneratedFile('foobarqux')) copier.add('foo/hoge/fuga', GeneratedFile('foohogefuga')) copier.add('foo/toto/tata', GeneratedFile('footototata')) os.makedirs(os.path.join(dest, 'bar')) with open(os.path.join(dest, 'bar', 'bar'), 'w') as fh: fh.write('barbar'); os.makedirs(os.path.join(dest, 'foo', 'toto')) with open(os.path.join(dest, 'foo', 'toto', 'toto'), 'w') as fh: fh.write('foototototo'); result = copier.copy(dest, remove_unaccounted=False) self.assertEqual(self.all_files(dest), set(copier.paths()) | { 'foo/toto/toto', 'bar/bar'}) self.assertEqual(self.all_dirs(dest), {'foo/bar', 'foo/hoge', 'foo/toto', 'bar'}) copier2 = FileCopier() copier2.add('foo/hoge/fuga', GeneratedFile('foohogefuga')) # We expect only files copied from the first copier to be removed, # not the extra file that was there beforehand. result = copier2.copy(dest, remove_unaccounted=copier) self.assertEqual(self.all_files(dest), set(copier2.paths()) | { 'foo/toto/toto', 'bar/bar'}) self.assertEqual(self.all_dirs(dest), {'foo/hoge', 'foo/toto', 'bar'}) self.assertEqual(result.updated_files, {self.tmppath('dest/foo/hoge/fuga')}) self.assertEqual(result.existing_files, set()) self.assertEqual(result.removed_files, {self.tmppath(p) for p in ('dest/foo/bar/baz', 'dest/foo/bar/qux', 'dest/foo/toto/tata')}) self.assertEqual(result.removed_directories, {self.tmppath('dest/foo/bar')})
def test_remove_unaccounted_file_registry(self): """Test FileCopier.copy(remove_unaccounted=FileRegistry())""" dest = self.tmppath("dest") copier = FileCopier() copier.add("foo/bar/baz", GeneratedFile("foobarbaz")) copier.add("foo/bar/qux", GeneratedFile("foobarqux")) copier.add("foo/hoge/fuga", GeneratedFile("foohogefuga")) copier.add("foo/toto/tata", GeneratedFile("footototata")) os.makedirs(os.path.join(dest, "bar")) with open(os.path.join(dest, "bar", "bar"), "w") as fh: fh.write("barbar") os.makedirs(os.path.join(dest, "foo", "toto")) with open(os.path.join(dest, "foo", "toto", "toto"), "w") as fh: fh.write("foototototo") result = copier.copy(dest, remove_unaccounted=False) self.assertEqual(self.all_files(dest), set(copier.paths()) | {"foo/toto/toto", "bar/bar"}) self.assertEqual(self.all_dirs(dest), {"foo/bar", "foo/hoge", "foo/toto", "bar"}) copier2 = FileCopier() copier2.add("foo/hoge/fuga", GeneratedFile("foohogefuga")) # We expect only files copied from the first copier to be removed, # not the extra file that was there beforehand. result = copier2.copy(dest, remove_unaccounted=copier) self.assertEqual(self.all_files(dest), set(copier2.paths()) | {"foo/toto/toto", "bar/bar"}) self.assertEqual(self.all_dirs(dest), {"foo/hoge", "foo/toto", "bar"}) self.assertEqual(result.updated_files, {self.tmppath("dest/foo/hoge/fuga")}) self.assertEqual(result.existing_files, set()) self.assertEqual( result.removed_files, {self.tmppath(p) for p in ("dest/foo/bar/baz", "dest/foo/bar/qux", "dest/foo/toto/tata")}, ) self.assertEqual(result.removed_directories, {self.tmppath("dest/foo/bar")})
def test_remove_unaccounted_directory_symlinks(self): """Directory symlinks in destination that are not in the way are deleted according to remove_unaccounted and remove_all_directory_symlinks. """ if not self.symlink_supported: return dest = self.tmppath("dest") copier = FileCopier() copier.add("foo/bar/baz", GeneratedFile(b"foobarbaz")) os.makedirs(self.tmppath("dest/foo")) dummy = self.tmppath("dummy") os.mkdir(dummy) os.mkdir(self.tmppath("dest/zot")) link = self.tmppath("dest/zot/zap") os.symlink(dummy, link) # If not remove_unaccounted but remove_empty_directories, then # the symlinked directory remains (as does its containing # directory). result = copier.copy( dest, remove_unaccounted=False, remove_empty_directories=True, remove_all_directory_symlinks=False, ) st = os.lstat(link) self.assertTrue(stat.S_ISLNK(st.st_mode)) self.assertFalse(stat.S_ISDIR(st.st_mode)) self.assertEqual(self.all_files(dest), set(copier.paths())) self.assertEqual(self.all_dirs(dest), set(["foo/bar"])) self.assertEqual(result.removed_directories, set()) self.assertEqual(len(result.updated_files), 1) # If remove_unaccounted but not remove_empty_directories, then # only the symlinked directory is removed. result = copier.copy( dest, remove_unaccounted=True, remove_empty_directories=False, remove_all_directory_symlinks=False, ) st = os.lstat(self.tmppath("dest/zot")) self.assertFalse(stat.S_ISLNK(st.st_mode)) self.assertTrue(stat.S_ISDIR(st.st_mode)) self.assertEqual(result.removed_files, set([link])) self.assertEqual(result.removed_directories, set()) self.assertEqual(self.all_files(dest), set(copier.paths())) self.assertEqual(self.all_dirs(dest), set(["foo/bar", "zot"])) # If remove_unaccounted and remove_empty_directories, then # both the symlink and its containing directory are removed. link = self.tmppath("dest/zot/zap") os.symlink(dummy, link) result = copier.copy( dest, remove_unaccounted=True, remove_empty_directories=True, remove_all_directory_symlinks=False, ) self.assertEqual(result.removed_files, set([link])) self.assertEqual(result.removed_directories, set([self.tmppath("dest/zot")])) self.assertEqual(self.all_files(dest), set(copier.paths())) self.assertEqual(self.all_dirs(dest), set(["foo/bar"]))
def test_file_copier(self): copier = FileCopier() copier.add("foo/bar", GeneratedFile("foobar")) copier.add("foo/qux", GeneratedFile("fooqux")) copier.add("foo/deep/nested/directory/file", GeneratedFile("fooz")) copier.add("bar", GeneratedFile("bar")) copier.add("qux/foo", GeneratedFile("quxfoo")) copier.add("qux/bar", GeneratedFile("")) result = copier.copy(self.tmpdir) self.assertEqual(self.all_files(self.tmpdir), set(copier.paths())) self.assertEqual(self.all_dirs(self.tmpdir), set(["foo/deep/nested/directory", "qux"])) self.assertEqual(result.updated_files, set(self.tmppath(p) for p in self.all_files(self.tmpdir))) self.assertEqual(result.existing_files, set()) self.assertEqual(result.removed_files, set()) self.assertEqual(result.removed_directories, set()) copier.remove("foo") copier.add("test", GeneratedFile("test")) result = copier.copy(self.tmpdir) self.assertEqual(self.all_files(self.tmpdir), set(copier.paths())) self.assertEqual(self.all_dirs(self.tmpdir), set(["qux"])) self.assertEqual( result.removed_files, set(self.tmppath(p) for p in ("foo/bar", "foo/qux", "foo/deep/nested/directory/file")) )
def test_file_copier(self): copier = FileCopier() copier.add("foo/bar", GeneratedFile(b"foobar")) copier.add("foo/qux", GeneratedFile(b"fooqux")) copier.add("foo/deep/nested/directory/file", GeneratedFile(b"fooz")) copier.add("bar", GeneratedFile(b"bar")) copier.add("qux/foo", GeneratedFile(b"quxfoo")) copier.add("qux/bar", GeneratedFile(b"")) result = copier.copy(self.tmpdir) self.assertEqual(self.all_files(self.tmpdir), set(copier.paths())) self.assertEqual(self.all_dirs(self.tmpdir), set(["foo/deep/nested/directory", "qux"])) self.assertEqual( result.updated_files, set(self.tmppath(p) for p in self.all_files(self.tmpdir)), ) self.assertEqual(result.existing_files, set()) self.assertEqual(result.removed_files, set()) self.assertEqual(result.removed_directories, set()) copier.remove("foo") copier.add("test", GeneratedFile(b"test")) result = copier.copy(self.tmpdir) self.assertEqual(self.all_files(self.tmpdir), set(copier.paths())) self.assertEqual(self.all_dirs(self.tmpdir), set(["qux"])) self.assertEqual( result.removed_files, set( self.tmppath(p) for p in ("foo/bar", "foo/qux", "foo/deep/nested/directory/file")), )
def main(): parser = ArgumentParser() parser.add_argument('-D', dest='defines', action='append', metavar="VAR[=VAL]", help='Define a variable') parser.add_argument('--format', default='omni', help='Choose the chrome format for packaging ' + '(omni, jar or flat ; default: %(default)s)') parser.add_argument('--removals', default=None, help='removed-files source file') parser.add_argument('--ignore-errors', action='store_true', default=False, help='Transform errors into warnings.') parser.add_argument('--minify', action='store_true', default=False, help='Make some files more compact while packaging') parser.add_argument('--minify-js', action='store_true', help='Minify JavaScript files while packaging.') parser.add_argument('--js-binary', help='Path to js binary. This is used to verify ' 'minified JavaScript. If this is not defined, ' 'minification verification will not be performed.') parser.add_argument('--jarlog', default='', help='File containing jar ' + 'access logs') parser.add_argument('--optimizejars', action='store_true', default=False, help='Enable jar optimizations') parser.add_argument('--unify', default='', help='Base directory of another build to unify with') parser.add_argument('manifest', default=None, nargs='?', help='Manifest file name') parser.add_argument('source', help='Source directory') parser.add_argument('destination', help='Destination directory') parser.add_argument('--non-resource', nargs='+', metavar='PATTERN', default=[], help='Extra files not to be considered as resources') args = parser.parse_args() defines = dict(buildconfig.defines) if args.ignore_errors: errors.ignore_errors() if args.defines: for name, value in [split_define(d) for d in args.defines]: defines[name] = value copier = FileCopier() if args.format == 'flat': formatter = FlatFormatter(copier) elif args.format == 'jar': formatter = JarFormatter(copier, optimize=args.optimizejars) elif args.format == 'omni': formatter = OmniJarFormatter(copier, buildconfig.substs['OMNIJAR_NAME'], optimize=args.optimizejars, non_resources=args.non_resource) else: errors.fatal('Unknown format: %s' % args.format) # Adjust defines according to the requested format. if isinstance(formatter, OmniJarFormatter): defines['MOZ_OMNIJAR'] = 1 elif 'MOZ_OMNIJAR' in defines: del defines['MOZ_OMNIJAR'] binpath = '' if 'BINPATH' in defines: binpath = SimpleManifestSink.normalize_path(defines['BINPATH']) while binpath.startswith('/'): binpath = binpath[1:] if args.unify: def is_native(path): path = os.path.abspath(path) return platform.machine() in mozpack.path.split(path) # Invert args.unify and args.source if args.unify points to the # native architecture. args.source, args.unify = sorted([args.source, args.unify], key=is_native, reverse=True) if is_native(args.source): launcher.tooldir = args.source elif not buildconfig.substs['CROSS_COMPILE']: launcher.tooldir = buildconfig.substs['LIBXUL_DIST'] with errors.accumulate(): finder_args = dict( minify=args.minify, minify_js=args.minify_js, ) if args.js_binary: finder_args['minify_js_verify_command'] = [ args.js_binary, os.path.join(os.path.abspath(os.path.dirname(__file__)), 'js-compare-ast.js') ] if args.unify: finder = UnifiedBuildFinder(FileFinder(args.source), FileFinder(args.unify), **finder_args) else: finder = FileFinder(args.source, **finder_args) if 'NO_PKG_FILES' in os.environ: sinkformatter = NoPkgFilesRemover(formatter, args.manifest is not None) else: sinkformatter = formatter sink = SimpleManifestSink(finder, sinkformatter) if args.manifest: preprocess_manifest(sink, args.manifest, defines) else: sink.add(Component(''), 'bin/*') sink.close(args.manifest is not None) if args.removals: lines = [l.lstrip() for l in open(args.removals).readlines()] removals_in = StringIO(''.join(lines)) removals_in.name = args.removals removals = RemovedFiles(copier) preprocess(removals_in, removals, defines) copier.add(mozpack.path.join(binpath, 'removed-files'), removals) # shlibsign libraries if launcher.can_launch(): for lib in SIGN_LIBS: libbase = mozpack.path.join(binpath, '%s%s') \ % (buildconfig.substs['DLL_PREFIX'], lib) libname = '%s%s' % (libbase, buildconfig.substs['DLL_SUFFIX']) if copier.contains(libname): copier.add( libbase + '.chk', LibSignFile(os.path.join(args.destination, libname))) # Setup preloading if args.jarlog and os.path.exists(args.jarlog): from mozpack.mozjar import JarLog log = JarLog(args.jarlog) for p, f in copier: if not isinstance(f, Jarrer): continue key = JarLog.canonicalize(os.path.join(args.destination, p)) if key in log: f.preload(log[key]) # Fill startup cache if isinstance(formatter, OmniJarFormatter) and launcher.can_launch() \ and buildconfig.substs['MOZ_DISABLE_STARTUPCACHE'] != '1': if buildconfig.substs['LIBXUL_SDK']: gre_path = mozpack.path.join(buildconfig.substs['LIBXUL_DIST'], 'bin') else: gre_path = None for base in sorted([[ p for p in [mozpack.path.join('bin', b), b] if os.path.exists(os.path.join(args.source, p)) ][0] for b in sink.packager.get_bases()]): if not gre_path: gre_path = base base_path = sink.normalize_path(base) if base_path in formatter.omnijars: precompile_cache(formatter.omnijars[base_path], args.source, gre_path, base) copier.copy(args.destination) generate_precomplete( os.path.normpath(os.path.join(args.destination, binpath)))
def test_file_copier(self): copier = FileCopier() copier.add('foo/bar', GeneratedFile('foobar')) copier.add('foo/qux', GeneratedFile('fooqux')) copier.add('foo/deep/nested/directory/file', GeneratedFile('fooz')) copier.add('bar', GeneratedFile('bar')) copier.add('qux/foo', GeneratedFile('quxfoo')) copier.add('qux/bar', GeneratedFile('')) copier.copy(self.tmpdir) self.assertEqual(self.all_files(self.tmpdir), set(copier.paths())) self.assertEqual(self.all_dirs(self.tmpdir), set(['foo/deep/nested/directory', 'qux'])) copier.remove('foo') copier.add('test', GeneratedFile('test')) copier.copy(self.tmpdir) self.assertEqual(self.all_files(self.tmpdir), set(copier.paths())) self.assertEqual(self.all_dirs(self.tmpdir), set(['qux']))
def test_file_copier(self): copier = FileCopier() copier.add('foo/bar', GeneratedFile('foobar')) copier.add('foo/qux', GeneratedFile('fooqux')) copier.add('foo/deep/nested/directory/file', GeneratedFile('fooz')) copier.add('bar', GeneratedFile('bar')) copier.add('qux/foo', GeneratedFile('quxfoo')) copier.add('qux/bar', GeneratedFile('')) result = copier.copy(self.tmpdir) self.assertEqual(self.all_files(self.tmpdir), set(copier.paths())) self.assertEqual(self.all_dirs(self.tmpdir), set(['foo/deep/nested/directory', 'qux'])) self.assertEqual( result.updated_files, set(self.tmppath(p) for p in self.all_files(self.tmpdir))) self.assertEqual(result.existing_files, set()) self.assertEqual(result.removed_files, set()) self.assertEqual(result.removed_directories, set()) copier.remove('foo') copier.add('test', GeneratedFile('test')) result = copier.copy(self.tmpdir) self.assertEqual(self.all_files(self.tmpdir), set(copier.paths())) self.assertEqual(self.all_dirs(self.tmpdir), set(['qux'])) self.assertEqual( result.removed_files, set( self.tmppath(p) for p in ('foo/bar', 'foo/qux', 'foo/deep/nested/directory/file')))
def test_file_copier(self): copier = FileCopier() copier.add('foo/bar', GeneratedFile('foobar')) copier.add('foo/qux', GeneratedFile('fooqux')) copier.add('foo/deep/nested/directory/file', GeneratedFile('fooz')) copier.add('bar', GeneratedFile('bar')) copier.add('qux/foo', GeneratedFile('quxfoo')) copier.add('qux/bar', GeneratedFile('')) result = copier.copy(self.tmpdir) self.assertEqual(self.all_files(self.tmpdir), set(copier.paths())) self.assertEqual(self.all_dirs(self.tmpdir), set(['foo/deep/nested/directory', 'qux'])) self.assertEqual(result.updated_files, set(self.tmppath(p) for p in self.all_files(self.tmpdir))) self.assertEqual(result.existing_files, set()) self.assertEqual(result.removed_files, set()) self.assertEqual(result.removed_directories, set()) copier.remove('foo') copier.add('test', GeneratedFile('test')) result = copier.copy(self.tmpdir) self.assertEqual(self.all_files(self.tmpdir), set(copier.paths())) self.assertEqual(self.all_dirs(self.tmpdir), set(['qux'])) self.assertEqual(result.removed_files, set(self.tmppath(p) for p in ('foo/bar', 'foo/qux', 'foo/deep/nested/directory/file')))
def fat_aar(distdir, aars_paths, no_process=False, no_compatibility_check=False): if no_process: print('Not processing architecture-specific artifact Maven AARs.') return 0 # Map {filename: {fingerprint: [arch1, arch2, ...]}}. diffs = defaultdict(lambda: defaultdict(list)) missing_arch_prefs = set() # Collect multi-architecture inputs to the fat AAR. copier = FileCopier() for arch, aar_path in aars_paths.items(): # Map old non-architecture-specific path to new architecture-specific path. old_rewrite_map = { 'greprefs.js': '{}/greprefs.js'.format(arch), 'defaults/pref/geckoview-prefs.js': 'defaults/pref/{}/geckoview-prefs.js'.format(arch), } # Architecture-specific preferences files. arch_prefs = set(old_rewrite_map.values()) missing_arch_prefs |= set(arch_prefs) jar_finder = JarFinder(aar_path, JarReader(aar_path)) for path, fileobj in UnpackFinder(jar_finder): # Native libraries go straight through. if mozpath.match(path, 'jni/**'): copier.add(path, fileobj) elif path in arch_prefs: copier.add(path, fileobj) elif path in ('classes.jar', 'annotations.zip'): # annotations.zip differs due to timestamps, but the contents should not. # `JarReader` fails on the non-standard `classes.jar` produced by Gradle/aapt, # and it's not worth working around, so we use Python's zip functionality # instead. z = ZipFile(BytesIO(fileobj.open().read())) for r in z.namelist(): fingerprint = sha1(z.open(r).read()).hexdigest() diffs['{}!/{}'.format(path, r)][fingerprint].append(arch) else: fingerprint = sha1(fileobj.open().read()).hexdigest() # There's no need to distinguish `target.maven.zip` from `assets/omni.ja` here, # since in practice they will never overlap. diffs[path][fingerprint].append(arch) missing_arch_prefs.discard(path) # Some differences are allowed across the architecture-specific AARs. We could allow-list # the actual content, but it's not necessary right now. allow_pattern_list = { 'AndroidManifest.xml', # Min SDK version is different for 32- and 64-bit builds. 'classes.jar!/org/mozilla/gecko/util/HardwareUtils.class', # Min SDK as well. 'classes.jar!/org/mozilla/geckoview/BuildConfig.class', # Each input captures its CPU architecture. 'chrome/toolkit/content/global/buildconfig.html', # Bug 1556162: localized resources are not deterministic across # per-architecture builds triggered from the same push. '**/*.ftl', '**/*.dtd', '**/*.properties', } not_allowed = OrderedDict() def format_diffs(ds): # Like ' armeabi-v7a, arm64-v8a -> XXX\n x86, x86_64 -> YYY'. return '\n'.join( sorted(' {archs} -> {fingerprint}'.format( archs=', '.join(sorted(archs)), fingerprint=fingerprint) for fingerprint, archs in ds.iteritems())) for p, ds in sorted(diffs.iteritems()): if len(ds) <= 1: # Only one hash across all inputs: roll on. continue if any(mozpath.match(p, pat) for pat in allow_pattern_list): print( 'Allowed: Path "{path}" has architecture-specific versions:\n{ds_repr}' .format(path=p, ds_repr=format_diffs(ds))) continue not_allowed[p] = ds for p, ds in not_allowed.iteritems(): print( 'Disallowed: Path "{path}" has architecture-specific versions:\n{ds_repr}' .format(path=p, ds_repr=format_diffs(ds))) for missing in sorted(missing_arch_prefs): print( 'Disallowed: Inputs missing expected architecture-specific input: {missing}' .format(missing=missing)) if not no_compatibility_check and (missing_arch_prefs or not_allowed): return 1 output_dir = mozpath.join(distdir, 'output') copier.copy(output_dir) return 0
def main(): parser = ArgumentParser() parser.add_argument('-D', dest='defines', action='append', metavar="VAR[=VAL]", help='Define a variable') parser.add_argument('--format', default='omni', help='Choose the chrome format for packaging ' + '(omni, jar or flat ; default: %(default)s)') parser.add_argument('--removals', default=None, help='removed-files source file') parser.add_argument('--ignore-errors', action='store_true', default=False, help='Transform errors into warnings.') parser.add_argument('--minify', action='store_true', default=False, help='Make some files more compact while packaging') parser.add_argument('--jarlog', default='', help='File containing jar ' + 'access logs') parser.add_argument('--optimizejars', action='store_true', default=False, help='Enable jar optimizations') parser.add_argument('--unify', default='', help='Base directory of another build to unify with') parser.add_argument('manifest', default=None, nargs='?', help='Manifest file name') parser.add_argument('source', help='Source directory') parser.add_argument('destination', help='Destination directory') parser.add_argument('--non-resource', nargs='+', metavar='PATTERN', default=[], help='Extra files not to be considered as resources') args = parser.parse_args() defines = dict(buildconfig.defines) if args.ignore_errors: errors.ignore_errors() if args.defines: for name, value in [split_define(d) for d in args.defines]: defines[name] = value copier = FileCopier() if args.format == 'flat': formatter = FlatFormatter(copier) elif args.format == 'jar': formatter = JarFormatter(copier, optimize=args.optimizejars) elif args.format == 'omni': formatter = OmniJarFormatter(copier, buildconfig.substs['OMNIJAR_NAME'], optimize=args.optimizejars, non_resources=args.non_resource) else: errors.fatal('Unknown format: %s' % args.format) # Adjust defines according to the requested format. if isinstance(formatter, OmniJarFormatter): defines['MOZ_OMNIJAR'] = 1 elif 'MOZ_OMNIJAR' in defines: del defines['MOZ_OMNIJAR'] binpath = '' if 'BINPATH' in defines: binpath = SimpleManifestSink.normalize_path(defines['BINPATH']) while binpath.startswith('/'): binpath = binpath[1:] if args.unify: def is_native(path): path = os.path.abspath(path) return platform.machine() in mozpack.path.split(path) # Invert args.unify and args.source if args.unify points to the # native architecture. args.source, args.unify = sorted([args.source, args.unify], key=is_native, reverse=True) if is_native(args.source): launcher.tooldir = args.source elif not buildconfig.substs['CROSS_COMPILE']: launcher.tooldir = buildconfig.substs['LIBXUL_DIST'] with errors.accumulate(): if args.unify: finder = UnifiedBuildFinder(FileFinder(args.source), FileFinder(args.unify), minify=args.minify) else: finder = FileFinder(args.source, minify=args.minify) if 'NO_PKG_FILES' in os.environ: sinkformatter = NoPkgFilesRemover(formatter, args.manifest is not None) else: sinkformatter = formatter sink = SimpleManifestSink(finder, sinkformatter) if args.manifest: preprocess_manifest(sink, args.manifest, defines) else: sink.add(Component(''), 'bin/*') sink.close(args.manifest is not None) if args.removals: lines = [l.lstrip() for l in open(args.removals).readlines()] removals_in = StringIO(''.join(lines)) removals_in.name = args.removals removals = RemovedFiles(copier) preprocess(removals_in, removals, defines) copier.add(mozpack.path.join(binpath, 'removed-files'), removals) # shlibsign libraries if launcher.can_launch(): for lib in SIGN_LIBS: libbase = mozpack.path.join(binpath, '%s%s') \ % (buildconfig.substs['DLL_PREFIX'], lib) libname = '%s%s' % (libbase, buildconfig.substs['DLL_SUFFIX']) if copier.contains(libname): copier.add(libbase + '.chk', LibSignFile(os.path.join(args.destination, libname))) # Setup preloading if args.jarlog and os.path.exists(args.jarlog): from mozpack.mozjar import JarLog log = JarLog(args.jarlog) for p, f in copier: if not isinstance(f, Jarrer): continue key = JarLog.canonicalize(os.path.join(args.destination, p)) if key in log: f.preload(log[key]) # Fill startup cache on Windows and Linux only # (this currently causes build failure on BSD, so skip on that platfom) if sys.platform == 'win32' or sys.platform.startswith ('linux'): if isinstance(formatter, OmniJarFormatter) and launcher.can_launch(): if buildconfig.substs['LIBXUL_SDK']: gre_path = mozpack.path.join(buildconfig.substs['LIBXUL_DIST'], 'bin') else: gre_path = None for base in sorted([[p for p in [mozpack.path.join('bin', b), b] if os.path.exists(os.path.join(args.source, p))][0] for b in sink.packager.get_bases()]): if not gre_path: gre_path = base base_path = sink.normalize_path(base) if base_path in formatter.omnijars: precompile_cache(formatter.omnijars[base_path], args.source, gre_path, base) copier.copy(args.destination) generate_precomplete(os.path.normpath(os.path.join(args.destination, binpath)))
def repackage_msix( dir_or_package, channel=None, branding=None, template=None, distribution_dirs=[], locale_allowlist=set(), version=None, vendor=None, displayname=None, app_name="firefox", identity=None, publisher=None, publisher_display_name="Mozilla Corporation", arch=None, output=None, force=False, log=None, verbose=False, makeappx=None, ): if not channel: raise Exception("channel is required") if channel not in ["official", "beta", "aurora", "nightly", "unofficial"]: raise Exception("channel is unrecognized: {}".format(channel)) if not branding: raise Exception("branding dir is required") if not os.path.isdir(branding): raise Exception("branding dir {} does not exist".format(branding)) # TODO: maybe we can fish this from the package directly? Maybe from a DLL, # maybe from application.ini? if arch is None or arch not in _MSIX_ARCH.keys(): raise Exception( "arch name must be provided and one of {}.".format(_MSIX_ARCH.keys()) ) if not os.path.exists(dir_or_package): raise Exception("{} does not exist".format(dir_or_package)) if ( os.path.isfile(dir_or_package) and os.path.splitext(dir_or_package)[1] == ".msix" ): # The convention is $MOZBUILD_STATE_PATH/cache/$FEATURE. msix_dir = mozpath.normsep( mozpath.join( get_state_dir(), "cache", "mach-msix", "msix-unpack", ) ) if os.path.exists(msix_dir): shutil.rmtree(msix_dir) ensureParentDir(msix_dir) dir_or_package = unpack_msix(dir_or_package, msix_dir, log=log, verbose=verbose) log( logging.INFO, "msix", { "input": dir_or_package, }, "Adding files from '{input}'", ) if os.path.isdir(dir_or_package): finder = FileFinder(dir_or_package) else: finder = JarFinder(dir_or_package, JarReader(dir_or_package)) values = get_application_ini_values( finder, dict(section="App", value="CodeName", fallback="Name"), dict(section="App", value="Vendor"), ) first = next(values) displayname = displayname or "Mozilla {}".format(first) second = next(values) vendor = vendor or second # For `AppConstants.jsm` and `brand.properties`, which are in the omnijar in packaged builds. # The nested langpack XPI files can't be read by `mozjar.py`. unpack_finder = UnpackFinder(finder, unpack_xpi=False) if not version: values = get_appconstants_jsm_values( unpack_finder, "MOZ_APP_VERSION_DISPLAY", "MOZ_BUILDID" ) display_version = next(values) buildid = next(values) version = get_embedded_version(display_version, buildid) log( logging.INFO, "msix", { "version": version, "display_version": display_version, "buildid": buildid, }, "AppConstants.jsm display version is '{display_version}' and build ID is '{buildid}':" + " embedded version will be '{version}'", ) # TODO: Bug 1721922: localize this description via Fluent. lines = [] for _, f in unpack_finder.find("**/chrome/en-US/locale/branding/brand.properties"): lines.extend( line for line in f.open().read().decode("utf-8").splitlines() if "brandFullName" in line ) (brandFullName,) = lines # We expect exactly one definition. _, _, brandFullName = brandFullName.partition("=") brandFullName = brandFullName.strip() # We don't have a build at repackage-time to gives us this value, and the # source of truth is a branding-specific `configure.sh` shell script that we # can't easily evaluate completely here. Instead, we take the last value # from `configure.sh`. lines = [ line for line in open(mozpath.join(branding, "configure.sh")).readlines() if "MOZ_IGECKOBACKCHANNEL_IID" in line ] MOZ_IGECKOBACKCHANNEL_IID = lines[-1] _, _, MOZ_IGECKOBACKCHANNEL_IID = MOZ_IGECKOBACKCHANNEL_IID.partition("=") MOZ_IGECKOBACKCHANNEL_IID = MOZ_IGECKOBACKCHANNEL_IID.strip() if MOZ_IGECKOBACKCHANNEL_IID.startswith(('"', "'")): MOZ_IGECKOBACKCHANNEL_IID = MOZ_IGECKOBACKCHANNEL_IID[1:-1] # The convention is $MOZBUILD_STATE_PATH/cache/$FEATURE. output_dir = mozpath.normsep( mozpath.join( get_state_dir(), "cache", "mach-msix", "msix-temp-{}".format(channel) ) ) if channel == "beta": # Release (official) and Beta share branding. Differentiate Beta a little bit. displayname += " Beta" brandFullName += " Beta" # Like 'Firefox Package Root', 'Firefox Nightly Package Root', 'Firefox Beta # Package Root'. This is `BrandFullName` in the installer, and we want to # be close but to not match. By not matching, we hope to prevent confusion # and/or errors between regularly installed builds and App Package builds. instdir = "{} Package Root".format(displayname) # The standard package name is like "CompanyNoSpaces.ProductNoSpaces". identity = identity or "{}.{}".format(vendor, displayname).replace(" ", "") # We might want to include the publisher ID hash here. I.e., # "__{publisherID}". My locally produced MSIX was named like # `Mozilla.MozillaFirefoxNightly_89.0.0.0_x64__4gf61r4q480j0`, suggesting also a # missing field, but it's not necessary, since this is just an output file name. package_output_name = "{identity}_{version}_{arch}".format( identity=identity, version=version, arch=_MSIX_ARCH[arch] ) # The convention is $MOZBUILD_STATE_PATH/cache/$FEATURE. default_output = mozpath.normsep( mozpath.join( get_state_dir(), "cache", "mach-msix", "{}.msix".format(package_output_name) ) ) output = output or default_output log(logging.INFO, "msix", {"output": output}, "Repackaging to: {output}") m = InstallManifest() m.add_copy(mozpath.join(template, "Resources.pri"), "Resources.pri") m.add_pattern_copy(mozpath.join(branding, "msix", "Assets"), "**", "Assets") m.add_pattern_copy(mozpath.join(template, "VFS"), "**", "VFS") copier = FileCopier() # TODO: Bug 1710147: filter out MSVCRT files and use a dependency instead. for p, f in finder: if not os.path.isdir(dir_or_package): # In archived builds, `p` is like "firefox/firefox.exe"; we want just "firefox.exe". pp = os.path.relpath(p, "firefox") else: # In local builds and unpacked MSIX directories, `p` is like "firefox.exe" already. pp = p if pp.startswith("distribution"): # Treat any existing distribution as a distribution directory, # potentially with language packs. This makes it easy to repack # unpacked MSIXes. distribution_dir = mozpath.join(dir_or_package, "distribution") if distribution_dir not in distribution_dirs: distribution_dirs.append(distribution_dir) continue copier.add(mozpath.normsep(mozpath.join("VFS", "ProgramFiles", instdir, pp)), f) # Locales to declare as supported in `AppxManifest.xml`. locales = set(["en-US"]) for distribution_dir in [ mozpath.join(template, "distribution") ] + distribution_dirs: log( logging.INFO, "msix", {"dir": distribution_dir}, "Adding distribution files from {dir}", ) # In automation, we have no easy way to remap the names of artifacts fetched from dependent # tasks. In particular, langpacks will be named like `target.langpack.xpi`. The fetch # tasks do allow us to put them in a per-locale directory, so that the entire set can be # fetched. Here we remap the names. finder = FileFinder(distribution_dir) for p, f in finder: locale = None if os.path.basename(p) == "target.langpack.xpi": # Turn "/path/to/LOCALE/target.langpack.xpi" into "LOCALE". This is how langpacks # are presented in CI. base, locale = os.path.split(os.path.dirname(p)) # Like "locale-LOCALE/[email protected]". This is what AMO # serves and how flatpak builds name langpacks, but not how snap builds name # langpacks. I can't explain the discrepancy. dest = mozpath.normsep( mozpath.join( base, f"locale-{locale}", f"langpack-{locale}@firefox.mozilla.org.xpi", ) ) log( logging.DEBUG, "msix", {"path": p, "dest": dest}, "Renaming langpack {path} to {dest}", ) elif os.path.basename(p).startswith("langpack-"): # Turn "/path/to/[email protected]" into "LOCALE". This is # how langpacks are presented from an unpacked MSIX. _, _, locale = os.path.basename(p).partition("langpack-") locale, _, _ = locale.partition("@") dest = p else: dest = p if locale: locale = locale.strip().lower() locales.add(locale) log( logging.DEBUG, "msix", {"locale": locale, "dest": dest}, "Distributing locale '{locale}' from {dest}", ) dest = mozpath.normsep( mozpath.join("VFS", "ProgramFiles", instdir, "distribution", dest) ) if copier.contains(dest): log( logging.INFO, "msix", {"dest": dest, "path": mozpath.join(finder.base, p)}, "Skipping duplicate: {dest} from {path}", ) continue log( logging.DEBUG, "msix", {"dest": dest, "path": mozpath.join(finder.base, p)}, "Adding distribution path: {dest} from {path}", ) copier.add( dest, f, ) locales.remove("en-US") # Windows MSIX packages support a finite set of locales: see # https://docs.microsoft.com/en-us/windows/uwp/publish/supported-languages, which is encoded in # https://searchfox.org/mozilla-central/source/browser/installer/windows/msix/msix-all-locales. # We distribute all of the langpacks supported by the release channel in our MSIX, which is # encoded in https://searchfox.org/mozilla-central/source/browser/locales/all-locales. But we # only advertise support in the App manifest for the intersection of that set and the set of # supported locales. # # We distribute all langpacks to avoid the following issue. Suppose a user manually installs a # langpack that is not supported by Windows, and then updates the installed MSIX package. MSIX # package upgrades are essentially paveover installs, so there is no opportunity for Firefox to # update the langpack before the update. But, since all langpacks are bundled with the MSIX, # that langpack will be up-to-date, preventing one class of YSOD. unadvertised = set() if locale_allowlist: unadvertised = locales - locale_allowlist locales = locales & locale_allowlist for locale in sorted(unadvertised): log( logging.INFO, "msix", {"locale": locale}, "Not advertising distributed locale '{locale}' that is not recognized by Windows", ) locales = ["en-US"] + list(sorted(locales)) resource_language_list = "\n".join( f' <Resource Language="{locale}" />' for locale in sorted(locales) ) defines = { "APPX_ARCH": _MSIX_ARCH[arch], "APPX_DISPLAYNAME": brandFullName, "APPX_DESCRIPTION": brandFullName, # Like 'Mozilla.MozillaFirefox', 'Mozilla.MozillaFirefoxBeta', or # 'Mozilla.MozillaFirefoxNightly'. "APPX_IDENTITY": identity, # Like 'Firefox Package Root', 'Firefox Nightly Package Root', 'Firefox # Beta Package Root'. See above. "APPX_INSTDIR": instdir, # Like 'Firefox%20Package%20Root'. "APPX_INSTDIR_QUOTED": urllib.parse.quote(instdir), "APPX_PUBLISHER": publisher, "APPX_PUBLISHER_DISPLAY_NAME": publisher_display_name, "APPX_RESOURCE_LANGUAGE_LIST": resource_language_list, "APPX_VERSION": version, "MOZ_APP_DISPLAYNAME": displayname, "MOZ_APP_NAME": app_name, "MOZ_IGECKOBACKCHANNEL_IID": MOZ_IGECKOBACKCHANNEL_IID, } m.add_preprocess( mozpath.join(template, "AppxManifest.xml.in"), "AppxManifest.xml", [], defines=defines, marker="<!-- #", # So that we can have well-formed XML. ) m.populate_registry(copier) output_dir = mozpath.abspath(output_dir) ensureParentDir(output_dir) start = time.time() result = copier.copy( output_dir, remove_empty_directories=True, skip_if_older=not force ) if log: log_copy_result(log, time.time() - start, output_dir, result) if verbose: # Dump AppxManifest.xml contents for ease of debugging. log(logging.DEBUG, "msix", {}, "AppxManifest.xml") log(logging.DEBUG, "msix", {}, ">>>") for line in open(mozpath.join(output_dir, "AppxManifest.xml")).readlines(): log(logging.DEBUG, "msix", {}, line[:-1]) # Drop trailing line terminator. log(logging.DEBUG, "msix", {}, "<<<") if not makeappx: makeappx = find_sdk_tool("makeappx.exe", log=log) if not makeappx: raise ValueError( "makeappx is required; " "set MAKEAPPX or WINDOWSSDKDIR or PATH" ) # `makeappx.exe` supports both slash and hyphen style arguments; `makemsix` # supports only hyphen style. `makeappx.exe` allows to overwrite and to # provide more feedback, so we prefer invoking with these flags. This will # also accommodate `wine makeappx.exe`. stdout = subprocess.run( [makeappx], check=False, capture_output=True, universal_newlines=True ).stdout is_makeappx = "MakeAppx Tool" in stdout if is_makeappx: args = [makeappx, "pack", "/d", output_dir, "/p", output, "/overwrite"] else: args = [makeappx, "pack", "-d", output_dir, "-p", output] if verbose and is_makeappx: args.append("/verbose") joined = " ".join(shlex_quote(arg) for arg in args) log(logging.INFO, "msix", {"args": args, "joined": joined}, "Invoking: {joined}") sys.stdout.flush() # Otherwise the subprocess output can be interleaved. if verbose: subprocess.check_call(args, universal_newlines=True) else: # Suppress output unless we fail. try: subprocess.check_output(args, universal_newlines=True) except subprocess.CalledProcessError as e: sys.stderr.write(e.output) raise return output
def main(): parser = ArgumentParser() parser.add_argument( "-D", dest="defines", action="append", metavar="VAR[=VAL]", help="Define a variable", ) parser.add_argument( "--format", default="omni", help="Choose the chrome format for packaging " + "(omni, jar or flat ; default: %(default)s)", ) parser.add_argument("--removals", default=None, help="removed-files source file") parser.add_argument( "--ignore-errors", action="store_true", default=False, help="Transform errors into warnings.", ) parser.add_argument( "--ignore-broken-symlinks", action="store_true", default=False, help="Do not fail when processing broken symlinks.", ) parser.add_argument( "--minify", action="store_true", default=False, help="Make some files more compact while packaging", ) parser.add_argument( "--minify-js", action="store_true", help="Minify JavaScript files while packaging.", ) parser.add_argument( "--js-binary", help="Path to js binary. This is used to verify " "minified JavaScript. If this is not defined, " "minification verification will not be performed.", ) parser.add_argument("--jarlog", default="", help="File containing jar " + "access logs") parser.add_argument( "--compress", choices=("none", "deflate"), default="deflate", help="Use given jar compression (default: deflate)", ) parser.add_argument("manifest", default=None, nargs="?", help="Manifest file name") parser.add_argument("source", help="Source directory") parser.add_argument("destination", help="Destination directory") parser.add_argument( "--non-resource", nargs="+", metavar="PATTERN", default=[], help="Extra files not to be considered as resources", ) args = parser.parse_args() defines = dict(buildconfig.defines["ALLDEFINES"]) if args.ignore_errors: errors.ignore_errors() if args.defines: for name, value in [split_define(d) for d in args.defines]: defines[name] = value compress = { "none": False, "deflate": True, }[args.compress] copier = FileCopier() if args.format == "flat": formatter = FlatFormatter(copier) elif args.format == "jar": formatter = JarFormatter(copier, compress=compress) elif args.format == "omni": formatter = OmniJarFormatter( copier, buildconfig.substs["OMNIJAR_NAME"], compress=compress, non_resources=args.non_resource, ) else: errors.fatal("Unknown format: %s" % args.format) # Adjust defines according to the requested format. if isinstance(formatter, OmniJarFormatter): defines["MOZ_OMNIJAR"] = 1 elif "MOZ_OMNIJAR" in defines: del defines["MOZ_OMNIJAR"] respath = "" if "RESPATH" in defines: respath = SimpleManifestSink.normalize_path(defines["RESPATH"]) while respath.startswith("/"): respath = respath[1:] with errors.accumulate(): finder_args = dict( minify=args.minify, minify_js=args.minify_js, ignore_broken_symlinks=args.ignore_broken_symlinks, ) if args.js_binary: finder_args["minify_js_verify_command"] = [ args.js_binary, os.path.join(os.path.abspath(os.path.dirname(__file__)), "js-compare-ast.js"), ] finder = PackagerFileFinder(args.source, find_executables=True, **finder_args) if "NO_PKG_FILES" in os.environ: sinkformatter = NoPkgFilesRemover(formatter, args.manifest is not None) else: sinkformatter = formatter sink = SimpleManifestSink(finder, sinkformatter) if args.manifest: preprocess_manifest(sink, args.manifest, defines) else: sink.add(Component(""), "bin/*") sink.close(args.manifest is not None) if args.removals: removals_in = StringIO(open(args.removals).read()) removals_in.name = args.removals removals = RemovedFiles(copier) preprocess(removals_in, removals, defines) copier.add(mozpath.join(respath, "removed-files"), removals) # If a pdb file is present and we were instructed to copy it, include it. # Run on all OSes to capture MinGW builds if buildconfig.substs.get("MOZ_COPY_PDBS"): # We want to mutate the copier while we're iterating through it, so copy # the items to a list first. copier_items = [(p, f) for p, f in copier] for p, f in copier_items: if isinstance(f, ExecutableFile): pdbname = os.path.splitext(f.inputs()[0])[0] + ".pdb" if os.path.exists(pdbname): copier.add(os.path.basename(pdbname), File(pdbname)) # Setup preloading if args.jarlog: if not os.path.exists(args.jarlog): raise Exception("Cannot find jar log: %s" % args.jarlog) omnijars = [] if isinstance(formatter, OmniJarFormatter): omnijars = [ mozpath.join(base, buildconfig.substs["OMNIJAR_NAME"]) for base in sink.packager.get_bases(addons=False) ] from mozpack.mozjar import JarLog log = JarLog(args.jarlog) for p, f in copier: if not isinstance(f, Jarrer): continue if respath: p = mozpath.relpath(p, respath) if p in log: f.preload(log[p]) elif p in omnijars: raise Exception("No jar log data for %s" % p) copier.copy(args.destination) generate_precomplete( os.path.normpath(os.path.join(args.destination, respath)))
def test_remove_unaccounted_directory_symlinks(self): """Directory symlinks in destination that are not in the way are deleted according to remove_unaccounted and remove_all_directory_symlinks. """ if not self.symlink_supported: return dest = self.tmppath('dest') copier = FileCopier() copier.add('foo/bar/baz', GeneratedFile('foobarbaz')) os.makedirs(self.tmppath('dest/foo')) dummy = self.tmppath('dummy') os.mkdir(dummy) os.mkdir(self.tmppath('dest/zot')) link = self.tmppath('dest/zot/zap') os.symlink(dummy, link) # If not remove_unaccounted but remove_empty_directories, then # the symlinked directory remains (as does its containing # directory). result = copier.copy(dest, remove_unaccounted=False, remove_empty_directories=True, remove_all_directory_symlinks=False) st = os.lstat(link) self.assertTrue(stat.S_ISLNK(st.st_mode)) self.assertFalse(stat.S_ISDIR(st.st_mode)) self.assertEqual(self.all_files(dest), set(copier.paths())) self.assertEqual(self.all_dirs(dest), set(['foo/bar'])) self.assertEqual(result.removed_directories, set()) self.assertEqual(len(result.updated_files), 1) # If remove_unaccounted but not remove_empty_directories, then # only the symlinked directory is removed. result = copier.copy(dest, remove_unaccounted=True, remove_empty_directories=False, remove_all_directory_symlinks=False) st = os.lstat(self.tmppath('dest/zot')) self.assertFalse(stat.S_ISLNK(st.st_mode)) self.assertTrue(stat.S_ISDIR(st.st_mode)) self.assertEqual(result.removed_files, set([link])) self.assertEqual(result.removed_directories, set()) self.assertEqual(self.all_files(dest), set(copier.paths())) self.assertEqual(self.all_dirs(dest), set(['foo/bar', 'zot'])) # If remove_unaccounted and remove_empty_directories, then # both the symlink and its containing directory are removed. link = self.tmppath('dest/zot/zap') os.symlink(dummy, link) result = copier.copy(dest, remove_unaccounted=True, remove_empty_directories=True, remove_all_directory_symlinks=False) self.assertEqual(result.removed_files, set([link])) self.assertEqual(result.removed_directories, set([self.tmppath('dest/zot')])) self.assertEqual(self.all_files(dest), set(copier.paths())) self.assertEqual(self.all_dirs(dest), set(['foo/bar']))
def _process_android_eclipse_project_data(self, data, srcdir, objdir): # This can't be relative to the environment's topsrcdir, # because during testing topsrcdir is faked. template_directory = os.path.abspath(mozpath.join(os.path.dirname(__file__), 'templates', 'android_eclipse')) project_directory = mozpath.join(self.environment.topobjdir, 'android_eclipse', data.name) manifest_path = mozpath.join(self.environment.topobjdir, 'android_eclipse', '%s.manifest' % data.name) manifest = self._manifest_for_project(srcdir, data) ensureParentDir(manifest_path) manifest.write(path=manifest_path) classpathentries = [] for cpe in sorted(data._classpathentries, key=lambda x: x.path): e = self._Element_for_classpathentry(cpe) classpathentries.append(ET.tostring(e)) for name in sorted(data.referenced_projects): e = self._Element_for_referenced_project(name) classpathentries.append(ET.tostring(e)) for name in sorted(data.extra_jars): e = self._Element_for_extra_jar(mozpath.join(srcdir, name)) classpathentries.append(ET.tostring(e)) defines = {} defines['IDE_OBJDIR'] = objdir defines['IDE_TOPOBJDIR'] = self.environment.topobjdir defines['IDE_SRCDIR'] = srcdir defines['IDE_TOPSRCDIR'] = self.environment.topsrcdir defines['IDE_PROJECT_NAME'] = data.name defines['IDE_PACKAGE_NAME'] = data.package_name defines['IDE_PROJECT_DIRECTORY'] = project_directory defines['IDE_RELSRCDIR'] = mozpath.relpath(srcdir, self.environment.topsrcdir) defines['IDE_CLASSPATH_ENTRIES'] = '\n'.join('\t' + cpe for cpe in classpathentries) defines['IDE_RECURSIVE_MAKE_TARGETS'] = ' '.join(sorted(data.recursive_make_targets)) # Like android.library=true defines['IDE_PROJECT_LIBRARY_SETTING'] = 'android.library=true' if data.is_library else '' # Like android.library.reference.1=FennecBrandingResources defines['IDE_PROJECT_LIBRARY_REFERENCES'] = '\n'.join( 'android.library.reference.%s=%s' % (i + 1, ref) for i, ref in enumerate(sorted(data.included_projects))) if data.filtered_resources: filteredResources = self._Element_for_filtered_resources(data.filtered_resources) defines['IDE_PROJECT_FILTERED_RESOURCES'] = pretty_print(filteredResources).strip() else: defines['IDE_PROJECT_FILTERED_RESOURCES'] = '' copier = FileCopier() finder = FileFinder(template_directory) for input_filename, f in itertools.chain(finder.find('**'), finder.find('.**')): if input_filename == 'AndroidManifest.xml' and not data.is_library: # Main projects supply their own manifests. continue copier.add(input_filename, PreprocessedFile( mozpath.join(finder.base, input_filename), depfile_path=None, marker='#', defines=defines, extra_depends={mozpath.join(finder.base, input_filename)})) # When we re-create the build backend, we kill everything that was there. if os.path.isdir(project_directory): self.summary.updated_count += 1 else: self.summary.created_count += 1 copier.copy(project_directory, skip_if_older=False, remove_unaccounted=True)
def main(): parser = ArgumentParser() parser.add_argument('-D', dest='defines', action='append', metavar="VAR[=VAL]", help='Define a variable') parser.add_argument('--format', default='omni', help='Choose the chrome format for packaging ' + '(omni, jar or flat ; default: %(default)s)') parser.add_argument('--removals', default=None, help='removed-files source file') parser.add_argument('--ignore-errors', action='store_true', default=False, help='Transform errors into warnings.') parser.add_argument('--minify', action='store_true', default=False, help='Make some files more compact while packaging') parser.add_argument('--minify-js', action='store_true', help='Minify JavaScript files while packaging.') parser.add_argument('--js-binary', help='Path to js binary. This is used to verify ' 'minified JavaScript. If this is not defined, ' 'minification verification will not be performed.') parser.add_argument('--jarlog', default='', help='File containing jar ' + 'access logs') parser.add_argument('--optimizejars', action='store_true', default=False, help='Enable jar optimizations') parser.add_argument('--disable-compression', action='store_false', dest='compress', default=True, help='Disable jar compression') parser.add_argument('manifest', default=None, nargs='?', help='Manifest file name') parser.add_argument('source', help='Source directory') parser.add_argument('destination', help='Destination directory') parser.add_argument('--non-resource', nargs='+', metavar='PATTERN', default=[], help='Extra files not to be considered as resources') args = parser.parse_args() defines = dict(buildconfig.defines) if args.ignore_errors: errors.ignore_errors() if args.defines: for name, value in [split_define(d) for d in args.defines]: defines[name] = value copier = FileCopier() if args.format == 'flat': formatter = FlatFormatter(copier) elif args.format == 'jar': formatter = JarFormatter(copier, compress=args.compress, optimize=args.optimizejars) elif args.format == 'omni': formatter = OmniJarFormatter(copier, buildconfig.substs['OMNIJAR_NAME'], compress=args.compress, optimize=args.optimizejars, non_resources=args.non_resource) else: errors.fatal('Unknown format: %s' % args.format) # Adjust defines according to the requested format. if isinstance(formatter, OmniJarFormatter): defines['MOZ_OMNIJAR'] = 1 elif 'MOZ_OMNIJAR' in defines: del defines['MOZ_OMNIJAR'] respath = '' if 'RESPATH' in defines: respath = SimpleManifestSink.normalize_path(defines['RESPATH']) while respath.startswith('/'): respath = respath[1:] if not buildconfig.substs['CROSS_COMPILE']: launcher.tooldir = mozpath.join(buildconfig.topobjdir, 'dist') with errors.accumulate(): finder_args = dict( minify=args.minify, minify_js=args.minify_js, ) if args.js_binary: finder_args['minify_js_verify_command'] = [ args.js_binary, os.path.join(os.path.abspath(os.path.dirname(__file__)), 'js-compare-ast.js') ] finder = FileFinder(args.source, find_executables=True, **finder_args) if 'NO_PKG_FILES' in os.environ: sinkformatter = NoPkgFilesRemover(formatter, args.manifest is not None) else: sinkformatter = formatter sink = SimpleManifestSink(finder, sinkformatter) if args.manifest: preprocess_manifest(sink, args.manifest, defines) else: sink.add(Component(''), 'bin/*') sink.close(args.manifest is not None) if args.removals: removals_in = StringIO(open(args.removals).read()) removals_in.name = args.removals removals = RemovedFiles(copier) preprocess(removals_in, removals, defines) copier.add(mozpath.join(respath, 'removed-files'), removals) # shlibsign libraries if launcher.can_launch(): if not mozinfo.isMac and buildconfig.substs.get('COMPILE_ENVIRONMENT'): for lib in SIGN_LIBS: libbase = mozpath.join(respath, '%s%s') \ % (buildconfig.substs['DLL_PREFIX'], lib) libname = '%s%s' % (libbase, buildconfig.substs['DLL_SUFFIX']) if copier.contains(libname): copier.add(libbase + '.chk', LibSignFile(os.path.join(args.destination, libname))) # Setup preloading if args.jarlog and os.path.exists(args.jarlog): from mozpack.mozjar import JarLog log = JarLog(args.jarlog) for p, f in copier: if not isinstance(f, Jarrer): continue key = JarLog.canonicalize(os.path.join(args.destination, p)) if key in log: f.preload(log[key]) copier.copy(args.destination) generate_precomplete(os.path.normpath(os.path.join(args.destination, respath)))
def _process_android_eclipse_project_data(self, data, srcdir, objdir): # This can't be relative to the environment's topsrcdir, # because during testing topsrcdir is faked. template_directory = os.path.abspath( mozpath.join(os.path.dirname(__file__), 'templates', 'android_eclipse')) project_directory = mozpath.join(self.environment.topobjdir, 'android_eclipse', data.name) manifest_path = mozpath.join(self.environment.topobjdir, 'android_eclipse', '%s.manifest' % data.name) manifest = self._manifest_for_project(srcdir, data) ensureParentDir(manifest_path) manifest.write(path=manifest_path) classpathentries = [] for cpe in sorted(data._classpathentries, key=lambda x: x.path): e = self._Element_for_classpathentry(cpe) classpathentries.append(ET.tostring(e)) for name in sorted(data.referenced_projects): e = self._Element_for_referenced_project(name) classpathentries.append(ET.tostring(e)) for name in sorted(data.extra_jars): e = self._Element_for_extra_jar(mozpath.join(srcdir, name)) classpathentries.append(ET.tostring(e)) defines = {} defines['IDE_OBJDIR'] = objdir defines['IDE_TOPOBJDIR'] = self.environment.topobjdir defines['IDE_SRCDIR'] = srcdir defines['IDE_TOPSRCDIR'] = self.environment.topsrcdir defines['IDE_PROJECT_NAME'] = data.name defines['IDE_PACKAGE_NAME'] = data.package_name defines['IDE_PROJECT_DIRECTORY'] = project_directory defines['IDE_RELSRCDIR'] = mozpath.relpath(srcdir, self.environment.topsrcdir) defines['IDE_CLASSPATH_ENTRIES'] = '\n'.join( '\t' + cpe for cpe in classpathentries) defines['IDE_RECURSIVE_MAKE_TARGETS'] = ' '.join( sorted(data.recursive_make_targets)) # Like android.library=true defines[ 'IDE_PROJECT_LIBRARY_SETTING'] = 'android.library=true' if data.is_library else '' # Like android.library.reference.1=FennecBrandingResources defines['IDE_PROJECT_LIBRARY_REFERENCES'] = '\n'.join( 'android.library.reference.%s=%s' % (i + 1, ref) for i, ref in enumerate(sorted(data.included_projects))) if data.filtered_resources: filteredResources = self._Element_for_filtered_resources( data.filtered_resources) defines['IDE_PROJECT_FILTERED_RESOURCES'] = pretty_print( filteredResources).strip() else: defines['IDE_PROJECT_FILTERED_RESOURCES'] = '' defines['ANDROID_TARGET_SDK'] = self.environment.substs[ 'ANDROID_TARGET_SDK'] defines['MOZ_ANDROID_MIN_SDK_VERSION'] = self.environment.defines[ 'MOZ_ANDROID_MIN_SDK_VERSION'] copier = FileCopier() finder = FileFinder(template_directory) for input_filename, f in itertools.chain(finder.find('**'), finder.find('.**')): if input_filename == 'AndroidManifest.xml' and not data.is_library: # Main projects supply their own manifests. continue copier.add( input_filename, PreprocessedFile( mozpath.join(finder.base, input_filename), depfile_path=None, marker='#', defines=defines, extra_depends={mozpath.join(finder.base, input_filename)})) # When we re-create the build backend, we kill everything that was there. if os.path.isdir(project_directory): self._updated_count += 1 else: self._created_count += 1 copier.copy(project_directory, skip_if_older=False, remove_unaccounted=True)
def main(): parser = ArgumentParser() parser.add_argument('-D', dest='defines', action='append', metavar="VAR[=VAL]", help='Define a variable') parser.add_argument('--format', default='omni', help='Choose the chrome format for packaging ' + '(omni, jar or flat ; default: %(default)s)') parser.add_argument('--removals', default=None, help='removed-files source file') parser.add_argument('--ignore-errors', action='store_true', default=False, help='Transform errors into warnings.') parser.add_argument('--ignore-broken-symlinks', action='store_true', default=False, help='Do not fail when processing broken symlinks.') parser.add_argument('--minify', action='store_true', default=False, help='Make some files more compact while packaging') parser.add_argument('--minify-js', action='store_true', help='Minify JavaScript files while packaging.') parser.add_argument('--js-binary', help='Path to js binary. This is used to verify ' 'minified JavaScript. If this is not defined, ' 'minification verification will not be performed.') parser.add_argument('--jarlog', default='', help='File containing jar ' + 'access logs') parser.add_argument('--compress', choices=('none', 'deflate', 'brotli'), default='deflate', help='Use given jar compression (default: deflate)') parser.add_argument('manifest', default=None, nargs='?', help='Manifest file name') parser.add_argument('source', help='Source directory') parser.add_argument('destination', help='Destination directory') parser.add_argument('--non-resource', nargs='+', metavar='PATTERN', default=[], help='Extra files not to be considered as resources') args = parser.parse_args() defines = dict(buildconfig.defines['ALLDEFINES']) if args.ignore_errors: errors.ignore_errors() if args.defines: for name, value in [split_define(d) for d in args.defines]: defines[name] = value compress = { 'none': False, 'deflate': True, 'brotli': JAR_BROTLI, }[args.compress] copier = FileCopier() if args.format == 'flat': formatter = FlatFormatter(copier) elif args.format == 'jar': formatter = JarFormatter(copier, compress=compress) elif args.format == 'omni': formatter = OmniJarFormatter(copier, buildconfig.substs['OMNIJAR_NAME'], compress=compress, non_resources=args.non_resource) else: errors.fatal('Unknown format: %s' % args.format) # Adjust defines according to the requested format. if isinstance(formatter, OmniJarFormatter): defines['MOZ_OMNIJAR'] = 1 elif 'MOZ_OMNIJAR' in defines: del defines['MOZ_OMNIJAR'] respath = '' if 'RESPATH' in defines: respath = SimpleManifestSink.normalize_path(defines['RESPATH']) while respath.startswith('/'): respath = respath[1:] if not buildconfig.substs['CROSS_COMPILE']: launcher.tooldir = mozpath.join(buildconfig.topobjdir, 'dist') with errors.accumulate(): finder_args = dict( minify=args.minify, minify_js=args.minify_js, ignore_broken_symlinks=args.ignore_broken_symlinks, ) if args.js_binary: finder_args['minify_js_verify_command'] = [ args.js_binary, os.path.join(os.path.abspath(os.path.dirname(__file__)), 'js-compare-ast.js') ] finder = FileFinder(args.source, find_executables=True, **finder_args) if 'NO_PKG_FILES' in os.environ: sinkformatter = NoPkgFilesRemover(formatter, args.manifest is not None) else: sinkformatter = formatter sink = SimpleManifestSink(finder, sinkformatter) if args.manifest: preprocess_manifest(sink, args.manifest, defines) else: sink.add(Component(''), 'bin/*') sink.close(args.manifest is not None) if args.removals: removals_in = StringIO(open(args.removals).read()) removals_in.name = args.removals removals = RemovedFiles(copier) preprocess(removals_in, removals, defines) copier.add(mozpath.join(respath, 'removed-files'), removals) # shlibsign libraries if launcher.can_launch(): if not mozinfo.isMac and buildconfig.substs.get('COMPILE_ENVIRONMENT'): for lib in SIGN_LIBS: libbase = mozpath.join(respath, '%s%s') \ % (buildconfig.substs['DLL_PREFIX'], lib) libname = '%s%s' % (libbase, buildconfig.substs['DLL_SUFFIX']) if copier.contains(libname): copier.add(libbase + '.chk', LibSignFile(os.path.join(args.destination, libname))) # If a pdb file is present and we were instructed to copy it, include it. # Run on all OSes to capture MinGW builds if buildconfig.substs.get('MOZ_COPY_PDBS'): for p, f in copier: if isinstance(f, ExecutableFile): pdbname = os.path.splitext(f.inputs()[0])[0] + '.pdb' if os.path.exists(pdbname): copier.add(os.path.basename(pdbname), File(pdbname)) # Setup preloading if args.jarlog: if not os.path.exists(args.jarlog): raise Exception('Cannot find jar log: %s' % args.jarlog) omnijars = [] if isinstance(formatter, OmniJarFormatter): omnijars = [mozpath.join(base, buildconfig.substs['OMNIJAR_NAME']) for base in sink.packager.get_bases(addons=False)] from mozpack.mozjar import JarLog log = JarLog(args.jarlog) for p, f in copier: if not isinstance(f, Jarrer): continue if respath: p = mozpath.relpath(p, respath) if p in log: f.preload(log[p]) elif p in omnijars: raise Exception('No jar log data for %s' % p) copier.copy(args.destination) generate_precomplete(os.path.normpath(os.path.join(args.destination, respath)))
def main(): parser = ArgumentParser() parser.add_argument('-D', dest='defines', action='append', metavar="VAR[=VAL]", help='Define a variable') parser.add_argument('--format', default='omni', help='Choose the chrome format for packaging ' + '(omni, jar or flat ; default: %(default)s)') parser.add_argument('--removals', default=None, help='removed-files source file') parser.add_argument('--ignore-errors', action='store_true', default=False, help='Transform errors into warnings.') parser.add_argument('--minify', action='store_true', default=False, help='Make some files more compact while packaging') parser.add_argument('--minify-js', action='store_true', help='Minify JavaScript files while packaging.') parser.add_argument('--js-binary', help='Path to js binary. This is used to verify ' 'minified JavaScript. If this is not defined, ' 'minification verification will not be performed.') parser.add_argument('--jarlog', default='', help='File containing jar ' + 'access logs') parser.add_argument('--optimizejars', action='store_true', default=False, help='Enable jar optimizations') parser.add_argument('--unify', default='', help='Base directory of another build to unify with') parser.add_argument('manifest', default=None, nargs='?', help='Manifest file name') parser.add_argument('source', help='Source directory') parser.add_argument('destination', help='Destination directory') parser.add_argument('--non-resource', nargs='+', metavar='PATTERN', default=[], help='Extra files not to be considered as resources') args = parser.parse_args() defines = dict(buildconfig.defines) if args.ignore_errors: errors.ignore_errors() if args.defines: for name, value in [split_define(d) for d in args.defines]: defines[name] = value copier = FileCopier() if args.format == 'flat': formatter = FlatFormatter(copier) elif args.format == 'jar': formatter = JarFormatter(copier, optimize=args.optimizejars) elif args.format == 'omni': formatter = OmniJarFormatter(copier, buildconfig.substs['OMNIJAR_NAME'], optimize=args.optimizejars, non_resources=args.non_resource) else: errors.fatal('Unknown format: %s' % args.format) # Adjust defines according to the requested format. if isinstance(formatter, OmniJarFormatter): defines['MOZ_OMNIJAR'] = 1 elif 'MOZ_OMNIJAR' in defines: del defines['MOZ_OMNIJAR'] respath = '' if 'RESPATH' in defines: respath = SimpleManifestSink.normalize_path(defines['RESPATH']) while respath.startswith('/'): respath = respath[1:] if args.unify: def is_native(path): path = os.path.abspath(path) return platform.machine() in mozpath.split(path) # Invert args.unify and args.source if args.unify points to the # native architecture. args.source, args.unify = sorted([args.source, args.unify], key=is_native, reverse=True) if is_native(args.source): launcher.tooldir = args.source elif not buildconfig.substs['CROSS_COMPILE']: launcher.tooldir = buildconfig.substs['LIBXUL_DIST'] with errors.accumulate(): finder_args = dict( minify=args.minify, minify_js=args.minify_js, ) if args.js_binary: finder_args['minify_js_verify_command'] = [ args.js_binary, os.path.join(os.path.abspath(os.path.dirname(__file__)), 'js-compare-ast.js') ] if args.unify: finder = UnifiedBuildFinder(FileFinder(args.source), FileFinder(args.unify), **finder_args) else: finder = FileFinder(args.source, **finder_args) if 'NO_PKG_FILES' in os.environ: sinkformatter = NoPkgFilesRemover(formatter, args.manifest is not None) else: sinkformatter = formatter sink = SimpleManifestSink(finder, sinkformatter) if args.manifest: preprocess_manifest(sink, args.manifest, defines) else: sink.add(Component(''), 'bin/*') sink.close(args.manifest is not None) if args.removals: removals_in = StringIO(open(args.removals).read()) removals_in.name = args.removals removals = RemovedFiles(copier) preprocess(removals_in, removals, defines) copier.add(mozpath.join(respath, 'removed-files'), removals) # shlibsign libraries if launcher.can_launch(): if not mozinfo.isMac: for lib in SIGN_LIBS: libbase = mozpath.join(respath, '%s%s') \ % (buildconfig.substs['DLL_PREFIX'], lib) libname = '%s%s' % (libbase, buildconfig.substs['DLL_SUFFIX']) if copier.contains(libname): copier.add(libbase + '.chk', LibSignFile(os.path.join(args.destination, libname))) # Setup preloading if args.jarlog and os.path.exists(args.jarlog): from mozpack.mozjar import JarLog log = JarLog(args.jarlog) for p, f in copier: if not isinstance(f, Jarrer): continue key = JarLog.canonicalize(os.path.join(args.destination, p)) if key in log: f.preload(log[key]) # Fill startup cache if isinstance(formatter, OmniJarFormatter) and launcher.can_launch() \ and buildconfig.substs['MOZ_DISABLE_STARTUPCACHE'] != '1': gre_path = None def get_bases(): for b in sink.packager.get_bases(addons=False): for p in (mozpath.join('bin', b), b): if os.path.exists(os.path.join(args.source, p)): yield p break for base in sorted(get_bases()): if not gre_path: gre_path = base base_path = sink.normalize_path(base) if base_path in formatter.omnijars: precompile_cache(formatter.omnijars[base_path], args.source, gre_path, base) copier.copy(args.destination)
def main(): parser = ArgumentParser() parser.add_argument("-D", dest="defines", action="append", metavar="VAR[=VAL]", help="Define a variable") parser.add_argument( "--format", default="omni", help="Choose the chrome format for packaging " + "(omni, jar or flat ; default: %(default)s)", ) parser.add_argument("--removals", default=None, help="removed-files source file") parser.add_argument("--ignore-errors", action="store_true", default=False, help="Transform errors into warnings.") parser.add_argument( "--minify", action="store_true", default=False, help="Make some files more compact while packaging" ) parser.add_argument("--minify-js", action="store_true", help="Minify JavaScript files while packaging.") parser.add_argument( "--js-binary", help="Path to js binary. This is used to verify " "minified JavaScript. If this is not defined, " "minification verification will not be performed.", ) parser.add_argument("--jarlog", default="", help="File containing jar " + "access logs") parser.add_argument("--optimizejars", action="store_true", default=False, help="Enable jar optimizations") parser.add_argument("--unify", default="", help="Base directory of another build to unify with") parser.add_argument( "--disable-compression", action="store_false", dest="compress", default=True, help="Disable jar compression" ) parser.add_argument("manifest", default=None, nargs="?", help="Manifest file name") parser.add_argument("source", help="Source directory") parser.add_argument("destination", help="Destination directory") parser.add_argument( "--non-resource", nargs="+", metavar="PATTERN", default=[], help="Extra files not to be considered as resources" ) args = parser.parse_args() defines = dict(buildconfig.defines) if args.ignore_errors: errors.ignore_errors() if args.defines: for name, value in [split_define(d) for d in args.defines]: defines[name] = value copier = FileCopier() if args.format == "flat": formatter = FlatFormatter(copier) elif args.format == "jar": formatter = JarFormatter(copier, compress=args.compress, optimize=args.optimizejars) elif args.format == "omni": formatter = OmniJarFormatter( copier, buildconfig.substs["OMNIJAR_NAME"], compress=args.compress, optimize=args.optimizejars, non_resources=args.non_resource, ) else: errors.fatal("Unknown format: %s" % args.format) # Adjust defines according to the requested format. if isinstance(formatter, OmniJarFormatter): defines["MOZ_OMNIJAR"] = 1 elif "MOZ_OMNIJAR" in defines: del defines["MOZ_OMNIJAR"] respath = "" if "RESPATH" in defines: respath = SimpleManifestSink.normalize_path(defines["RESPATH"]) while respath.startswith("/"): respath = respath[1:] if args.unify: def is_native(path): path = os.path.abspath(path) return platform.machine() in mozpath.split(path) # Invert args.unify and args.source if args.unify points to the # native architecture. args.source, args.unify = sorted([args.source, args.unify], key=is_native, reverse=True) if is_native(args.source): launcher.tooldir = args.source elif not buildconfig.substs["CROSS_COMPILE"]: launcher.tooldir = mozpath.join(buildconfig.topobjdir, "dist") with errors.accumulate(): finder_args = dict(minify=args.minify, minify_js=args.minify_js) if args.js_binary: finder_args["minify_js_verify_command"] = [ args.js_binary, os.path.join(os.path.abspath(os.path.dirname(__file__)), "js-compare-ast.js"), ] if args.unify: finder = UnifiedBuildFinder(FileFinder(args.source), FileFinder(args.unify), **finder_args) else: finder = FileFinder(args.source, **finder_args) if "NO_PKG_FILES" in os.environ: sinkformatter = NoPkgFilesRemover(formatter, args.manifest is not None) else: sinkformatter = formatter sink = SimpleManifestSink(finder, sinkformatter) if args.manifest: preprocess_manifest(sink, args.manifest, defines) else: sink.add(Component(""), "bin/*") sink.close(args.manifest is not None) if args.removals: removals_in = StringIO(open(args.removals).read()) removals_in.name = args.removals removals = RemovedFiles(copier) preprocess(removals_in, removals, defines) copier.add(mozpath.join(respath, "removed-files"), removals) # shlibsign libraries if launcher.can_launch(): if not mozinfo.isMac and buildconfig.substs.get("COMPILE_ENVIRONMENT"): for lib in SIGN_LIBS: libbase = mozpath.join(respath, "%s%s") % (buildconfig.substs["DLL_PREFIX"], lib) libname = "%s%s" % (libbase, buildconfig.substs["DLL_SUFFIX"]) if copier.contains(libname): copier.add(libbase + ".chk", LibSignFile(os.path.join(args.destination, libname))) # Setup preloading if args.jarlog and os.path.exists(args.jarlog): from mozpack.mozjar import JarLog log = JarLog(args.jarlog) for p, f in copier: if not isinstance(f, Jarrer): continue key = JarLog.canonicalize(os.path.join(args.destination, p)) if key in log: f.preload(log[key]) # Fill startup cache if ( isinstance(formatter, OmniJarFormatter) and launcher.can_launch() and buildconfig.substs["MOZ_DISABLE_STARTUPCACHE"] != "1" ): gre_path = None def get_bases(): for b in sink.packager.get_bases(addons=False): for p in (mozpath.join("bin", b), b): if os.path.exists(os.path.join(args.source, p)): yield p break for base in sorted(get_bases()): if not gre_path: gre_path = base omnijar_path = mozpath.join(sink.normalize_path(base), buildconfig.substs["OMNIJAR_NAME"]) if formatter.contains(omnijar_path): precompile_cache(formatter.copier[omnijar_path], args.source, gre_path, base) copier.copy(args.destination)