def unpack(source): ''' Transform a jar chrome or omnijar packaged directory into a flat package. ''' copier = FileCopier() unpack_to_registry(source, copier) copier.copy(source, skip_if_older=False)
def test_optional_exists_creates_unneeded_directory(self): """Demonstrate that a directory not strictly required, but specified as the path to an optional file, will be unnecessarily created. This behaviour is wrong; fixing it is tracked by Bug 972432; and this test exists to guard against unexpected changes in behaviour. """ dest = self.tmppath('dest') copier = FileCopier() copier.add('foo/bar', ExistingFile(required=False)) result = copier.copy(dest) st = os.lstat(self.tmppath('dest/foo')) self.assertFalse(stat.S_ISLNK(st.st_mode)) self.assertTrue(stat.S_ISDIR(st.st_mode)) # What's worse, we have no record that dest was created. self.assertEquals(len(result.updated_files), 0) # But we do have an erroneous record of an optional file # existing when it does not. self.assertIn(self.tmppath('dest/foo/bar'), result.existing_files)
def test_symlink_directory_replaced(self): """Directory symlinks in destination are replaced if they need to be real directories.""" if not self.symlink_supported: return dest = self.tmppath('dest') copier = FileCopier() copier.add('foo/bar/baz', GeneratedFile('foobarbaz')) os.makedirs(self.tmppath('dest/foo')) dummy = self.tmppath('dummy') os.mkdir(dummy) link = self.tmppath('dest/foo/bar') os.symlink(dummy, link) result = copier.copy(dest) st = os.lstat(link) self.assertFalse(stat.S_ISLNK(st.st_mode)) self.assertTrue(stat.S_ISDIR(st.st_mode)) self.assertEqual(self.all_files(dest), set(copier.paths())) self.assertEqual(result.removed_directories, set()) self.assertEqual(len(result.updated_files), 1)
def output_changes(self, verbose=True): ''' Return an iterator of `FasterBuildChange` instances as outputs from the faster build system are updated. ''' for change in self.input_changes(verbose=verbose): now = datetime.datetime.utcnow() for unrecognized in sorted(change.unrecognized): print_line('watch', '! {}'.format(unrecognized), now=now) all_outputs = set() for input in sorted(change.input_to_outputs): outputs = change.input_to_outputs[input] print_line('watch', '< {}'.format(input), now=now) for output in sorted(outputs): print_line('watch', '> {}'.format(output), now=now) all_outputs |= outputs if all_outputs: partial_copier = FileCopier() for output in all_outputs: partial_copier.add(output, self.file_copier[output]) self.incremental_copy(partial_copier, force=True, verbose=verbose) yield change
def _synchronize_docs(self): m = InstallManifest() m.add_symlink(self._conf_py_path, "conf.py") for dest, source in sorted(self._trees.items()): source_dir = os.path.join(self._topsrcdir, source) for root, dirs, files in os.walk(source_dir): for f in files: source_path = os.path.join(root, f) rel_source = source_path[len(source_dir) + 1 :] m.add_symlink(source_path, os.path.join(dest, rel_source)) copier = FileCopier() m.populate_registry(copier) copier.copy(self._docs_dir) with open(self._index_path, "rb") as fh: data = fh.read() indexes = ["%s/index" % p for p in sorted(self._trees.keys())] indexes = "\n ".join(indexes) packages = [os.path.basename(p) for p in self._python_package_dirs] packages = ["python/%s" % p for p in packages] packages = "\n ".join(sorted(packages)) data = data.format(indexes=indexes, python_packages=packages) with open(os.path.join(self._docs_dir, "index.rst"), "wb") as fh: fh.write(data)
def _synchronize_docs(self): m = InstallManifest() m.add_symlink(self._conf_py_path, 'conf.py') for dest, source in sorted(self._trees.items()): source_dir = os.path.join(self._topsrcdir, source) for root, dirs, files in os.walk(source_dir): for f in files: source_path = os.path.join(root, f) rel_source = source_path[len(source_dir) + 1:] m.add_symlink(source_path, os.path.join(dest, rel_source)) stage_dir = os.path.join(self._output_dir, 'staging') copier = FileCopier() m.populate_registry(copier) copier.copy(stage_dir) with open(self._index_path, 'rb') as fh: data = fh.read() indexes = ['%s/index' % p for p in sorted(self._trees.keys())] indexes = '\n '.join(indexes) packages = [os.path.basename(p) for p in self._python_package_dirs] packages = ['python/%s' % p for p in packages] packages = '\n '.join(sorted(packages)) data = data.format(indexes=indexes, python_packages=packages) with open(os.path.join(stage_dir, 'index.rst'), 'wb') as fh: fh.write(data)
def strip(dir): copier = FileCopier() # The FileFinder will give use ExecutableFile instances for files # that can be stripped, and copying ExecutableFiles defaults to # stripping them unless buildconfig.substs['PKG_SKIP_STRIP'] is set. for p, f in FileFinder(dir): copier.add(p, f) copier.copy(dir)
def process_manifest(destdir, paths, remove_unaccounted=True): manifest = InstallManifest() for path in paths: manifest |= InstallManifest(path=path) copier = FileCopier() manifest.populate_registry(copier) return copier.copy(destdir, remove_unaccounted=remove_unaccounted)
def process_manifest(destdir, *paths): manifest = InstallManifest() for path in paths: manifest |= InstallManifest(path=path) copier = FileCopier() manifest.populate_registry(copier) return copier.copy(destdir)
def install_test_files(topsrcdir, topobjdir, tests_root, test_objs): """Installs the requested test files to the objdir. This is invoked by test runners to avoid installing tens of thousands of test files when only a few tests need to be run. """ flavor_info = {flavor: (root, prefix, install) for (flavor, root, prefix, install) in TEST_MANIFESTS.values()} objdir_dest = mozpath.join(topobjdir, tests_root) converter = SupportFilesConverter() install_info = TestInstallInfo() for o in test_objs: flavor = o['flavor'] if flavor not in flavor_info: # This is a test flavor that isn't installed by the build system. continue root, prefix, install = flavor_info[flavor] if not install: # This flavor isn't installed to the objdir. continue manifest_path = o['manifest'] manifest_dir = mozpath.dirname(manifest_path) out_dir = mozpath.join(root, prefix, manifest_dir[len(topsrcdir) + 1:]) file_relpath = o['file_relpath'] source = mozpath.join(topsrcdir, file_relpath) dest = mozpath.join(root, prefix, file_relpath) if 'install-to-subdir' in o: out_dir = mozpath.join(out_dir, o['install-to-subdir']) manifest_relpath = mozpath.relpath(source, mozpath.dirname(manifest_path)) dest = mozpath.join(out_dir, manifest_relpath) install_info.installs.append((source, dest)) install_info |= converter.convert_support_files(o, root, manifest_dir, out_dir) manifest = InstallManifest() for source, dest in set(install_info.installs): if dest in install_info.external_installs: continue manifest.add_symlink(source, dest) for base, pattern, dest in install_info.pattern_installs: manifest.add_pattern_symlink(base, pattern, dest) _resolve_installs(install_info.deferred_installs, topobjdir, manifest) # Harness files are treated as a monolith and installed each time we run tests. # Fortunately there are not very many. manifest |= InstallManifest(mozpath.join(topobjdir, '_build_manifests', 'install', tests_root)) copier = FileCopier() manifest.populate_registry(copier) copier.copy(objdir_dest, remove_unaccounted=False)
def unpack(source): ''' Transform a jar chrome or omnijar packaged directory into a flat package. ''' copier = FileCopier() finder = UnpackFinder(source) packager = SimplePackager(FlatFormatter(copier)) for p, f in finder.find('*'): if mozpack.path.split(p)[0] not in STARTUP_CACHE_PATHS: packager.add(p, f) packager.close() copier.copy(source, skip_if_older=False)
def test_copier_application(self): dest = self.tmppath('dest') os.mkdir(dest) to_delete = self.tmppath('dest/to_delete') with open(to_delete, 'a'): pass with open(self.tmppath('s_source'), 'wt') as fh: fh.write('symlink!') with open(self.tmppath('c_source'), 'wt') as fh: fh.write('copy!') with open(self.tmppath('p_source'), 'wt') as fh: fh.write('#define FOO 1\npreprocess!') with open(self.tmppath('dest/e_dest'), 'a'): pass with open(self.tmppath('dest/o_dest'), 'a'): pass m = self._get_test_manifest() c = FileCopier() m.populate_registry(c) result = c.copy(dest) self.assertTrue(os.path.exists(self.tmppath('dest/s_dest'))) self.assertTrue(os.path.exists(self.tmppath('dest/c_dest'))) self.assertTrue(os.path.exists(self.tmppath('dest/p_dest'))) self.assertTrue(os.path.exists(self.tmppath('dest/e_dest'))) self.assertTrue(os.path.exists(self.tmppath('dest/o_dest'))) self.assertTrue(os.path.exists(self.tmppath('dest/content'))) self.assertFalse(os.path.exists(to_delete)) with open(self.tmppath('dest/s_dest'), 'rt') as fh: self.assertEqual(fh.read(), 'symlink!') with open(self.tmppath('dest/c_dest'), 'rt') as fh: self.assertEqual(fh.read(), 'copy!') with open(self.tmppath('dest/p_dest'), 'rt') as fh: self.assertEqual(fh.read(), 'preprocess!') self.assertEqual(result.updated_files, set(self.tmppath(p) for p in ( 'dest/s_dest', 'dest/c_dest', 'dest/p_dest', 'dest/content'))) self.assertEqual(result.existing_files, set([self.tmppath('dest/e_dest'), self.tmppath('dest/o_dest')])) self.assertEqual(result.removed_files, {to_delete}) self.assertEqual(result.removed_directories, set())
def process_manifest(destdir, paths, remove_unaccounted=True, remove_all_directory_symlinks=True, remove_empty_directories=True): manifest = InstallManifest() for path in paths: manifest |= InstallManifest(path=path) copier = FileCopier() manifest.populate_registry(copier) return copier.copy(destdir, remove_unaccounted=remove_unaccounted, remove_all_directory_symlinks=remove_all_directory_symlinks, remove_empty_directories=remove_empty_directories)
def test_copier_application(self): dest = self.tmppath("dest") os.mkdir(dest) to_delete = self.tmppath("dest/to_delete") with open(to_delete, "a"): pass with open(self.tmppath("s_source"), "wt") as fh: fh.write("symlink!") with open(self.tmppath("c_source"), "wt") as fh: fh.write("copy!") with open(self.tmppath("p_source"), "wt") as fh: fh.write("#define FOO 1\npreprocess!") with open(self.tmppath("dest/e_dest"), "a"): pass with open(self.tmppath("dest/o_dest"), "a"): pass m = self._get_test_manifest() c = FileCopier() m.populate_registry(c) result = c.copy(dest) self.assertTrue(os.path.exists(self.tmppath("dest/s_dest"))) self.assertTrue(os.path.exists(self.tmppath("dest/c_dest"))) self.assertTrue(os.path.exists(self.tmppath("dest/p_dest"))) self.assertTrue(os.path.exists(self.tmppath("dest/e_dest"))) self.assertTrue(os.path.exists(self.tmppath("dest/o_dest"))) self.assertFalse(os.path.exists(to_delete)) with open(self.tmppath("dest/s_dest"), "rt") as fh: self.assertEqual(fh.read(), "symlink!") with open(self.tmppath("dest/c_dest"), "rt") as fh: self.assertEqual(fh.read(), "copy!") with open(self.tmppath("dest/p_dest"), "rt") as fh: self.assertEqual(fh.read(), "preprocess!") self.assertEqual( result.updated_files, set(self.tmppath(p) for p in ("dest/s_dest", "dest/c_dest", "dest/p_dest")) ) self.assertEqual(result.existing_files, set([self.tmppath("dest/e_dest"), self.tmppath("dest/o_dest")])) self.assertEqual(result.removed_files, {to_delete}) self.assertEqual(result.removed_directories, set())
def test_no_remove(self): copier = FileCopier() copier.add('foo', GeneratedFile('foo')) with open(self.tmppath('bar'), 'a'): pass os.mkdir(self.tmppath('emptydir')) result = copier.copy(self.tmpdir, remove_unaccounted=False) self.assertEqual(self.all_files(self.tmpdir), set(['foo', 'bar'])) self.assertEqual(result.removed_files, set()) self.assertEqual(result.removed_directories, set([self.tmppath('emptydir')]))
def process_manifest( destdir, paths, track=None, remove_unaccounted=True, remove_all_directory_symlinks=True, remove_empty_directories=True, defines={}, ): if track: if os.path.exists(track): # We use the same format as install manifests for the tracking # data. manifest = InstallManifest(path=track) remove_unaccounted = FileRegistry() dummy_file = BaseFile() finder = FileFinder(destdir, find_executables=False, find_dotfiles=True) for dest in manifest._dests: for p, f in finder.find(dest): remove_unaccounted.add(p, dummy_file) else: # If tracking is enabled and there is no file, we don't want to # be removing anything. remove_unaccounted = False remove_empty_directories = False remove_all_directory_symlinks = False manifest = InstallManifest() for path in paths: manifest |= InstallManifest(path=path) copier = FileCopier() manifest.populate_registry(copier, defines_override=defines) result = copier.copy( destdir, remove_unaccounted=remove_unaccounted, remove_all_directory_symlinks=remove_all_directory_symlinks, remove_empty_directories=remove_empty_directories, ) if track: manifest.write(path=track) return result
def repack(source, l10n, extra_l10n={}, non_resources=[], non_chrome=set()): ''' Replace localized data from the `source` directory with localized data from `l10n` and `extra_l10n`. The `source` argument points to a directory containing a packaged application (in omnijar, jar or flat form). The `l10n` argument points to a directory containing the main localized data (usually in the form of a language pack addon) to use to replace in the packaged application. The `extra_l10n` argument contains a dict associating relative paths in the source to separate directories containing localized data for them. This can be used to point at different language pack addons for different parts of the package application. The `non_resources` argument gives a list of relative paths in the source that should not be added in an omnijar in case the packaged application is in that format. The `non_chrome` argument gives a list of file/directory patterns for localized files that are not listed in a chrome.manifest. ''' app_finder = UnpackFinder(source) l10n_finder = UnpackFinder(l10n) if extra_l10n: finders = { '': l10n_finder, } for base, path in extra_l10n.iteritems(): finders[base] = UnpackFinder(path) l10n_finder = ComposedFinder(finders) copier = FileCopier() compress = min(app_finder.compressed, JAR_DEFLATED) if app_finder.kind == 'flat': formatter = FlatFormatter(copier) elif app_finder.kind == 'jar': formatter = JarFormatter(copier, optimize=app_finder.optimizedjars, compress=compress) elif app_finder.kind == 'omni': formatter = OmniJarFormatter(copier, app_finder.omnijar, optimize=app_finder.optimizedjars, compress=compress, non_resources=non_resources) with errors.accumulate(): _repack(app_finder, l10n_finder, copier, formatter, non_chrome) copier.copy(source, skip_if_older=False) generate_precomplete(source)
def repack(source, l10n, non_resources=[], non_chrome=set()): app_finder = UnpackFinder(source) l10n_finder = UnpackFinder(l10n) copier = FileCopier() if app_finder.kind == 'flat': formatter = FlatFormatter(copier) elif app_finder.kind == 'jar': formatter = JarFormatter(copier, optimize=app_finder.optimizedjars) elif app_finder.kind == 'omni': formatter = OmniJarFormatter(copier, app_finder.omnijar, optimize=app_finder.optimizedjars, non_resources=non_resources) with errors.accumulate(): _repack(app_finder, l10n_finder, copier, formatter, non_chrome) copier.copy(source, skip_if_older=False) generate_precomplete(source)
def test_pattern_expansion(self): source = self.tmppath("source") os.mkdir(source) os.mkdir("%s/base" % source) os.mkdir("%s/base/foo" % source) with open("%s/base/foo/file1" % source, "a"): pass with open("%s/base/foo/file2" % source, "a"): pass m = InstallManifest() m.add_pattern_symlink("%s/base" % source, "**", "dest") c = FileCopier() m.populate_registry(c) self.assertEqual(c.paths(), ["dest/foo/file1", "dest/foo/file2"])
def test_pattern_expansion(self): source = self.tmppath('source') os.mkdir(source) os.mkdir('%s/base' % source) os.mkdir('%s/base/foo' % source) with open('%s/base/foo/file1' % source, 'a'): pass with open('%s/base/foo/file2' % source, 'a'): pass m = InstallManifest() m.add_pattern_symlink('%s/base' % source, '**', 'dest') c = FileCopier() m.populate_registry(c) self.assertEqual(c.paths(), ['dest/foo/file1', 'dest/foo/file2'])
def test_no_remove_empty_directories(self): copier = FileCopier() copier.add("foo", GeneratedFile("foo")) with open(self.tmppath("bar"), "a"): pass os.mkdir(self.tmppath("emptydir")) d = self.tmppath("populateddir") os.mkdir(d) with open(self.tmppath("populateddir/foo"), "a"): pass result = copier.copy(self.tmpdir, remove_unaccounted=False, remove_empty_directories=False) self.assertEqual(self.all_files(self.tmpdir), set(["foo", "bar", "populateddir/foo"])) self.assertEqual(self.all_dirs(self.tmpdir), set(["emptydir", "populateddir"])) self.assertEqual(result.removed_files, set()) self.assertEqual(result.removed_directories, set())
def test_permissions(self): """Ensure files without write permission can be deleted.""" with open(self.tmppath('dummy'), 'a'): pass p = self.tmppath('no_perms') with open(p, 'a'): pass # Make file and directory unwritable. Reminder: making a directory # unwritable prevents modifications (including deletes) from the list # of files in that directory. os.chmod(p, 0400) os.chmod(self.tmpdir, 0400) copier = FileCopier() copier.add('dummy', GeneratedFile('content')) result = copier.copy(self.tmpdir) self.assertEqual(result.removed_files_count, 1) self.assertFalse(os.path.exists(p))
def test_add_entries_from(self): source = self.tmppath('source') os.mkdir(source) os.mkdir('%s/base' % source) os.mkdir('%s/base/foo' % source) with open('%s/base/foo/file1' % source, 'a'): pass with open('%s/base/foo/file2' % source, 'a'): pass m = InstallManifest() m.add_pattern_link('%s/base' % source, '**', 'dest') p = InstallManifest() p.add_entries_from(m) self.assertEqual(len(p), 1) c = FileCopier() p.populate_registry(c) self.assertEqual(c.paths(), ['dest/foo/file1', 'dest/foo/file2']) q = InstallManifest() q.add_entries_from(m, base='target') self.assertEqual(len(q), 1) d = FileCopier() q.populate_registry(d) self.assertEqual(d.paths(), ['target/dest/foo/file1', 'target/dest/foo/file2']) # Some of the values in an InstallManifest include destination # information that is present in the keys. Verify that we can # round-trip serialization. r = InstallManifest() r.add_entries_from(m) r.add_entries_from(m, base='target') self.assertEqual(len(r), 2) temp_path = self.tmppath('temp_path') r.write(path=temp_path) s = InstallManifest(path=temp_path) e = FileCopier() s.populate_registry(e) self.assertEqual(e.paths(), ['dest/foo/file1', 'dest/foo/file2', 'target/dest/foo/file1', 'target/dest/foo/file2'])
def test_file_copier(self): copier = FileCopier() copier.add('foo/bar', GeneratedFile('foobar')) copier.add('foo/qux', GeneratedFile('fooqux')) copier.add('foo/deep/nested/directory/file', GeneratedFile('fooz')) copier.add('bar', GeneratedFile('bar')) copier.add('qux/foo', GeneratedFile('quxfoo')) copier.add('qux/bar', GeneratedFile('')) copier.copy(self.tmpdir) self.assertEqual(self.all_files(self.tmpdir), set(copier.paths())) self.assertEqual(self.all_dirs(self.tmpdir), set(['foo/deep/nested/directory', 'qux'])) copier.remove('foo') copier.add('test', GeneratedFile('test')) copier.copy(self.tmpdir) self.assertEqual(self.all_files(self.tmpdir), set(copier.paths())) self.assertEqual(self.all_dirs(self.tmpdir), set(['qux']))
def main(): parser = ArgumentParser() parser.add_argument('-D', dest='defines', action='append', metavar="VAR[=VAL]", help='Define a variable') parser.add_argument('--format', default='omni', help='Choose the chrome format for packaging ' + '(omni, jar or flat ; default: %(default)s)') parser.add_argument('--removals', default=None, help='removed-files source file') parser.add_argument('--ignore-errors', action='store_true', default=False, help='Transform errors into warnings.') parser.add_argument('--minify', action='store_true', default=False, help='Make some files more compact while packaging') parser.add_argument('--minify-js', action='store_true', help='Minify JavaScript files while packaging.') parser.add_argument('--js-binary', help='Path to js binary. This is used to verify ' 'minified JavaScript. If this is not defined, ' 'minification verification will not be performed.') parser.add_argument('--jarlog', default='', help='File containing jar ' + 'access logs') parser.add_argument('--optimizejars', action='store_true', default=False, help='Enable jar optimizations') parser.add_argument('--compress', choices=('none', 'deflate', 'brotli'), default='deflate', help='Use given jar compression (default: deflate)') parser.add_argument('manifest', default=None, nargs='?', help='Manifest file name') parser.add_argument('source', help='Source directory') parser.add_argument('destination', help='Destination directory') parser.add_argument('--non-resource', nargs='+', metavar='PATTERN', default=[], help='Extra files not to be considered as resources') args = parser.parse_args() defines = dict(buildconfig.defines['ALLDEFINES']) if args.ignore_errors: errors.ignore_errors() if args.defines: for name, value in [split_define(d) for d in args.defines]: defines[name] = value compress = { 'none': False, 'deflate': True, 'brotli': JAR_BROTLI, }[args.compress] copier = FileCopier() if args.format == 'flat': formatter = FlatFormatter(copier) elif args.format == 'jar': formatter = JarFormatter(copier, compress=compress, optimize=args.optimizejars) elif args.format == 'omni': formatter = OmniJarFormatter(copier, buildconfig.substs['OMNIJAR_NAME'], compress=compress, optimize=args.optimizejars, non_resources=args.non_resource) else: errors.fatal('Unknown format: %s' % args.format) # Adjust defines according to the requested format. if isinstance(formatter, OmniJarFormatter): defines['MOZ_OMNIJAR'] = 1 elif 'MOZ_OMNIJAR' in defines: del defines['MOZ_OMNIJAR'] respath = '' if 'RESPATH' in defines: respath = SimpleManifestSink.normalize_path(defines['RESPATH']) while respath.startswith('/'): respath = respath[1:] if not buildconfig.substs['CROSS_COMPILE']: launcher.tooldir = mozpath.join(buildconfig.topobjdir, 'dist') with errors.accumulate(): finder_args = dict( minify=args.minify, minify_js=args.minify_js, ) if args.js_binary: finder_args['minify_js_verify_command'] = [ args.js_binary, os.path.join(os.path.abspath(os.path.dirname(__file__)), 'js-compare-ast.js') ] finder = FileFinder(args.source, find_executables=True, **finder_args) if 'NO_PKG_FILES' in os.environ: sinkformatter = NoPkgFilesRemover(formatter, args.manifest is not None) else: sinkformatter = formatter sink = SimpleManifestSink(finder, sinkformatter) if args.manifest: preprocess_manifest(sink, args.manifest, defines) else: sink.add(Component(''), 'bin/*') sink.close(args.manifest is not None) if args.removals: removals_in = StringIO(open(args.removals).read()) removals_in.name = args.removals removals = RemovedFiles(copier) preprocess(removals_in, removals, defines) copier.add(mozpath.join(respath, 'removed-files'), removals) # shlibsign libraries if launcher.can_launch(): if not mozinfo.isMac and buildconfig.substs.get('COMPILE_ENVIRONMENT'): for lib in SIGN_LIBS: libbase = mozpath.join(respath, '%s%s') \ % (buildconfig.substs['DLL_PREFIX'], lib) libname = '%s%s' % (libbase, buildconfig.substs['DLL_SUFFIX']) if copier.contains(libname): copier.add( libbase + '.chk', LibSignFile(os.path.join(args.destination, libname))) # Include pdb files for llvm-symbolizer to resolve symbols. if buildconfig.substs.get('LLVM_SYMBOLIZER') and mozinfo.isWin: for p, f in copier: if isinstance(f, ExecutableFile): pdbname = os.path.splitext(f.inputs()[0])[0] + '.pdb' if os.path.exists(pdbname): copier.add(os.path.basename(pdbname), File(pdbname)) # Setup preloading if args.jarlog and os.path.exists(args.jarlog): from mozpack.mozjar import JarLog log = JarLog(args.jarlog) for p, f in copier: if not isinstance(f, Jarrer): continue key = JarLog.canonicalize(os.path.join(args.destination, p)) if key in log: f.preload(log[key]) copier.copy(args.destination) generate_precomplete( os.path.normpath(os.path.join(args.destination, respath)))
def _process_android_eclipse_project_data(self, data, srcdir, objdir): # This can't be relative to the environment's topsrcdir, # because during testing topsrcdir is faked. template_directory = os.path.abspath( mozpath.join(os.path.dirname(__file__), 'templates', 'android_eclipse')) project_directory = mozpath.join(self.environment.topobjdir, 'android_eclipse', data.name) manifest_path = mozpath.join(self.environment.topobjdir, 'android_eclipse', '%s.manifest' % data.name) manifest = self._manifest_for_project(srcdir, data) ensureParentDir(manifest_path) manifest.write(path=manifest_path) classpathentries = [] for cpe in sorted(data._classpathentries, key=lambda x: x.path): e = self._Element_for_classpathentry(cpe) classpathentries.append(ET.tostring(e)) for name in sorted(data.referenced_projects): e = self._Element_for_referenced_project(name) classpathentries.append(ET.tostring(e)) for name in sorted(data.extra_jars): e = self._Element_for_extra_jar(mozpath.join(srcdir, name)) classpathentries.append(ET.tostring(e)) defines = {} defines['IDE_OBJDIR'] = objdir defines['IDE_TOPOBJDIR'] = self.environment.topobjdir defines['IDE_SRCDIR'] = srcdir defines['IDE_TOPSRCDIR'] = self.environment.topsrcdir defines['IDE_PROJECT_NAME'] = data.name defines['IDE_PACKAGE_NAME'] = data.package_name defines['IDE_PROJECT_DIRECTORY'] = project_directory defines['IDE_RELSRCDIR'] = mozpath.relpath(srcdir, self.environment.topsrcdir) defines['IDE_CLASSPATH_ENTRIES'] = '\n'.join( '\t' + cpe for cpe in classpathentries) defines['IDE_RECURSIVE_MAKE_TARGETS'] = ' '.join( sorted(data.recursive_make_targets)) # Like android.library=true defines[ 'IDE_PROJECT_LIBRARY_SETTING'] = 'android.library=true' if data.is_library else '' # Like android.library.reference.1=FennecBrandingResources defines['IDE_PROJECT_LIBRARY_REFERENCES'] = '\n'.join( 'android.library.reference.%s=%s' % (i + 1, ref) for i, ref in enumerate(sorted(data.included_projects))) if data.filtered_resources: filteredResources = self._Element_for_filtered_resources( data.filtered_resources) defines['IDE_PROJECT_FILTERED_RESOURCES'] = pretty_print( filteredResources).strip() else: defines['IDE_PROJECT_FILTERED_RESOURCES'] = '' copier = FileCopier() finder = FileFinder(template_directory) for input_filename, f in itertools.chain(finder.find('**'), finder.find('.**')): if input_filename == 'AndroidManifest.xml' and not data.is_library: # Main projects supply their own manifests. continue copier.add( input_filename, PreprocessedFile( mozpath.join(finder.base, input_filename), depfile_path=None, marker='#', defines=defines, extra_depends={mozpath.join(finder.base, input_filename)})) # When we re-create the build backend, we kill everything that was there. if os.path.isdir(project_directory): self.summary.updated_count += 1 else: self.summary.created_count += 1 copier.copy(project_directory, skip_if_older=False, remove_unaccounted=True)
def main(): parser = ArgumentParser() parser.add_argument('-D', dest='defines', action='append', metavar="VAR[=VAL]", help='Define a variable') parser.add_argument('--format', default='omni', help='Choose the chrome format for packaging ' + '(omni, jar or flat ; default: %(default)s)') parser.add_argument('--removals', default=None, help='removed-files source file') parser.add_argument('--ignore-errors', action='store_true', default=False, help='Transform errors into warnings.') parser.add_argument('--minify', action='store_true', default=False, help='Make some files more compact while packaging') parser.add_argument('--jarlog', default='', help='File containing jar ' + 'access logs') parser.add_argument('--optimizejars', action='store_true', default=False, help='Enable jar optimizations') parser.add_argument('--unify', default='', help='Base directory of another build to unify with') parser.add_argument('manifest', default=None, nargs='?', help='Manifest file name') parser.add_argument('source', help='Source directory') parser.add_argument('destination', help='Destination directory') parser.add_argument('--non-resource', nargs='+', metavar='PATTERN', default=[], help='Extra files not to be considered as resources') args = parser.parse_args() defines = dict(buildconfig.defines) if args.ignore_errors: errors.ignore_errors() if args.defines: for name, value in [split_define(d) for d in args.defines]: defines[name] = value copier = FileCopier() if args.format == 'flat': formatter = FlatFormatter(copier) elif args.format == 'jar': formatter = JarFormatter(copier, optimize=args.optimizejars) elif args.format == 'omni': formatter = OmniJarFormatter(copier, buildconfig.substs['OMNIJAR_NAME'], optimize=args.optimizejars, non_resources=args.non_resource) else: errors.fatal('Unknown format: %s' % args.format) # Adjust defines according to the requested format. if isinstance(formatter, OmniJarFormatter): defines['MOZ_OMNIJAR'] = 1 elif 'MOZ_OMNIJAR' in defines: del defines['MOZ_OMNIJAR'] binpath = '' if 'BINPATH' in defines: binpath = SimpleManifestSink.normalize_path(defines['BINPATH']) while binpath.startswith('/'): binpath = binpath[1:] if args.unify: def is_native(path): path = os.path.abspath(path) return platform.machine() in mozpack.path.split(path) # Invert args.unify and args.source if args.unify points to the # native architecture. args.source, args.unify = sorted([args.source, args.unify], key=is_native, reverse=True) if is_native(args.source): launcher.tooldir = args.source elif not buildconfig.substs['CROSS_COMPILE']: launcher.tooldir = buildconfig.substs['LIBXUL_DIST'] with errors.accumulate(): if args.unify: finder = UnifiedBuildFinder(FileFinder(args.source), FileFinder(args.unify), minify=args.minify) else: finder = FileFinder(args.source, minify=args.minify) if 'NO_PKG_FILES' in os.environ: sinkformatter = NoPkgFilesRemover(formatter, args.manifest is not None) else: sinkformatter = formatter sink = SimpleManifestSink(finder, sinkformatter) if args.manifest: preprocess_manifest(sink, args.manifest, defines) else: sink.add(Component(''), 'bin/*') sink.close(args.manifest is not None) if args.removals: lines = [l.lstrip() for l in open(args.removals).readlines()] removals_in = StringIO(''.join(lines)) removals_in.name = args.removals removals = RemovedFiles(copier) preprocess(removals_in, removals, defines) copier.add(mozpack.path.join(binpath, 'removed-files'), removals) # shlibsign libraries if launcher.can_launch(): for lib in SIGN_LIBS: libbase = mozpack.path.join(binpath, '%s%s') \ % (buildconfig.substs['DLL_PREFIX'], lib) libname = '%s%s' % (libbase, buildconfig.substs['DLL_SUFFIX']) if copier.contains(libname): copier.add(libbase + '.chk', LibSignFile(os.path.join(args.destination, libname))) # Setup preloading if args.jarlog and os.path.exists(args.jarlog): from mozpack.mozjar import JarLog log = JarLog(args.jarlog) for p, f in copier: if not isinstance(f, Jarrer): continue key = JarLog.canonicalize(os.path.join(args.destination, p)) if key in log: f.preload(log[key]) # Fill startup cache on Windows and Linux only # (this currently causes build failure on BSD, so skip on that platfom) if sys.platform == 'win32' or sys.platform.startswith ('linux'): if isinstance(formatter, OmniJarFormatter) and launcher.can_launch(): if buildconfig.substs['LIBXUL_SDK']: gre_path = mozpack.path.join(buildconfig.substs['LIBXUL_DIST'], 'bin') else: gre_path = None for base in sorted([[p for p in [mozpack.path.join('bin', b), b] if os.path.exists(os.path.join(args.source, p))][0] for b in sink.packager.get_bases()]): if not gre_path: gre_path = base base_path = sink.normalize_path(base) if base_path in formatter.omnijars: precompile_cache(formatter.omnijars[base_path], args.source, gre_path, base) copier.copy(args.destination) generate_precomplete(os.path.normpath(os.path.join(args.destination, binpath)))
def main(): parser = ArgumentParser() parser.add_argument('-D', dest='defines', action='append', metavar="VAR[=VAL]", help='Define a variable') parser.add_argument('--format', default='omni', help='Choose the chrome format for packaging ' + '(omni, jar or flat ; default: %(default)s)') parser.add_argument('--removals', default=None, help='removed-files source file') parser.add_argument('--ignore-errors', action='store_true', default=False, help='Transform errors into warnings.') parser.add_argument('--minify', action='store_true', default=False, help='Make some files more compact while packaging') parser.add_argument('--minify-js', action='store_true', help='Minify JavaScript files while packaging.') parser.add_argument('--js-binary', help='Path to js binary. This is used to verify ' 'minified JavaScript. If this is not defined, ' 'minification verification will not be performed.') parser.add_argument('--jarlog', default='', help='File containing jar ' + 'access logs') parser.add_argument('--optimizejars', action='store_true', default=False, help='Enable jar optimizations') parser.add_argument('--disable-compression', action='store_false', dest='compress', default=True, help='Disable jar compression') parser.add_argument('manifest', default=None, nargs='?', help='Manifest file name') parser.add_argument('source', help='Source directory') parser.add_argument('destination', help='Destination directory') parser.add_argument('--non-resource', nargs='+', metavar='PATTERN', default=[], help='Extra files not to be considered as resources') args = parser.parse_args() defines = dict(buildconfig.defines) if args.ignore_errors: errors.ignore_errors() if args.defines: for name, value in [split_define(d) for d in args.defines]: defines[name] = value copier = FileCopier() if args.format == 'flat': formatter = FlatFormatter(copier) elif args.format == 'jar': formatter = JarFormatter(copier, compress=args.compress, optimize=args.optimizejars) elif args.format == 'omni': formatter = OmniJarFormatter(copier, buildconfig.substs['OMNIJAR_NAME'], compress=args.compress, optimize=args.optimizejars, non_resources=args.non_resource) else: errors.fatal('Unknown format: %s' % args.format) # Adjust defines according to the requested format. if isinstance(formatter, OmniJarFormatter): defines['MOZ_OMNIJAR'] = 1 elif 'MOZ_OMNIJAR' in defines: del defines['MOZ_OMNIJAR'] respath = '' if 'RESPATH' in defines: respath = SimpleManifestSink.normalize_path(defines['RESPATH']) while respath.startswith('/'): respath = respath[1:] if not buildconfig.substs['CROSS_COMPILE']: launcher.tooldir = mozpath.join(buildconfig.topobjdir, 'dist') with errors.accumulate(): finder_args = dict( minify=args.minify, minify_js=args.minify_js, ) if args.js_binary: finder_args['minify_js_verify_command'] = [ args.js_binary, os.path.join(os.path.abspath(os.path.dirname(__file__)), 'js-compare-ast.js') ] finder = FileFinder(args.source, find_executables=True, **finder_args) if 'NO_PKG_FILES' in os.environ: sinkformatter = NoPkgFilesRemover(formatter, args.manifest is not None) else: sinkformatter = formatter sink = SimpleManifestSink(finder, sinkformatter) if args.manifest: preprocess_manifest(sink, args.manifest, defines) else: sink.add(Component(''), 'bin/*') sink.close(args.manifest is not None) if args.removals: removals_in = StringIO(open(args.removals).read()) removals_in.name = args.removals removals = RemovedFiles(copier) preprocess(removals_in, removals, defines) copier.add(mozpath.join(respath, 'removed-files'), removals) # shlibsign libraries if launcher.can_launch(): if not mozinfo.isMac and buildconfig.substs.get('COMPILE_ENVIRONMENT'): for lib in SIGN_LIBS: libbase = mozpath.join(respath, '%s%s') \ % (buildconfig.substs['DLL_PREFIX'], lib) libname = '%s%s' % (libbase, buildconfig.substs['DLL_SUFFIX']) if copier.contains(libname): copier.add(libbase + '.chk', LibSignFile(os.path.join(args.destination, libname))) # Setup preloading if args.jarlog and os.path.exists(args.jarlog): from mozpack.mozjar import JarLog log = JarLog(args.jarlog) for p, f in copier: if not isinstance(f, Jarrer): continue key = JarLog.canonicalize(os.path.join(args.destination, p)) if key in log: f.preload(log[key]) copier.copy(args.destination) generate_precomplete(os.path.normpath(os.path.join(args.destination, respath)))
def _process_android_eclipse_project_data(self, data, srcdir, objdir): # This can't be relative to the environment's topsrcdir, # because during testing topsrcdir is faked. template_directory = os.path.abspath(mozpath.join(os.path.dirname(__file__), 'templates', 'android_eclipse')) project_directory = mozpath.join(self.environment.topobjdir, 'android_eclipse', data.name) manifest_path = mozpath.join(self.environment.topobjdir, 'android_eclipse', '%s.manifest' % data.name) manifest = self._manifest_for_project(srcdir, data) ensureParentDir(manifest_path) manifest.write(path=manifest_path) classpathentries = [] for cpe in sorted(data._classpathentries, key=lambda x: x.path): e = self._Element_for_classpathentry(cpe) classpathentries.append(ET.tostring(e)) for name in sorted(data.referenced_projects): e = self._Element_for_referenced_project(name) classpathentries.append(ET.tostring(e)) for name in sorted(data.extra_jars): e = self._Element_for_extra_jar(mozpath.join(srcdir, name)) classpathentries.append(ET.tostring(e)) defines = {} defines['IDE_OBJDIR'] = objdir defines['IDE_TOPOBJDIR'] = self.environment.topobjdir defines['IDE_SRCDIR'] = srcdir defines['IDE_TOPSRCDIR'] = self.environment.topsrcdir defines['IDE_PROJECT_NAME'] = data.name defines['IDE_PACKAGE_NAME'] = data.package_name defines['IDE_PROJECT_DIRECTORY'] = project_directory defines['IDE_RELSRCDIR'] = mozpath.relpath(srcdir, self.environment.topsrcdir) defines['IDE_CLASSPATH_ENTRIES'] = '\n'.join('\t' + cpe for cpe in classpathentries) defines['IDE_RECURSIVE_MAKE_TARGETS'] = ' '.join(sorted(data.recursive_make_targets)) # Like android.library=true defines['IDE_PROJECT_LIBRARY_SETTING'] = 'android.library=true' if data.is_library else '' # Like android.library.reference.1=FennecBrandingResources defines['IDE_PROJECT_LIBRARY_REFERENCES'] = '\n'.join( 'android.library.reference.%s=%s' % (i + 1, ref) for i, ref in enumerate(sorted(data.included_projects))) if data.filtered_resources: filteredResources = self._Element_for_filtered_resources(data.filtered_resources) defines['IDE_PROJECT_FILTERED_RESOURCES'] = pretty_print(filteredResources).strip() else: defines['IDE_PROJECT_FILTERED_RESOURCES'] = '' copier = FileCopier() finder = FileFinder(template_directory) for input_filename, f in itertools.chain(finder.find('**'), finder.find('.**')): if input_filename == 'AndroidManifest.xml' and not data.is_library: # Main projects supply their own manifests. continue copier.add(input_filename, PreprocessedFile( mozpath.join(finder.base, input_filename), depfile_path=None, marker='#', defines=defines, extra_depends={mozpath.join(finder.base, input_filename)})) # When we re-create the build backend, we kill everything that was there. if os.path.isdir(project_directory): self.summary.updated_count += 1 else: self.summary.created_count += 1 copier.copy(project_directory, skip_if_older=False, remove_unaccounted=True)
def test_file_copier(self): copier = FileCopier() copier.add('foo/bar', GeneratedFile('foobar')) copier.add('foo/qux', GeneratedFile('fooqux')) copier.add('foo/deep/nested/directory/file', GeneratedFile('fooz')) copier.add('bar', GeneratedFile('bar')) copier.add('qux/foo', GeneratedFile('quxfoo')) copier.add('qux/bar', GeneratedFile('')) result = copier.copy(self.tmpdir) self.assertEqual(self.all_files(self.tmpdir), set(copier.paths())) self.assertEqual(self.all_dirs(self.tmpdir), set(['foo/deep/nested/directory', 'qux'])) self.assertEqual(result.updated_files, set(self.tmppath(p) for p in self.all_files(self.tmpdir))) self.assertEqual(result.existing_files, set()) self.assertEqual(result.removed_files, set()) self.assertEqual(result.removed_directories, set()) copier.remove('foo') copier.add('test', GeneratedFile('test')) result = copier.copy(self.tmpdir) self.assertEqual(self.all_files(self.tmpdir), set(copier.paths())) self.assertEqual(self.all_dirs(self.tmpdir), set(['qux'])) self.assertEqual(result.removed_files, set(self.tmppath(p) for p in ('foo/bar', 'foo/qux', 'foo/deep/nested/directory/file')))
def test_remove_unaccounted_directory_symlinks(self): """Directory symlinks in destination that are not in the way are deleted according to remove_unaccounted and remove_all_directory_symlinks. """ if not self.symlink_supported: return dest = self.tmppath('dest') copier = FileCopier() copier.add('foo/bar/baz', GeneratedFile('foobarbaz')) os.makedirs(self.tmppath('dest/foo')) dummy = self.tmppath('dummy') os.mkdir(dummy) os.mkdir(self.tmppath('dest/zot')) link = self.tmppath('dest/zot/zap') os.symlink(dummy, link) # If not remove_unaccounted but remove_empty_directories, then # the symlinked directory remains (as does its containing # directory). result = copier.copy(dest, remove_unaccounted=False, remove_empty_directories=True, remove_all_directory_symlinks=False) st = os.lstat(link) self.assertTrue(stat.S_ISLNK(st.st_mode)) self.assertFalse(stat.S_ISDIR(st.st_mode)) self.assertEqual(self.all_files(dest), set(copier.paths())) self.assertEqual(self.all_dirs(dest), set(['foo/bar'])) self.assertEqual(result.removed_directories, set()) self.assertEqual(len(result.updated_files), 1) # If remove_unaccounted but not remove_empty_directories, then # only the symlinked directory is removed. result = copier.copy(dest, remove_unaccounted=True, remove_empty_directories=False, remove_all_directory_symlinks=False) st = os.lstat(self.tmppath('dest/zot')) self.assertFalse(stat.S_ISLNK(st.st_mode)) self.assertTrue(stat.S_ISDIR(st.st_mode)) self.assertEqual(result.removed_files, set([link])) self.assertEqual(result.removed_directories, set()) self.assertEqual(self.all_files(dest), set(copier.paths())) self.assertEqual(self.all_dirs(dest), set(['foo/bar', 'zot'])) # If remove_unaccounted and remove_empty_directories, then # both the symlink and its containing directory are removed. link = self.tmppath('dest/zot/zap') os.symlink(dummy, link) result = copier.copy(dest, remove_unaccounted=True, remove_empty_directories=True, remove_all_directory_symlinks=False) self.assertEqual(result.removed_files, set([link])) self.assertEqual(result.removed_directories, set([self.tmppath('dest/zot')])) self.assertEqual(self.all_files(dest), set(copier.paths())) self.assertEqual(self.all_dirs(dest), set(['foo/bar']))
def main(): parser = ArgumentParser() parser.add_argument('-D', dest='defines', action='append', metavar="VAR[=VAL]", help='Define a variable') parser.add_argument('--format', default='omni', help='Choose the chrome format for packaging ' + '(omni, jar or flat ; default: %(default)s)') parser.add_argument('--removals', default=None, help='removed-files source file') parser.add_argument('--ignore-errors', action='store_true', default=False, help='Transform errors into warnings.') parser.add_argument('--minify', action='store_true', default=False, help='Make some files more compact while packaging') parser.add_argument('--minify-js', action='store_true', help='Minify JavaScript files while packaging.') parser.add_argument('--js-binary', help='Path to js binary. This is used to verify ' 'minified JavaScript. If this is not defined, ' 'minification verification will not be performed.') parser.add_argument('--jarlog', default='', help='File containing jar ' + 'access logs') parser.add_argument('--optimizejars', action='store_true', default=False, help='Enable jar optimizations') parser.add_argument('--unify', default='', help='Base directory of another build to unify with') parser.add_argument('manifest', default=None, nargs='?', help='Manifest file name') parser.add_argument('source', help='Source directory') parser.add_argument('destination', help='Destination directory') parser.add_argument('--non-resource', nargs='+', metavar='PATTERN', default=[], help='Extra files not to be considered as resources') args = parser.parse_args() defines = dict(buildconfig.defines) if args.ignore_errors: errors.ignore_errors() if args.defines: for name, value in [split_define(d) for d in args.defines]: defines[name] = value copier = FileCopier() if args.format == 'flat': formatter = FlatFormatter(copier) elif args.format == 'jar': formatter = JarFormatter(copier, optimize=args.optimizejars) elif args.format == 'omni': formatter = OmniJarFormatter(copier, buildconfig.substs['OMNIJAR_NAME'], optimize=args.optimizejars, non_resources=args.non_resource) else: errors.fatal('Unknown format: %s' % args.format) # Adjust defines according to the requested format. if isinstance(formatter, OmniJarFormatter): defines['MOZ_OMNIJAR'] = 1 elif 'MOZ_OMNIJAR' in defines: del defines['MOZ_OMNIJAR'] respath = '' if 'RESPATH' in defines: respath = SimpleManifestSink.normalize_path(defines['RESPATH']) while respath.startswith('/'): respath = respath[1:] if args.unify: def is_native(path): path = os.path.abspath(path) return platform.machine() in mozpath.split(path) # Invert args.unify and args.source if args.unify points to the # native architecture. args.source, args.unify = sorted([args.source, args.unify], key=is_native, reverse=True) if is_native(args.source): launcher.tooldir = args.source elif not buildconfig.substs['CROSS_COMPILE']: launcher.tooldir = buildconfig.substs['LIBXUL_DIST'] with errors.accumulate(): finder_args = dict( minify=args.minify, minify_js=args.minify_js, ) if args.js_binary: finder_args['minify_js_verify_command'] = [ args.js_binary, os.path.join(os.path.abspath(os.path.dirname(__file__)), 'js-compare-ast.js') ] if args.unify: finder = UnifiedBuildFinder(FileFinder(args.source), FileFinder(args.unify), **finder_args) else: finder = FileFinder(args.source, **finder_args) if 'NO_PKG_FILES' in os.environ: sinkformatter = NoPkgFilesRemover(formatter, args.manifest is not None) else: sinkformatter = formatter sink = SimpleManifestSink(finder, sinkformatter) if args.manifest: preprocess_manifest(sink, args.manifest, defines) else: sink.add(Component(''), 'bin/*') sink.close(args.manifest is not None) if args.removals: removals_in = StringIO(open(args.removals).read()) removals_in.name = args.removals removals = RemovedFiles(copier) preprocess(removals_in, removals, defines) copier.add(mozpath.join(respath, 'removed-files'), removals) # shlibsign libraries if launcher.can_launch(): if not mozinfo.isMac: for lib in SIGN_LIBS: libbase = mozpath.join(respath, '%s%s') \ % (buildconfig.substs['DLL_PREFIX'], lib) libname = '%s%s' % (libbase, buildconfig.substs['DLL_SUFFIX']) if copier.contains(libname): copier.add(libbase + '.chk', LibSignFile(os.path.join(args.destination, libname))) # Setup preloading if args.jarlog and os.path.exists(args.jarlog): from mozpack.mozjar import JarLog log = JarLog(args.jarlog) for p, f in copier: if not isinstance(f, Jarrer): continue key = JarLog.canonicalize(os.path.join(args.destination, p)) if key in log: f.preload(log[key]) # Fill startup cache if isinstance(formatter, OmniJarFormatter) and launcher.can_launch() \ and buildconfig.substs['MOZ_DISABLE_STARTUPCACHE'] != '1': gre_path = None def get_bases(): for b in sink.packager.get_bases(addons=False): for p in (mozpath.join('bin', b), b): if os.path.exists(os.path.join(args.source, p)): yield p break for base in sorted(get_bases()): if not gre_path: gre_path = base base_path = sink.normalize_path(base) if base_path in formatter.omnijars: precompile_cache(formatter.omnijars[base_path], args.source, gre_path, base) copier.copy(args.destination)