Example #1
0
def unpack(source):
    '''
    Transform a jar chrome or omnijar packaged directory into a flat package.
    '''
    copier = FileCopier()
    unpack_to_registry(source, copier)
    copier.copy(source, skip_if_older=False)
Example #2
0
    def test_optional_exists_creates_unneeded_directory(self):
        """Demonstrate that a directory not strictly required, but specified
        as the path to an optional file, will be unnecessarily created.

        This behaviour is wrong; fixing it is tracked by Bug 972432;
        and this test exists to guard against unexpected changes in
        behaviour.
        """

        dest = self.tmppath('dest')

        copier = FileCopier()
        copier.add('foo/bar', ExistingFile(required=False))

        result = copier.copy(dest)

        st = os.lstat(self.tmppath('dest/foo'))
        self.assertFalse(stat.S_ISLNK(st.st_mode))
        self.assertTrue(stat.S_ISDIR(st.st_mode))

        # What's worse, we have no record that dest was created.
        self.assertEquals(len(result.updated_files), 0)

        # But we do have an erroneous record of an optional file
        # existing when it does not.
        self.assertIn(self.tmppath('dest/foo/bar'), result.existing_files)
Example #3
0
    def test_symlink_directory_replaced(self):
        """Directory symlinks in destination are replaced if they need to be
        real directories."""
        if not self.symlink_supported:
            return

        dest = self.tmppath('dest')

        copier = FileCopier()
        copier.add('foo/bar/baz', GeneratedFile('foobarbaz'))

        os.makedirs(self.tmppath('dest/foo'))
        dummy = self.tmppath('dummy')
        os.mkdir(dummy)
        link = self.tmppath('dest/foo/bar')
        os.symlink(dummy, link)

        result = copier.copy(dest)

        st = os.lstat(link)
        self.assertFalse(stat.S_ISLNK(st.st_mode))
        self.assertTrue(stat.S_ISDIR(st.st_mode))

        self.assertEqual(self.all_files(dest), set(copier.paths()))

        self.assertEqual(result.removed_directories, set())
        self.assertEqual(len(result.updated_files), 1)
Example #4
0
    def output_changes(self, verbose=True):
        '''
        Return an iterator of `FasterBuildChange` instances as outputs
        from the faster build system are updated.
        '''
        for change in self.input_changes(verbose=verbose):
            now = datetime.datetime.utcnow()

            for unrecognized in sorted(change.unrecognized):
                print_line('watch', '! {}'.format(unrecognized), now=now)

            all_outputs = set()
            for input in sorted(change.input_to_outputs):
                outputs = change.input_to_outputs[input]

                print_line('watch', '< {}'.format(input), now=now)
                for output in sorted(outputs):
                    print_line('watch', '> {}'.format(output), now=now)
                all_outputs |= outputs

            if all_outputs:
                partial_copier = FileCopier()
                for output in all_outputs:
                    partial_copier.add(output, self.file_copier[output])

                self.incremental_copy(partial_copier, force=True, verbose=verbose)
                yield change
Example #5
0
    def _synchronize_docs(self):
        m = InstallManifest()

        m.add_symlink(self._conf_py_path, "conf.py")

        for dest, source in sorted(self._trees.items()):
            source_dir = os.path.join(self._topsrcdir, source)
            for root, dirs, files in os.walk(source_dir):
                for f in files:
                    source_path = os.path.join(root, f)
                    rel_source = source_path[len(source_dir) + 1 :]

                    m.add_symlink(source_path, os.path.join(dest, rel_source))

        copier = FileCopier()
        m.populate_registry(copier)
        copier.copy(self._docs_dir)

        with open(self._index_path, "rb") as fh:
            data = fh.read()

        indexes = ["%s/index" % p for p in sorted(self._trees.keys())]
        indexes = "\n   ".join(indexes)

        packages = [os.path.basename(p) for p in self._python_package_dirs]
        packages = ["python/%s" % p for p in packages]
        packages = "\n   ".join(sorted(packages))
        data = data.format(indexes=indexes, python_packages=packages)

        with open(os.path.join(self._docs_dir, "index.rst"), "wb") as fh:
            fh.write(data)
Example #6
0
    def _synchronize_docs(self):
        m = InstallManifest()

        m.add_symlink(self._conf_py_path, 'conf.py')

        for dest, source in sorted(self._trees.items()):
            source_dir = os.path.join(self._topsrcdir, source)
            for root, dirs, files in os.walk(source_dir):
                for f in files:
                    source_path = os.path.join(root, f)
                    rel_source = source_path[len(source_dir) + 1:]

                    m.add_symlink(source_path, os.path.join(dest, rel_source))

        stage_dir = os.path.join(self._output_dir, 'staging')
        copier = FileCopier()
        m.populate_registry(copier)
        copier.copy(stage_dir)

        with open(self._index_path, 'rb') as fh:
            data = fh.read()

        indexes = ['%s/index' % p for p in sorted(self._trees.keys())]
        indexes = '\n   '.join(indexes)

        packages = [os.path.basename(p) for p in self._python_package_dirs]
        packages = ['python/%s' % p for p in packages]
        packages = '\n   '.join(sorted(packages))
        data = data.format(indexes=indexes, python_packages=packages)

        with open(os.path.join(stage_dir, 'index.rst'), 'wb') as fh:
            fh.write(data)
Example #7
0
def strip(dir):
    copier = FileCopier()
    # The FileFinder will give use ExecutableFile instances for files
    # that can be stripped, and copying ExecutableFiles defaults to
    # stripping them unless buildconfig.substs['PKG_SKIP_STRIP'] is set.
    for p, f in FileFinder(dir):
        copier.add(p, f)
    copier.copy(dir)
def process_manifest(destdir, paths, remove_unaccounted=True):
    manifest = InstallManifest()
    for path in paths:
        manifest |= InstallManifest(path=path)

    copier = FileCopier()
    manifest.populate_registry(copier)
    return copier.copy(destdir, remove_unaccounted=remove_unaccounted)
def process_manifest(destdir, *paths):
    manifest = InstallManifest()
    for path in paths:
        manifest |= InstallManifest(path=path)

    copier = FileCopier()
    manifest.populate_registry(copier)
    return copier.copy(destdir)
Example #10
0
def install_test_files(topsrcdir, topobjdir, tests_root, test_objs):
    """Installs the requested test files to the objdir. This is invoked by
    test runners to avoid installing tens of thousands of test files when
    only a few tests need to be run.
    """
    flavor_info = {flavor: (root, prefix, install)
                   for (flavor, root, prefix, install) in TEST_MANIFESTS.values()}
    objdir_dest = mozpath.join(topobjdir, tests_root)

    converter = SupportFilesConverter()
    install_info = TestInstallInfo()
    for o in test_objs:
        flavor = o['flavor']
        if flavor not in flavor_info:
            # This is a test flavor that isn't installed by the build system.
            continue
        root, prefix, install = flavor_info[flavor]
        if not install:
            # This flavor isn't installed to the objdir.
            continue

        manifest_path = o['manifest']
        manifest_dir = mozpath.dirname(manifest_path)

        out_dir = mozpath.join(root, prefix, manifest_dir[len(topsrcdir) + 1:])
        file_relpath = o['file_relpath']
        source = mozpath.join(topsrcdir, file_relpath)
        dest = mozpath.join(root, prefix, file_relpath)
        if 'install-to-subdir' in o:
            out_dir = mozpath.join(out_dir, o['install-to-subdir'])
            manifest_relpath = mozpath.relpath(source, mozpath.dirname(manifest_path))
            dest = mozpath.join(out_dir, manifest_relpath)

        install_info.installs.append((source, dest))
        install_info |= converter.convert_support_files(o, root,
                                                        manifest_dir,
                                                        out_dir)

    manifest = InstallManifest()

    for source, dest in set(install_info.installs):
        if dest in install_info.external_installs:
            continue
        manifest.add_symlink(source, dest)
    for base, pattern, dest in install_info.pattern_installs:
        manifest.add_pattern_symlink(base, pattern, dest)

    _resolve_installs(install_info.deferred_installs, topobjdir, manifest)

    # Harness files are treated as a monolith and installed each time we run tests.
    # Fortunately there are not very many.
    manifest |= InstallManifest(mozpath.join(topobjdir,
                                             '_build_manifests',
                                             'install', tests_root))
    copier = FileCopier()
    manifest.populate_registry(copier)
    copier.copy(objdir_dest,
                remove_unaccounted=False)
Example #11
0
def unpack(source):
    '''
    Transform a jar chrome or omnijar packaged directory into a flat package.
    '''
    copier = FileCopier()
    finder = UnpackFinder(source)
    packager = SimplePackager(FlatFormatter(copier))
    for p, f in finder.find('*'):
        if mozpack.path.split(p)[0] not in STARTUP_CACHE_PATHS:
            packager.add(p, f)
    packager.close()
    copier.copy(source, skip_if_older=False)
Example #12
0
    def test_copier_application(self):
        dest = self.tmppath('dest')
        os.mkdir(dest)

        to_delete = self.tmppath('dest/to_delete')
        with open(to_delete, 'a'):
            pass

        with open(self.tmppath('s_source'), 'wt') as fh:
            fh.write('symlink!')

        with open(self.tmppath('c_source'), 'wt') as fh:
            fh.write('copy!')

        with open(self.tmppath('p_source'), 'wt') as fh:
            fh.write('#define FOO 1\npreprocess!')

        with open(self.tmppath('dest/e_dest'), 'a'):
            pass

        with open(self.tmppath('dest/o_dest'), 'a'):
            pass

        m = self._get_test_manifest()
        c = FileCopier()
        m.populate_registry(c)
        result = c.copy(dest)

        self.assertTrue(os.path.exists(self.tmppath('dest/s_dest')))
        self.assertTrue(os.path.exists(self.tmppath('dest/c_dest')))
        self.assertTrue(os.path.exists(self.tmppath('dest/p_dest')))
        self.assertTrue(os.path.exists(self.tmppath('dest/e_dest')))
        self.assertTrue(os.path.exists(self.tmppath('dest/o_dest')))
        self.assertTrue(os.path.exists(self.tmppath('dest/content')))
        self.assertFalse(os.path.exists(to_delete))

        with open(self.tmppath('dest/s_dest'), 'rt') as fh:
            self.assertEqual(fh.read(), 'symlink!')

        with open(self.tmppath('dest/c_dest'), 'rt') as fh:
            self.assertEqual(fh.read(), 'copy!')

        with open(self.tmppath('dest/p_dest'), 'rt') as fh:
            self.assertEqual(fh.read(), 'preprocess!')

        self.assertEqual(result.updated_files, set(self.tmppath(p) for p in (
            'dest/s_dest', 'dest/c_dest', 'dest/p_dest', 'dest/content')))
        self.assertEqual(result.existing_files,
            set([self.tmppath('dest/e_dest'), self.tmppath('dest/o_dest')]))
        self.assertEqual(result.removed_files, {to_delete})
        self.assertEqual(result.removed_directories, set())
def process_manifest(destdir, paths,
        remove_unaccounted=True,
        remove_all_directory_symlinks=True,
        remove_empty_directories=True):
    manifest = InstallManifest()
    for path in paths:
        manifest |= InstallManifest(path=path)

    copier = FileCopier()
    manifest.populate_registry(copier)
    return copier.copy(destdir,
        remove_unaccounted=remove_unaccounted,
        remove_all_directory_symlinks=remove_all_directory_symlinks,
        remove_empty_directories=remove_empty_directories)
Example #14
0
    def test_copier_application(self):
        dest = self.tmppath("dest")
        os.mkdir(dest)

        to_delete = self.tmppath("dest/to_delete")
        with open(to_delete, "a"):
            pass

        with open(self.tmppath("s_source"), "wt") as fh:
            fh.write("symlink!")

        with open(self.tmppath("c_source"), "wt") as fh:
            fh.write("copy!")

        with open(self.tmppath("p_source"), "wt") as fh:
            fh.write("#define FOO 1\npreprocess!")

        with open(self.tmppath("dest/e_dest"), "a"):
            pass

        with open(self.tmppath("dest/o_dest"), "a"):
            pass

        m = self._get_test_manifest()
        c = FileCopier()
        m.populate_registry(c)
        result = c.copy(dest)

        self.assertTrue(os.path.exists(self.tmppath("dest/s_dest")))
        self.assertTrue(os.path.exists(self.tmppath("dest/c_dest")))
        self.assertTrue(os.path.exists(self.tmppath("dest/p_dest")))
        self.assertTrue(os.path.exists(self.tmppath("dest/e_dest")))
        self.assertTrue(os.path.exists(self.tmppath("dest/o_dest")))
        self.assertFalse(os.path.exists(to_delete))

        with open(self.tmppath("dest/s_dest"), "rt") as fh:
            self.assertEqual(fh.read(), "symlink!")

        with open(self.tmppath("dest/c_dest"), "rt") as fh:
            self.assertEqual(fh.read(), "copy!")

        with open(self.tmppath("dest/p_dest"), "rt") as fh:
            self.assertEqual(fh.read(), "preprocess!")

        self.assertEqual(
            result.updated_files, set(self.tmppath(p) for p in ("dest/s_dest", "dest/c_dest", "dest/p_dest"))
        )
        self.assertEqual(result.existing_files, set([self.tmppath("dest/e_dest"), self.tmppath("dest/o_dest")]))
        self.assertEqual(result.removed_files, {to_delete})
        self.assertEqual(result.removed_directories, set())
Example #15
0
    def test_no_remove(self):
        copier = FileCopier()
        copier.add('foo', GeneratedFile('foo'))

        with open(self.tmppath('bar'), 'a'):
            pass

        os.mkdir(self.tmppath('emptydir'))

        result = copier.copy(self.tmpdir, remove_unaccounted=False)

        self.assertEqual(self.all_files(self.tmpdir), set(['foo', 'bar']))
        self.assertEqual(result.removed_files, set())
        self.assertEqual(result.removed_directories,
            set([self.tmppath('emptydir')]))
def process_manifest(
    destdir,
    paths,
    track=None,
    remove_unaccounted=True,
    remove_all_directory_symlinks=True,
    remove_empty_directories=True,
    defines={},
):

    if track:
        if os.path.exists(track):
            # We use the same format as install manifests for the tracking
            # data.
            manifest = InstallManifest(path=track)
            remove_unaccounted = FileRegistry()
            dummy_file = BaseFile()

            finder = FileFinder(destdir, find_executables=False, find_dotfiles=True)
            for dest in manifest._dests:
                for p, f in finder.find(dest):
                    remove_unaccounted.add(p, dummy_file)

        else:
            # If tracking is enabled and there is no file, we don't want to
            # be removing anything.
            remove_unaccounted = False
            remove_empty_directories = False
            remove_all_directory_symlinks = False

    manifest = InstallManifest()
    for path in paths:
        manifest |= InstallManifest(path=path)

    copier = FileCopier()
    manifest.populate_registry(copier, defines_override=defines)
    result = copier.copy(
        destdir,
        remove_unaccounted=remove_unaccounted,
        remove_all_directory_symlinks=remove_all_directory_symlinks,
        remove_empty_directories=remove_empty_directories,
    )

    if track:
        manifest.write(path=track)

    return result
Example #17
0
def repack(source, l10n, extra_l10n={}, non_resources=[], non_chrome=set()):
    '''
    Replace localized data from the `source` directory with localized data
    from `l10n` and `extra_l10n`.

    The `source` argument points to a directory containing a packaged
    application (in omnijar, jar or flat form).
    The `l10n` argument points to a directory containing the main localized
    data (usually in the form of a language pack addon) to use to replace
    in the packaged application.
    The `extra_l10n` argument contains a dict associating relative paths in
    the source to separate directories containing localized data for them.
    This can be used to point at different language pack addons for different
    parts of the package application.
    The `non_resources` argument gives a list of relative paths in the source
    that should not be added in an omnijar in case the packaged application
    is in that format.
    The `non_chrome` argument gives a list of file/directory patterns for
    localized files that are not listed in a chrome.manifest.
    '''
    app_finder = UnpackFinder(source)
    l10n_finder = UnpackFinder(l10n)
    if extra_l10n:
        finders = {
            '': l10n_finder,
        }
        for base, path in extra_l10n.iteritems():
            finders[base] = UnpackFinder(path)
        l10n_finder = ComposedFinder(finders)
    copier = FileCopier()
    compress = min(app_finder.compressed, JAR_DEFLATED)
    if app_finder.kind == 'flat':
        formatter = FlatFormatter(copier)
    elif app_finder.kind == 'jar':
        formatter = JarFormatter(copier,
                                 optimize=app_finder.optimizedjars,
                                 compress=compress)
    elif app_finder.kind == 'omni':
        formatter = OmniJarFormatter(copier, app_finder.omnijar,
                                     optimize=app_finder.optimizedjars,
                                     compress=compress,
                                     non_resources=non_resources)

    with errors.accumulate():
        _repack(app_finder, l10n_finder, copier, formatter, non_chrome)
    copier.copy(source, skip_if_older=False)
    generate_precomplete(source)
Example #18
0
def repack(source, l10n, non_resources=[], non_chrome=set()):
    app_finder = UnpackFinder(source)
    l10n_finder = UnpackFinder(l10n)
    copier = FileCopier()
    if app_finder.kind == 'flat':
        formatter = FlatFormatter(copier)
    elif app_finder.kind == 'jar':
        formatter = JarFormatter(copier, optimize=app_finder.optimizedjars)
    elif app_finder.kind == 'omni':
        formatter = OmniJarFormatter(copier, app_finder.omnijar,
                                     optimize=app_finder.optimizedjars,
                                     non_resources=non_resources)

    with errors.accumulate():
        _repack(app_finder, l10n_finder, copier, formatter, non_chrome)
    copier.copy(source, skip_if_older=False)
    generate_precomplete(source)
Example #19
0
    def test_pattern_expansion(self):
        source = self.tmppath("source")
        os.mkdir(source)
        os.mkdir("%s/base" % source)
        os.mkdir("%s/base/foo" % source)

        with open("%s/base/foo/file1" % source, "a"):
            pass

        with open("%s/base/foo/file2" % source, "a"):
            pass

        m = InstallManifest()
        m.add_pattern_symlink("%s/base" % source, "**", "dest")

        c = FileCopier()
        m.populate_registry(c)
        self.assertEqual(c.paths(), ["dest/foo/file1", "dest/foo/file2"])
Example #20
0
    def test_pattern_expansion(self):
        source = self.tmppath('source')
        os.mkdir(source)
        os.mkdir('%s/base' % source)
        os.mkdir('%s/base/foo' % source)

        with open('%s/base/foo/file1' % source, 'a'):
            pass

        with open('%s/base/foo/file2' % source, 'a'):
            pass

        m = InstallManifest()
        m.add_pattern_symlink('%s/base' % source, '**', 'dest')

        c = FileCopier()
        m.populate_registry(c)
        self.assertEqual(c.paths(), ['dest/foo/file1', 'dest/foo/file2'])
Example #21
0
    def test_no_remove_empty_directories(self):
        copier = FileCopier()
        copier.add("foo", GeneratedFile("foo"))

        with open(self.tmppath("bar"), "a"):
            pass

        os.mkdir(self.tmppath("emptydir"))
        d = self.tmppath("populateddir")
        os.mkdir(d)

        with open(self.tmppath("populateddir/foo"), "a"):
            pass

        result = copier.copy(self.tmpdir, remove_unaccounted=False, remove_empty_directories=False)

        self.assertEqual(self.all_files(self.tmpdir), set(["foo", "bar", "populateddir/foo"]))
        self.assertEqual(self.all_dirs(self.tmpdir), set(["emptydir", "populateddir"]))
        self.assertEqual(result.removed_files, set())
        self.assertEqual(result.removed_directories, set())
Example #22
0
    def test_permissions(self):
        """Ensure files without write permission can be deleted."""
        with open(self.tmppath('dummy'), 'a'):
            pass

        p = self.tmppath('no_perms')
        with open(p, 'a'):
            pass

        # Make file and directory unwritable. Reminder: making a directory
        # unwritable prevents modifications (including deletes) from the list
        # of files in that directory.
        os.chmod(p, 0400)
        os.chmod(self.tmpdir, 0400)

        copier = FileCopier()
        copier.add('dummy', GeneratedFile('content'))
        result = copier.copy(self.tmpdir)
        self.assertEqual(result.removed_files_count, 1)
        self.assertFalse(os.path.exists(p))
Example #23
0
    def test_add_entries_from(self):
        source = self.tmppath('source')
        os.mkdir(source)
        os.mkdir('%s/base' % source)
        os.mkdir('%s/base/foo' % source)

        with open('%s/base/foo/file1' % source, 'a'):
            pass

        with open('%s/base/foo/file2' % source, 'a'):
            pass

        m = InstallManifest()
        m.add_pattern_link('%s/base' % source, '**', 'dest')

        p = InstallManifest()
        p.add_entries_from(m)
        self.assertEqual(len(p), 1)

        c = FileCopier()
        p.populate_registry(c)
        self.assertEqual(c.paths(), ['dest/foo/file1', 'dest/foo/file2'])

        q = InstallManifest()
        q.add_entries_from(m, base='target')
        self.assertEqual(len(q), 1)

        d = FileCopier()
        q.populate_registry(d)
        self.assertEqual(d.paths(), ['target/dest/foo/file1', 'target/dest/foo/file2'])

        # Some of the values in an InstallManifest include destination
        # information that is present in the keys.  Verify that we can
        # round-trip serialization.
        r = InstallManifest()
        r.add_entries_from(m)
        r.add_entries_from(m, base='target')
        self.assertEqual(len(r), 2)

        temp_path = self.tmppath('temp_path')
        r.write(path=temp_path)

        s = InstallManifest(path=temp_path)
        e = FileCopier()
        s.populate_registry(e)

        self.assertEqual(e.paths(),
                         ['dest/foo/file1', 'dest/foo/file2',
                          'target/dest/foo/file1', 'target/dest/foo/file2'])
Example #24
0
    def test_file_copier(self):
        copier = FileCopier()
        copier.add('foo/bar', GeneratedFile('foobar'))
        copier.add('foo/qux', GeneratedFile('fooqux'))
        copier.add('foo/deep/nested/directory/file', GeneratedFile('fooz'))
        copier.add('bar', GeneratedFile('bar'))
        copier.add('qux/foo', GeneratedFile('quxfoo'))
        copier.add('qux/bar', GeneratedFile(''))

        copier.copy(self.tmpdir)
        self.assertEqual(self.all_files(self.tmpdir), set(copier.paths()))
        self.assertEqual(self.all_dirs(self.tmpdir),
                         set(['foo/deep/nested/directory', 'qux']))

        copier.remove('foo')
        copier.add('test', GeneratedFile('test'))
        copier.copy(self.tmpdir)
        self.assertEqual(self.all_files(self.tmpdir), set(copier.paths()))
        self.assertEqual(self.all_dirs(self.tmpdir), set(['qux']))
def main():
    parser = ArgumentParser()
    parser.add_argument('-D',
                        dest='defines',
                        action='append',
                        metavar="VAR[=VAL]",
                        help='Define a variable')
    parser.add_argument('--format',
                        default='omni',
                        help='Choose the chrome format for packaging ' +
                        '(omni, jar or flat ; default: %(default)s)')
    parser.add_argument('--removals',
                        default=None,
                        help='removed-files source file')
    parser.add_argument('--ignore-errors',
                        action='store_true',
                        default=False,
                        help='Transform errors into warnings.')
    parser.add_argument('--minify',
                        action='store_true',
                        default=False,
                        help='Make some files more compact while packaging')
    parser.add_argument('--minify-js',
                        action='store_true',
                        help='Minify JavaScript files while packaging.')
    parser.add_argument('--js-binary',
                        help='Path to js binary. This is used to verify '
                        'minified JavaScript. If this is not defined, '
                        'minification verification will not be performed.')
    parser.add_argument('--jarlog',
                        default='',
                        help='File containing jar ' + 'access logs')
    parser.add_argument('--optimizejars',
                        action='store_true',
                        default=False,
                        help='Enable jar optimizations')
    parser.add_argument('--compress',
                        choices=('none', 'deflate', 'brotli'),
                        default='deflate',
                        help='Use given jar compression (default: deflate)')
    parser.add_argument('manifest',
                        default=None,
                        nargs='?',
                        help='Manifest file name')
    parser.add_argument('source', help='Source directory')
    parser.add_argument('destination', help='Destination directory')
    parser.add_argument('--non-resource',
                        nargs='+',
                        metavar='PATTERN',
                        default=[],
                        help='Extra files not to be considered as resources')
    args = parser.parse_args()

    defines = dict(buildconfig.defines['ALLDEFINES'])
    if args.ignore_errors:
        errors.ignore_errors()

    if args.defines:
        for name, value in [split_define(d) for d in args.defines]:
            defines[name] = value

    compress = {
        'none': False,
        'deflate': True,
        'brotli': JAR_BROTLI,
    }[args.compress]

    copier = FileCopier()
    if args.format == 'flat':
        formatter = FlatFormatter(copier)
    elif args.format == 'jar':
        formatter = JarFormatter(copier,
                                 compress=compress,
                                 optimize=args.optimizejars)
    elif args.format == 'omni':
        formatter = OmniJarFormatter(copier,
                                     buildconfig.substs['OMNIJAR_NAME'],
                                     compress=compress,
                                     optimize=args.optimizejars,
                                     non_resources=args.non_resource)
    else:
        errors.fatal('Unknown format: %s' % args.format)

    # Adjust defines according to the requested format.
    if isinstance(formatter, OmniJarFormatter):
        defines['MOZ_OMNIJAR'] = 1
    elif 'MOZ_OMNIJAR' in defines:
        del defines['MOZ_OMNIJAR']

    respath = ''
    if 'RESPATH' in defines:
        respath = SimpleManifestSink.normalize_path(defines['RESPATH'])
    while respath.startswith('/'):
        respath = respath[1:]

    if not buildconfig.substs['CROSS_COMPILE']:
        launcher.tooldir = mozpath.join(buildconfig.topobjdir, 'dist')

    with errors.accumulate():
        finder_args = dict(
            minify=args.minify,
            minify_js=args.minify_js,
        )
        if args.js_binary:
            finder_args['minify_js_verify_command'] = [
                args.js_binary,
                os.path.join(os.path.abspath(os.path.dirname(__file__)),
                             'js-compare-ast.js')
            ]
        finder = FileFinder(args.source, find_executables=True, **finder_args)
        if 'NO_PKG_FILES' in os.environ:
            sinkformatter = NoPkgFilesRemover(formatter, args.manifest
                                              is not None)
        else:
            sinkformatter = formatter
        sink = SimpleManifestSink(finder, sinkformatter)
        if args.manifest:
            preprocess_manifest(sink, args.manifest, defines)
        else:
            sink.add(Component(''), 'bin/*')
        sink.close(args.manifest is not None)

        if args.removals:
            removals_in = StringIO(open(args.removals).read())
            removals_in.name = args.removals
            removals = RemovedFiles(copier)
            preprocess(removals_in, removals, defines)
            copier.add(mozpath.join(respath, 'removed-files'), removals)

    # shlibsign libraries
    if launcher.can_launch():
        if not mozinfo.isMac and buildconfig.substs.get('COMPILE_ENVIRONMENT'):
            for lib in SIGN_LIBS:
                libbase = mozpath.join(respath, '%s%s') \
                    % (buildconfig.substs['DLL_PREFIX'], lib)
                libname = '%s%s' % (libbase, buildconfig.substs['DLL_SUFFIX'])
                if copier.contains(libname):
                    copier.add(
                        libbase + '.chk',
                        LibSignFile(os.path.join(args.destination, libname)))

    # Include pdb files for llvm-symbolizer to resolve symbols.
    if buildconfig.substs.get('LLVM_SYMBOLIZER') and mozinfo.isWin:
        for p, f in copier:
            if isinstance(f, ExecutableFile):
                pdbname = os.path.splitext(f.inputs()[0])[0] + '.pdb'
                if os.path.exists(pdbname):
                    copier.add(os.path.basename(pdbname), File(pdbname))

    # Setup preloading
    if args.jarlog and os.path.exists(args.jarlog):
        from mozpack.mozjar import JarLog
        log = JarLog(args.jarlog)
        for p, f in copier:
            if not isinstance(f, Jarrer):
                continue
            key = JarLog.canonicalize(os.path.join(args.destination, p))
            if key in log:
                f.preload(log[key])

    copier.copy(args.destination)
    generate_precomplete(
        os.path.normpath(os.path.join(args.destination, respath)))
Example #26
0
    def _process_android_eclipse_project_data(self, data, srcdir, objdir):
        # This can't be relative to the environment's topsrcdir,
        # because during testing topsrcdir is faked.
        template_directory = os.path.abspath(
            mozpath.join(os.path.dirname(__file__), 'templates',
                         'android_eclipse'))

        project_directory = mozpath.join(self.environment.topobjdir,
                                         'android_eclipse', data.name)
        manifest_path = mozpath.join(self.environment.topobjdir,
                                     'android_eclipse',
                                     '%s.manifest' % data.name)

        manifest = self._manifest_for_project(srcdir, data)
        ensureParentDir(manifest_path)
        manifest.write(path=manifest_path)

        classpathentries = []
        for cpe in sorted(data._classpathentries, key=lambda x: x.path):
            e = self._Element_for_classpathentry(cpe)
            classpathentries.append(ET.tostring(e))

        for name in sorted(data.referenced_projects):
            e = self._Element_for_referenced_project(name)
            classpathentries.append(ET.tostring(e))

        for name in sorted(data.extra_jars):
            e = self._Element_for_extra_jar(mozpath.join(srcdir, name))
            classpathentries.append(ET.tostring(e))

        defines = {}
        defines['IDE_OBJDIR'] = objdir
        defines['IDE_TOPOBJDIR'] = self.environment.topobjdir
        defines['IDE_SRCDIR'] = srcdir
        defines['IDE_TOPSRCDIR'] = self.environment.topsrcdir
        defines['IDE_PROJECT_NAME'] = data.name
        defines['IDE_PACKAGE_NAME'] = data.package_name
        defines['IDE_PROJECT_DIRECTORY'] = project_directory
        defines['IDE_RELSRCDIR'] = mozpath.relpath(srcdir,
                                                   self.environment.topsrcdir)
        defines['IDE_CLASSPATH_ENTRIES'] = '\n'.join(
            '\t' + cpe for cpe in classpathentries)
        defines['IDE_RECURSIVE_MAKE_TARGETS'] = ' '.join(
            sorted(data.recursive_make_targets))
        # Like android.library=true
        defines[
            'IDE_PROJECT_LIBRARY_SETTING'] = 'android.library=true' if data.is_library else ''
        # Like android.library.reference.1=FennecBrandingResources
        defines['IDE_PROJECT_LIBRARY_REFERENCES'] = '\n'.join(
            'android.library.reference.%s=%s' % (i + 1, ref)
            for i, ref in enumerate(sorted(data.included_projects)))
        if data.filtered_resources:
            filteredResources = self._Element_for_filtered_resources(
                data.filtered_resources)
            defines['IDE_PROJECT_FILTERED_RESOURCES'] = pretty_print(
                filteredResources).strip()
        else:
            defines['IDE_PROJECT_FILTERED_RESOURCES'] = ''

        copier = FileCopier()
        finder = FileFinder(template_directory)
        for input_filename, f in itertools.chain(finder.find('**'),
                                                 finder.find('.**')):
            if input_filename == 'AndroidManifest.xml' and not data.is_library:
                # Main projects supply their own manifests.
                continue
            copier.add(
                input_filename,
                PreprocessedFile(
                    mozpath.join(finder.base, input_filename),
                    depfile_path=None,
                    marker='#',
                    defines=defines,
                    extra_depends={mozpath.join(finder.base, input_filename)}))

        # When we re-create the build backend, we kill everything that was there.
        if os.path.isdir(project_directory):
            self.summary.updated_count += 1
        else:
            self.summary.created_count += 1
        copier.copy(project_directory,
                    skip_if_older=False,
                    remove_unaccounted=True)
Example #27
0
def main():
    parser = ArgumentParser()
    parser.add_argument('-D', dest='defines', action='append',
                        metavar="VAR[=VAL]", help='Define a variable')
    parser.add_argument('--format', default='omni',
                        help='Choose the chrome format for packaging ' +
                        '(omni, jar or flat ; default: %(default)s)')
    parser.add_argument('--removals', default=None,
                        help='removed-files source file')
    parser.add_argument('--ignore-errors', action='store_true', default=False,
                        help='Transform errors into warnings.')
    parser.add_argument('--minify', action='store_true', default=False,
                        help='Make some files more compact while packaging')
    parser.add_argument('--jarlog', default='', help='File containing jar ' +
                        'access logs')
    parser.add_argument('--optimizejars', action='store_true', default=False,
                        help='Enable jar optimizations')
    parser.add_argument('--unify', default='',
                        help='Base directory of another build to unify with')
    parser.add_argument('manifest', default=None, nargs='?',
                        help='Manifest file name')
    parser.add_argument('source', help='Source directory')
    parser.add_argument('destination', help='Destination directory')
    parser.add_argument('--non-resource', nargs='+', metavar='PATTERN',
                        default=[],
                        help='Extra files not to be considered as resources')
    args = parser.parse_args()

    defines = dict(buildconfig.defines)
    if args.ignore_errors:
        errors.ignore_errors()

    if args.defines:
        for name, value in [split_define(d) for d in args.defines]:
            defines[name] = value

    copier = FileCopier()
    if args.format == 'flat':
        formatter = FlatFormatter(copier)
    elif args.format == 'jar':
        formatter = JarFormatter(copier, optimize=args.optimizejars)
    elif args.format == 'omni':
        formatter = OmniJarFormatter(copier,
                                     buildconfig.substs['OMNIJAR_NAME'],
                                     optimize=args.optimizejars,
                                     non_resources=args.non_resource)
    else:
        errors.fatal('Unknown format: %s' % args.format)

    # Adjust defines according to the requested format.
    if isinstance(formatter, OmniJarFormatter):
        defines['MOZ_OMNIJAR'] = 1
    elif 'MOZ_OMNIJAR' in defines:
        del defines['MOZ_OMNIJAR']

    binpath = ''
    if 'BINPATH' in defines:
        binpath = SimpleManifestSink.normalize_path(defines['BINPATH'])
    while binpath.startswith('/'):
        binpath = binpath[1:]

    if args.unify:
        def is_native(path):
            path = os.path.abspath(path)
            return platform.machine() in mozpack.path.split(path)

        # Invert args.unify and args.source if args.unify points to the
        # native architecture.
        args.source, args.unify = sorted([args.source, args.unify],
                                         key=is_native, reverse=True)
        if is_native(args.source):
            launcher.tooldir = args.source
    elif not buildconfig.substs['CROSS_COMPILE']:
        launcher.tooldir = buildconfig.substs['LIBXUL_DIST']

    with errors.accumulate():
        if args.unify:
            finder = UnifiedBuildFinder(FileFinder(args.source),
                                        FileFinder(args.unify),
                                        minify=args.minify)
        else:
            finder = FileFinder(args.source, minify=args.minify)
        if 'NO_PKG_FILES' in os.environ:
            sinkformatter = NoPkgFilesRemover(formatter,
                                              args.manifest is not None)
        else:
            sinkformatter = formatter
        sink = SimpleManifestSink(finder, sinkformatter)
        if args.manifest:
            preprocess_manifest(sink, args.manifest, defines)
        else:
            sink.add(Component(''), 'bin/*')
        sink.close(args.manifest is not None)

        if args.removals:
            lines = [l.lstrip() for l in open(args.removals).readlines()]
            removals_in = StringIO(''.join(lines))
            removals_in.name = args.removals
            removals = RemovedFiles(copier)
            preprocess(removals_in, removals, defines)
            copier.add(mozpack.path.join(binpath, 'removed-files'), removals)

    # shlibsign libraries
    if launcher.can_launch():
        for lib in SIGN_LIBS:
            libbase = mozpack.path.join(binpath, '%s%s') \
                % (buildconfig.substs['DLL_PREFIX'], lib)
            libname = '%s%s' % (libbase, buildconfig.substs['DLL_SUFFIX'])
            if copier.contains(libname):
                copier.add(libbase + '.chk',
                           LibSignFile(os.path.join(args.destination,
                                                    libname)))

    # Setup preloading
    if args.jarlog and os.path.exists(args.jarlog):
        from mozpack.mozjar import JarLog
        log = JarLog(args.jarlog)
        for p, f in copier:
            if not isinstance(f, Jarrer):
                continue
            key = JarLog.canonicalize(os.path.join(args.destination, p))
            if key in log:
                f.preload(log[key])

    # Fill startup cache on Windows and Linux only
    # (this currently causes build failure on BSD, so skip on that platfom)
    if sys.platform == 'win32' or sys.platform.startswith ('linux'):
      if isinstance(formatter, OmniJarFormatter) and launcher.can_launch():
          if buildconfig.substs['LIBXUL_SDK']:
              gre_path = mozpack.path.join(buildconfig.substs['LIBXUL_DIST'],
                                           'bin')
          else:
              gre_path = None
          for base in sorted([[p for p in [mozpack.path.join('bin', b), b]
                              if os.path.exists(os.path.join(args.source, p))][0]
                             for b in sink.packager.get_bases()]):
              if not gre_path:
                  gre_path = base
              base_path = sink.normalize_path(base)
              if base_path in formatter.omnijars:
                  precompile_cache(formatter.omnijars[base_path],
                                   args.source, gre_path, base)

    copier.copy(args.destination)
    generate_precomplete(os.path.normpath(os.path.join(args.destination,
                                                       binpath)))
Example #28
0
def main():
    parser = ArgumentParser()
    parser.add_argument('-D', dest='defines', action='append',
                        metavar="VAR[=VAL]", help='Define a variable')
    parser.add_argument('--format', default='omni',
                        help='Choose the chrome format for packaging ' +
                        '(omni, jar or flat ; default: %(default)s)')
    parser.add_argument('--removals', default=None,
                        help='removed-files source file')
    parser.add_argument('--ignore-errors', action='store_true', default=False,
                        help='Transform errors into warnings.')
    parser.add_argument('--minify', action='store_true', default=False,
                        help='Make some files more compact while packaging')
    parser.add_argument('--minify-js', action='store_true',
                        help='Minify JavaScript files while packaging.')
    parser.add_argument('--js-binary',
                        help='Path to js binary. This is used to verify '
                        'minified JavaScript. If this is not defined, '
                        'minification verification will not be performed.')
    parser.add_argument('--jarlog', default='', help='File containing jar ' +
                        'access logs')
    parser.add_argument('--optimizejars', action='store_true', default=False,
                        help='Enable jar optimizations')
    parser.add_argument('--disable-compression', action='store_false',
                        dest='compress', default=True,
                        help='Disable jar compression')
    parser.add_argument('manifest', default=None, nargs='?',
                        help='Manifest file name')
    parser.add_argument('source', help='Source directory')
    parser.add_argument('destination', help='Destination directory')
    parser.add_argument('--non-resource', nargs='+', metavar='PATTERN',
                        default=[],
                        help='Extra files not to be considered as resources')
    args = parser.parse_args()

    defines = dict(buildconfig.defines)
    if args.ignore_errors:
        errors.ignore_errors()

    if args.defines:
        for name, value in [split_define(d) for d in args.defines]:
            defines[name] = value

    copier = FileCopier()
    if args.format == 'flat':
        formatter = FlatFormatter(copier)
    elif args.format == 'jar':
        formatter = JarFormatter(copier, compress=args.compress, optimize=args.optimizejars)
    elif args.format == 'omni':
        formatter = OmniJarFormatter(copier,
                                     buildconfig.substs['OMNIJAR_NAME'],
                                     compress=args.compress,
                                     optimize=args.optimizejars,
                                     non_resources=args.non_resource)
    else:
        errors.fatal('Unknown format: %s' % args.format)

    # Adjust defines according to the requested format.
    if isinstance(formatter, OmniJarFormatter):
        defines['MOZ_OMNIJAR'] = 1
    elif 'MOZ_OMNIJAR' in defines:
        del defines['MOZ_OMNIJAR']

    respath = ''
    if 'RESPATH' in defines:
        respath = SimpleManifestSink.normalize_path(defines['RESPATH'])
    while respath.startswith('/'):
        respath = respath[1:]

    if not buildconfig.substs['CROSS_COMPILE']:
        launcher.tooldir = mozpath.join(buildconfig.topobjdir, 'dist')

    with errors.accumulate():
        finder_args = dict(
            minify=args.minify,
            minify_js=args.minify_js,
        )
        if args.js_binary:
            finder_args['minify_js_verify_command'] = [
                args.js_binary,
                os.path.join(os.path.abspath(os.path.dirname(__file__)),
                    'js-compare-ast.js')
            ]
        finder = FileFinder(args.source, find_executables=True,
                            **finder_args)
        if 'NO_PKG_FILES' in os.environ:
            sinkformatter = NoPkgFilesRemover(formatter,
                                              args.manifest is not None)
        else:
            sinkformatter = formatter
        sink = SimpleManifestSink(finder, sinkformatter)
        if args.manifest:
            preprocess_manifest(sink, args.manifest, defines)
        else:
            sink.add(Component(''), 'bin/*')
        sink.close(args.manifest is not None)

        if args.removals:
            removals_in = StringIO(open(args.removals).read())
            removals_in.name = args.removals
            removals = RemovedFiles(copier)
            preprocess(removals_in, removals, defines)
            copier.add(mozpath.join(respath, 'removed-files'), removals)

    # shlibsign libraries
    if launcher.can_launch():
        if not mozinfo.isMac and buildconfig.substs.get('COMPILE_ENVIRONMENT'):
            for lib in SIGN_LIBS:
                libbase = mozpath.join(respath, '%s%s') \
                    % (buildconfig.substs['DLL_PREFIX'], lib)
                libname = '%s%s' % (libbase, buildconfig.substs['DLL_SUFFIX'])
                if copier.contains(libname):
                    copier.add(libbase + '.chk',
                               LibSignFile(os.path.join(args.destination,
                                                        libname)))

    # Setup preloading
    if args.jarlog and os.path.exists(args.jarlog):
        from mozpack.mozjar import JarLog
        log = JarLog(args.jarlog)
        for p, f in copier:
            if not isinstance(f, Jarrer):
                continue
            key = JarLog.canonicalize(os.path.join(args.destination, p))
            if key in log:
                f.preload(log[key])

    copier.copy(args.destination)
    generate_precomplete(os.path.normpath(os.path.join(args.destination,
                                                       respath)))
Example #29
0
    def _process_android_eclipse_project_data(self, data, srcdir, objdir):
        # This can't be relative to the environment's topsrcdir,
        # because during testing topsrcdir is faked.
        template_directory = os.path.abspath(mozpath.join(os.path.dirname(__file__),
            'templates', 'android_eclipse'))

        project_directory = mozpath.join(self.environment.topobjdir, 'android_eclipse', data.name)
        manifest_path = mozpath.join(self.environment.topobjdir, 'android_eclipse', '%s.manifest' % data.name)

        manifest = self._manifest_for_project(srcdir, data)
        ensureParentDir(manifest_path)
        manifest.write(path=manifest_path)

        classpathentries = []
        for cpe in sorted(data._classpathentries, key=lambda x: x.path):
            e = self._Element_for_classpathentry(cpe)
            classpathentries.append(ET.tostring(e))

        for name in sorted(data.referenced_projects):
            e = self._Element_for_referenced_project(name)
            classpathentries.append(ET.tostring(e))

        for name in sorted(data.extra_jars):
            e = self._Element_for_extra_jar(mozpath.join(srcdir, name))
            classpathentries.append(ET.tostring(e))

        defines = {}
        defines['IDE_OBJDIR'] = objdir
        defines['IDE_TOPOBJDIR'] = self.environment.topobjdir
        defines['IDE_SRCDIR'] = srcdir
        defines['IDE_TOPSRCDIR'] = self.environment.topsrcdir
        defines['IDE_PROJECT_NAME'] = data.name
        defines['IDE_PACKAGE_NAME'] = data.package_name
        defines['IDE_PROJECT_DIRECTORY'] = project_directory
        defines['IDE_RELSRCDIR'] = mozpath.relpath(srcdir, self.environment.topsrcdir)
        defines['IDE_CLASSPATH_ENTRIES'] = '\n'.join('\t' + cpe for cpe in classpathentries)
        defines['IDE_RECURSIVE_MAKE_TARGETS'] = ' '.join(sorted(data.recursive_make_targets))
        # Like android.library=true
        defines['IDE_PROJECT_LIBRARY_SETTING'] = 'android.library=true' if data.is_library else ''
        # Like android.library.reference.1=FennecBrandingResources
        defines['IDE_PROJECT_LIBRARY_REFERENCES'] = '\n'.join(
            'android.library.reference.%s=%s' % (i + 1, ref)
            for i, ref in enumerate(sorted(data.included_projects)))
        if data.filtered_resources:
            filteredResources = self._Element_for_filtered_resources(data.filtered_resources)
            defines['IDE_PROJECT_FILTERED_RESOURCES'] = pretty_print(filteredResources).strip()
        else:
            defines['IDE_PROJECT_FILTERED_RESOURCES'] = ''

        copier = FileCopier()
        finder = FileFinder(template_directory)
        for input_filename, f in itertools.chain(finder.find('**'), finder.find('.**')):
            if input_filename == 'AndroidManifest.xml' and not data.is_library:
                # Main projects supply their own manifests.
                continue
            copier.add(input_filename, PreprocessedFile(
                mozpath.join(finder.base, input_filename),
                depfile_path=None,
                marker='#',
                defines=defines,
                extra_depends={mozpath.join(finder.base, input_filename)}))

        # When we re-create the build backend, we kill everything that was there.
        if os.path.isdir(project_directory):
            self.summary.updated_count += 1
        else:
            self.summary.created_count += 1
        copier.copy(project_directory, skip_if_older=False, remove_unaccounted=True)
Example #30
0
    def test_file_copier(self):
        copier = FileCopier()
        copier.add('foo/bar', GeneratedFile('foobar'))
        copier.add('foo/qux', GeneratedFile('fooqux'))
        copier.add('foo/deep/nested/directory/file', GeneratedFile('fooz'))
        copier.add('bar', GeneratedFile('bar'))
        copier.add('qux/foo', GeneratedFile('quxfoo'))
        copier.add('qux/bar', GeneratedFile(''))

        result = copier.copy(self.tmpdir)
        self.assertEqual(self.all_files(self.tmpdir), set(copier.paths()))
        self.assertEqual(self.all_dirs(self.tmpdir),
                         set(['foo/deep/nested/directory', 'qux']))

        self.assertEqual(result.updated_files, set(self.tmppath(p) for p in
            self.all_files(self.tmpdir)))
        self.assertEqual(result.existing_files, set())
        self.assertEqual(result.removed_files, set())
        self.assertEqual(result.removed_directories, set())

        copier.remove('foo')
        copier.add('test', GeneratedFile('test'))
        result = copier.copy(self.tmpdir)
        self.assertEqual(self.all_files(self.tmpdir), set(copier.paths()))
        self.assertEqual(self.all_dirs(self.tmpdir), set(['qux']))
        self.assertEqual(result.removed_files, set(self.tmppath(p) for p in
            ('foo/bar', 'foo/qux', 'foo/deep/nested/directory/file')))
Example #31
0
    def test_remove_unaccounted_directory_symlinks(self):
        """Directory symlinks in destination that are not in the way are
        deleted according to remove_unaccounted and
        remove_all_directory_symlinks.
        """
        if not self.symlink_supported:
            return

        dest = self.tmppath('dest')

        copier = FileCopier()
        copier.add('foo/bar/baz', GeneratedFile('foobarbaz'))

        os.makedirs(self.tmppath('dest/foo'))
        dummy = self.tmppath('dummy')
        os.mkdir(dummy)

        os.mkdir(self.tmppath('dest/zot'))
        link = self.tmppath('dest/zot/zap')
        os.symlink(dummy, link)

        # If not remove_unaccounted but remove_empty_directories, then
        # the symlinked directory remains (as does its containing
        # directory).
        result = copier.copy(dest, remove_unaccounted=False,
            remove_empty_directories=True,
            remove_all_directory_symlinks=False)

        st = os.lstat(link)
        self.assertTrue(stat.S_ISLNK(st.st_mode))
        self.assertFalse(stat.S_ISDIR(st.st_mode))

        self.assertEqual(self.all_files(dest), set(copier.paths()))
        self.assertEqual(self.all_dirs(dest), set(['foo/bar']))

        self.assertEqual(result.removed_directories, set())
        self.assertEqual(len(result.updated_files), 1)

        # If remove_unaccounted but not remove_empty_directories, then
        # only the symlinked directory is removed.
        result = copier.copy(dest, remove_unaccounted=True,
            remove_empty_directories=False,
            remove_all_directory_symlinks=False)

        st = os.lstat(self.tmppath('dest/zot'))
        self.assertFalse(stat.S_ISLNK(st.st_mode))
        self.assertTrue(stat.S_ISDIR(st.st_mode))

        self.assertEqual(result.removed_files, set([link]))
        self.assertEqual(result.removed_directories, set())

        self.assertEqual(self.all_files(dest), set(copier.paths()))
        self.assertEqual(self.all_dirs(dest), set(['foo/bar', 'zot']))

        # If remove_unaccounted and remove_empty_directories, then
        # both the symlink and its containing directory are removed.
        link = self.tmppath('dest/zot/zap')
        os.symlink(dummy, link)

        result = copier.copy(dest, remove_unaccounted=True,
            remove_empty_directories=True,
            remove_all_directory_symlinks=False)

        self.assertEqual(result.removed_files, set([link]))
        self.assertEqual(result.removed_directories, set([self.tmppath('dest/zot')]))

        self.assertEqual(self.all_files(dest), set(copier.paths()))
        self.assertEqual(self.all_dirs(dest), set(['foo/bar']))
Example #32
0
def main():
    parser = ArgumentParser()
    parser.add_argument('-D', dest='defines', action='append',
                        metavar="VAR[=VAL]", help='Define a variable')
    parser.add_argument('--format', default='omni',
                        help='Choose the chrome format for packaging ' +
                        '(omni, jar or flat ; default: %(default)s)')
    parser.add_argument('--removals', default=None,
                        help='removed-files source file')
    parser.add_argument('--ignore-errors', action='store_true', default=False,
                        help='Transform errors into warnings.')
    parser.add_argument('--minify', action='store_true', default=False,
                        help='Make some files more compact while packaging')
    parser.add_argument('--minify-js', action='store_true',
                        help='Minify JavaScript files while packaging.')
    parser.add_argument('--js-binary',
                        help='Path to js binary. This is used to verify '
                        'minified JavaScript. If this is not defined, '
                        'minification verification will not be performed.')
    parser.add_argument('--jarlog', default='', help='File containing jar ' +
                        'access logs')
    parser.add_argument('--optimizejars', action='store_true', default=False,
                        help='Enable jar optimizations')
    parser.add_argument('--unify', default='',
                        help='Base directory of another build to unify with')
    parser.add_argument('manifest', default=None, nargs='?',
                        help='Manifest file name')
    parser.add_argument('source', help='Source directory')
    parser.add_argument('destination', help='Destination directory')
    parser.add_argument('--non-resource', nargs='+', metavar='PATTERN',
                        default=[],
                        help='Extra files not to be considered as resources')
    args = parser.parse_args()

    defines = dict(buildconfig.defines)
    if args.ignore_errors:
        errors.ignore_errors()

    if args.defines:
        for name, value in [split_define(d) for d in args.defines]:
            defines[name] = value

    copier = FileCopier()
    if args.format == 'flat':
        formatter = FlatFormatter(copier)
    elif args.format == 'jar':
        formatter = JarFormatter(copier, optimize=args.optimizejars)
    elif args.format == 'omni':
        formatter = OmniJarFormatter(copier,
                                     buildconfig.substs['OMNIJAR_NAME'],
                                     optimize=args.optimizejars,
                                     non_resources=args.non_resource)
    else:
        errors.fatal('Unknown format: %s' % args.format)

    # Adjust defines according to the requested format.
    if isinstance(formatter, OmniJarFormatter):
        defines['MOZ_OMNIJAR'] = 1
    elif 'MOZ_OMNIJAR' in defines:
        del defines['MOZ_OMNIJAR']

    respath = ''
    if 'RESPATH' in defines:
        respath = SimpleManifestSink.normalize_path(defines['RESPATH'])
    while respath.startswith('/'):
        respath = respath[1:]

    if args.unify:
        def is_native(path):
            path = os.path.abspath(path)
            return platform.machine() in mozpath.split(path)

        # Invert args.unify and args.source if args.unify points to the
        # native architecture.
        args.source, args.unify = sorted([args.source, args.unify],
                                         key=is_native, reverse=True)
        if is_native(args.source):
            launcher.tooldir = args.source
    elif not buildconfig.substs['CROSS_COMPILE']:
        launcher.tooldir = buildconfig.substs['LIBXUL_DIST']

    with errors.accumulate():
        finder_args = dict(
            minify=args.minify,
            minify_js=args.minify_js,
        )
        if args.js_binary:
            finder_args['minify_js_verify_command'] = [
                args.js_binary,
                os.path.join(os.path.abspath(os.path.dirname(__file__)),
                    'js-compare-ast.js')
            ]
        if args.unify:
            finder = UnifiedBuildFinder(FileFinder(args.source),
                                        FileFinder(args.unify),
                                        **finder_args)
        else:
            finder = FileFinder(args.source, **finder_args)
        if 'NO_PKG_FILES' in os.environ:
            sinkformatter = NoPkgFilesRemover(formatter,
                                              args.manifest is not None)
        else:
            sinkformatter = formatter
        sink = SimpleManifestSink(finder, sinkformatter)
        if args.manifest:
            preprocess_manifest(sink, args.manifest, defines)
        else:
            sink.add(Component(''), 'bin/*')
        sink.close(args.manifest is not None)

        if args.removals:
            removals_in = StringIO(open(args.removals).read())
            removals_in.name = args.removals
            removals = RemovedFiles(copier)
            preprocess(removals_in, removals, defines)
            copier.add(mozpath.join(respath, 'removed-files'), removals)

    # shlibsign libraries
    if launcher.can_launch():
        if not mozinfo.isMac:
            for lib in SIGN_LIBS:
                libbase = mozpath.join(respath, '%s%s') \
                    % (buildconfig.substs['DLL_PREFIX'], lib)
                libname = '%s%s' % (libbase, buildconfig.substs['DLL_SUFFIX'])
                if copier.contains(libname):
                    copier.add(libbase + '.chk',
                               LibSignFile(os.path.join(args.destination,
                                                        libname)))

    # Setup preloading
    if args.jarlog and os.path.exists(args.jarlog):
        from mozpack.mozjar import JarLog
        log = JarLog(args.jarlog)
        for p, f in copier:
            if not isinstance(f, Jarrer):
                continue
            key = JarLog.canonicalize(os.path.join(args.destination, p))
            if key in log:
                f.preload(log[key])

    # Fill startup cache
    if isinstance(formatter, OmniJarFormatter) and launcher.can_launch() \
      and buildconfig.substs['MOZ_DISABLE_STARTUPCACHE'] != '1':
        gre_path = None
        def get_bases():
            for b in sink.packager.get_bases(addons=False):
                for p in (mozpath.join('bin', b), b):
                    if os.path.exists(os.path.join(args.source, p)):
                        yield p
                        break
        for base in sorted(get_bases()):
            if not gre_path:
                gre_path = base
            base_path = sink.normalize_path(base)
            if base_path in formatter.omnijars:
                precompile_cache(formatter.omnijars[base_path],
                                 args.source, gre_path, base)

    copier.copy(args.destination)
    def test_file_copier(self):
        copier = FileCopier()
        copier.add('foo/bar', GeneratedFile('foobar'))
        copier.add('foo/qux', GeneratedFile('fooqux'))
        copier.add('foo/deep/nested/directory/file', GeneratedFile('fooz'))
        copier.add('bar', GeneratedFile('bar'))
        copier.add('qux/foo', GeneratedFile('quxfoo'))
        copier.add('qux/bar', GeneratedFile(''))

        result = copier.copy(self.tmpdir)
        self.assertEqual(self.all_files(self.tmpdir), set(copier.paths()))
        self.assertEqual(self.all_dirs(self.tmpdir),
                         set(['foo/deep/nested/directory', 'qux']))

        self.assertEqual(result.updated_files, set(self.tmppath(p) for p in
            self.all_files(self.tmpdir)))
        self.assertEqual(result.existing_files, set())
        self.assertEqual(result.removed_files, set())
        self.assertEqual(result.removed_directories, set())

        copier.remove('foo')
        copier.add('test', GeneratedFile('test'))
        result = copier.copy(self.tmpdir)
        self.assertEqual(self.all_files(self.tmpdir), set(copier.paths()))
        self.assertEqual(self.all_dirs(self.tmpdir), set(['qux']))
        self.assertEqual(result.removed_files, set(self.tmppath(p) for p in
            ('foo/bar', 'foo/qux', 'foo/deep/nested/directory/file')))