Exemplo n.º 1
0
def _path_args(depNames=None):
    """
    Gets the VM args for putting the dependencies named in `depNames` on the
    class path and module path (if running on JDK9 or later).

    :param names: a Dependency, str or list containing Dependency/str objects. If None,
           then all registered dependencies are used.
    """
    jdk = mx.get_jdk()
    if jdk.javaCompliance >= '1.9':
        modules = [
            as_java_module(dist, jdk) for dist in _suite.dists
            if get_java_module_info(dist)
        ]
        if modules:
            # Partition resources between the class path and module path
            modulepath = []
            classpath = []
            cpEntryToModule = {m.dist.path: m for m in modules}

            for e in mx.classpath(depNames).split(os.pathsep):
                if cpEntryToModule.has_key(e):
                    modulepath.append(cpEntryToModule[e].jarpath)
                else:
                    classpath.append(e)
            # The Truffle modules must be eagerly loaded as they could be referenced from
            # the main class hence the --add-modules argument
            return [
                '--add-modules=' + ','.join([m.name for m in modules]),
                '--module-path=' + os.pathsep.join(modulepath), '-cp',
                os.pathsep.join(classpath)
            ]
    return ['-cp', mx.classpath(depNames)]
Exemplo n.º 2
0
def _path_args(depNames=None):
    """
    Gets the VM args for putting the dependencies named in `depNames` on the
    class path and module path (if running on JDK9 or later).

    :param names: a Dependency, str or list containing Dependency/str objects. If None,
           then all registered dependencies are used.
    """
    jdk = mx.get_jdk()
    if jdk.javaCompliance >= '1.9':
        modules = [as_java_module(dist, jdk) for dist in _suite.dists if get_java_module_info(dist)]
        if modules:
            # Partition resources between the class path and module path
            modulepath = []
            classpath = []
            cpEntryToModule = {m.dist.path : m for m in modules}

            for e in mx.classpath(depNames).split(os.pathsep):
                if cpEntryToModule.has_key(e):
                    modulepath.append(cpEntryToModule[e].jarpath)
                else:
                    classpath.append(e)
            # The Truffle modules must be eagerly loaded as they could be referenced from
            # the main class hence the --add-modules argument
            return ['--add-modules=' + ','.join([m.name for m in modules]), '--module-path=' + os.pathsep.join(modulepath), '-cp', os.pathsep.join(classpath)]
    return ['-cp', mx.classpath(depNames)]
Exemplo n.º 3
0
        def get_module_jar(self):
            """
            Gets the path to the module jar file.

            :rtype: str
            """
            return as_java_module(self.dist(), jdk).jarpath
Exemplo n.º 4
0
def jlink_new_jdk(jdk, dst_jdk_dir, module_dists, root_module_names=None, missing_export_target_action='create', with_source=lambda x: True):
    """
    Uses jlink from `jdk` to create a new JDK image in `dst_jdk_dir` with `module_dists` and
    their dependencies added to the JDK image, replacing any existing modules of the same name.

    :param JDKConfig jdk: source JDK
    :param str dst_jdk_dir: path to use for the jlink --output option
    :param list module_dists: list of distributions defining modules
    :param list root_module_names: list of strings naming the module root set for the new JDK image.
                     The named modules must either be in `module_dists` or in `jdk`. If None, then
                     the root set will be all the modules in ``module_dists` and `jdk`.
    :param str missing_export_target_action: the action to perform for a qualifed export target that
                     is not present in `module_dists` and does not have a hash stored in java.base.
                     The choices are:
                       "create" - an empty module is created
                        "error" - raise an error
                           None - do nothing
    :param lambda with_source: returns True if the sources of a module distribution must be included in the new JDK
    """
    assert callable(with_source)

    if jdk.javaCompliance < '9':
        mx.abort('Cannot derive a new JDK from ' + jdk.home + ' with jlink since it is not JDK 9 or later')

    exploded_java_base_module = join(jdk.home, 'modules', 'java.base')
    if exists(exploded_java_base_module):
        mx.abort('Cannot derive a new JDK from ' + jdk.home + ' since it appears to be a developer build with exploded modules')

    jimage = join(jdk.home, 'lib', 'modules')
    jmods_dir = join(jdk.home, 'jmods')
    if not isfile(jimage):
        mx.abort('Cannot derive a new JDK from ' + jdk.home + ' since ' + jimage + ' is missing or is not an ordinary file')
    if not isdir(jmods_dir):
        mx.abort('Cannot derive a new JDK from ' + jdk.home + ' since ' + jmods_dir + ' is missing or is not a directory')

    jdk_modules = {jmd.name : jmd for jmd in jdk.get_modules()}
    modules = [as_java_module(dist, jdk) for dist in module_dists]
    all_module_names = frozenset(list(jdk_modules.keys()) + [m.name for m in modules])

    # Read hashes stored in java.base (the only module in the JDK where hashes are stored)
    out = mx.LinesOutputCapture()
    mx.run([jdk.exe_path('jmod'), 'describe', jdk_modules['java.base'].get_jmod_path()], out=out)
    lines = out.lines
    hashes = {}
    for line in lines:
        if line.startswith('hashes'):
            parts = line.split()
            assert len(parts) == 4, 'expected hashes line to have 4 fields, got {} fields: {}'.format(len(parts), line)
            _, module_name, algorithm, hash_value = parts
            hashes[module_name] = (algorithm, hash_value)

    build_dir = mx.ensure_dir_exists(join(dst_jdk_dir + ".build"))
    try:
        # Handle targets of qualified exports that are not present in `modules`
        target_requires = {}
        for jmd in modules:
            for targets in jmd.exports.values():
                for target in targets:
                    if target not in all_module_names and target not in hashes:
                        target_requires.setdefault(target, set()).add(jmd.name)
        if target_requires and missing_export_target_action is not None:
            if missing_export_target_action == 'error':
                mx.abort('Target(s) of qualified exports cannot be resolved: ' + '.'.join(target_requires.keys()))
            assert missing_export_target_action == 'create', 'invalid value for missing_export_target_action: ' + str(missing_export_target_action)

            extra_modules = []
            for name, requires in target_requires.items():
                module_jar = join(build_dir, name + '.jar')
                jmd = JavaModuleDescriptor(name, {}, requires={module : [] for module in requires}, uses=set(), provides={}, jarpath=module_jar)
                extra_modules.append(jmd)
                module_build_dir = mx.ensure_dir_exists(join(build_dir, name))
                module_info_java = join(module_build_dir, 'module-info.java')
                module_info_class = join(module_build_dir, 'module-info.class')
                with open(module_info_java, 'w') as fp:
                    print(jmd.as_module_info(), file=fp)
                mx.run([jdk.javac, '-d', module_build_dir, \
                        '--limit-modules=java.base,' + ','.join(jmd.requires.keys()), \
                        '--module-path=' + os.pathsep.join((m.jarpath for m in modules)), \
                        module_info_java])
                with ZipFile(module_jar, 'w') as zf:
                    zf.write(module_info_class, basename(module_info_class))
                if exists(jmd.get_jmod_path()):
                    os.remove(jmd.get_jmod_path())
                mx.run([jdk.javac.replace('javac', 'jmod'), 'create', '--class-path=' + module_build_dir, jmd.get_jmod_path()])

            modules.extend(extra_modules)
            all_module_names = frozenset(list(jdk_modules.keys()) + [m.name for m in modules])

        # Extract src.zip from source JDK
        jdk_src_zip = join(jdk.home, 'lib', 'src.zip')
        dst_src_zip_contents = {}
        if isfile(jdk_src_zip):
            mx.logv('[Extracting ' + jdk_src_zip + ']')
            with ZipFile(jdk_src_zip, 'r') as zf:
                for name in zf.namelist():
                    if not name.endswith('/'):
                        dst_src_zip_contents[name] = zf.read(name)
        else:
            mx.warn("'{}' does not exist or is not a file".format(jdk_src_zip))

        for jmd in modules:
            # Remove existing sources for all the modules that we include
            dst_src_zip_contents = {key : dst_src_zip_contents[key] for key in dst_src_zip_contents if not key.startswith(jmd.name)}

            if with_source(jmd.dist):
                # Add the sources that we can share.
                # Extract module sources
                jmd_src_zip = jmd.jarpath[0:-len('.jar')] + '.src.zip'
                if isfile(jmd_src_zip):
                    mx.logv('[Extracting ' + jmd_src_zip + ']')
                    with ZipFile(jmd_src_zip, 'r') as zf:
                        for name in zf.namelist():
                            if not name.endswith('/'):
                                dst_src_zip_contents[jmd.name + '/' + name] = zf.read(name)

                # Add module-info.java to sources
                dst_src_zip_contents[jmd.name + '/module-info.java'] = jmd.as_module_info(extras_as_comments=False)

        # Now build the new JDK image with jlink
        jlink = [jdk.javac.replace('javac', 'jlink')]

        if jdk_enables_jvmci_by_default(jdk):
            # On JDK 9+, +EnableJVMCI forces jdk.internal.vm.ci to be in the root set
            jlink.append('-J-XX:-EnableJVMCI')
        if root_module_names is not None:
            missing = frozenset(root_module_names) - all_module_names
            if missing:
                mx.abort('Invalid module(s): {}.\nAvailable modules: {}'.format(','.join(missing), ','.join(sorted(all_module_names))))
            jlink.append('--add-modules=' + ','.join(root_module_names))
        else:
            jlink.append('--add-modules=' + ','.join(sorted(all_module_names)))

        module_path = jmods_dir
        if modules:
            module_path = os.pathsep.join((m.get_jmod_path(respect_stripping=True) for m in modules)) + os.pathsep + module_path
        jlink.append('--module-path=' + module_path)
        jlink.append('--output=' + dst_jdk_dir)

        # These options are inspired by how OpenJDK runs jlink to produce the final runtime image.
        jlink.extend(['-J-XX:+UseSerialGC', '-J-Xms32M', '-J-Xmx512M', '-J-XX:TieredStopAtLevel=1'])
        jlink.append('-J-Dlink.debug=true')
        jlink.append('--dedup-legal-notices=error-if-not-same-content')
        jlink.append('--keep-packaged-modules=' + join(dst_jdk_dir, 'jmods'))

        # TODO: investigate the options below used by OpenJDK to see if they should be used:
        # --release-info: this allow extra properties to be written to the <jdk>/release file
        # --order-resources: specifies order of resources in generated lib/modules file.
        #       This is apparently not so important if a CDS archive is available.
        # --generate-jli-classes: pre-generates a set of java.lang.invoke classes.
        #       See https://github.com/openjdk/jdk/blob/master/make/GenerateLinkOptData.gmk
        mx.logv('[Creating JDK image]')
        mx.run(jlink)

        dst_src_zip = join(dst_jdk_dir, 'lib', 'src.zip')
        mx.logv('[Creating ' + dst_src_zip + ']')
        with ZipFile(dst_src_zip, 'w', compression=ZIP_DEFLATED, allowZip64=True) as zf:
            for name, contents in sorted(dst_src_zip_contents.items()):
                zf.writestr(name, contents)

        mx.logv('[Copying static libraries]')
        lib_prefix = mx.add_lib_prefix('')
        lib_suffix = '.lib' if mx.is_windows() else '.a'
        lib_directory = join(jdk.home, 'lib')
        dst_lib_directory = join(dst_jdk_dir, 'lib')
        for f in os.listdir(lib_directory):
            if f.startswith(lib_prefix) and f.endswith(lib_suffix):
                lib_path = join(lib_directory, f)
                if isfile(lib_path):
                    shutil.copy2(lib_path, dst_lib_directory)

        # Build the list of modules whose classes might have annotations
        # to be processed by native-image (GR-15192).
        with open(join(dst_jdk_dir, 'lib', 'native-image-modules.list'), 'w') as fp:
            print('# Modules whose classes might have annotations processed by native-image', file=fp)
            for m in modules:
                print(m.name, file=fp)

    finally:
        if not mx.get_opts().verbose:
            # Preserve build directory so that javac command can be re-executed
            # by cutting and pasting verbose output.
            shutil.rmtree(build_dir)

    # Create CDS archive (https://openjdk.java.net/jeps/341).
    out = mx.OutputCapture()
    mx.logv('[Creating CDS shared archive]')
    if mx.run([mx.exe_suffix(join(dst_jdk_dir, 'bin', 'java')), '-Xshare:dump', '-Xmx128M', '-Xms128M'], out=out, err=out, nonZeroIsFatal=False) != 0:
        mx.log(out.data)
        mx.abort('Error generating CDS shared archive')
Exemplo n.º 5
0
def _parseVmArgs(args, addDefaultArgs=True):
    args = mx.expand_project_in_args(args, insitu=False)

    argsPrefix = []
    jacocoArgs = mx_gate.get_jacoco_agent_args()
    if jacocoArgs:
        argsPrefix.extend(jacocoArgs)

    # Check for -G: options
    def checkGOption(arg):
        if arg.startswith('-G:+'):
            if '=' in arg:
                mx.abort('Mixing + and = in -G: option specification: ' + arg)
            translation = '-Dgraal.' + arg[len('-G:+'):] + '=true'
        elif arg.startswith('-G:-'):
            if '=' in arg:
                mx.abort('Mixing - and = in -G: option specification: ' + arg)
            translation = '-Dgraal.' + arg[len('-G:+'):] + '=false'
        elif arg.startswith('-G:'):
            if '=' not in arg:
                mx.abort('Missing "=" in non-boolean -G: option specification: ' + arg)
            translation = '-Dgraal.' + arg[len('-G:'):]
        else:
            return arg
        mx.warn('Support for -G options is deprecated and will soon be removed. Replace "' + arg + '" with "' + translation + '"')
        return translation

    # add default graal.options.file
    options_file = join(mx.primary_suite().dir, 'graal.options')
    if exists(options_file):
        argsPrefix.append('-Dgraal.options.file=' + options_file)
    args = [checkGOption(a) for a in args]

    if '-Dgraal.PrintFlags=true' in args and '-Xcomp' not in args:
        mx.warn('Using -Dgraal.PrintFlags=true may have no effect without -Xcomp as Graal initialization is lazy')

    if isJDK8:
        argsPrefix.append('-Djvmci.class.path.append=' + os.pathsep.join((e.get_path() for e in _jvmci_classpath)))
        argsPrefix.append('-Xbootclasspath/a:' + os.pathsep.join([dep.classpath_repr() for dep in _bootclasspath_appends]))
    else:
        deployedDists = [entry.dist() for entry in _jvmci_classpath] + \
                        [e for e in _bootclasspath_appends if e.isJARDistribution()]
        deployedModules = [as_java_module(dist, jdk) for dist in deployedDists]

        # Set or update module path to include Graal and its dependencies as modules
        graalModulepath = []
        for deployedModule in deployedModules:
            graalModulepath.extend([jmd.jarpath for jmd in deployedModule.modulepath if jmd.jarpath])
            graalModulepath.append(deployedModule.jarpath)
        graalModulepath = _uniqify(graalModulepath)

        # Update added exports to include concealed JDK packages required by Graal
        addedExports = {}
        args = _extract_added_exports(args, addedExports)
        for deployedModule in deployedModules:
            for concealingModule, packages in deployedModule.concealedRequires.iteritems():
                # No need to explicitly export JVMCI - it's exported via reflection
                if concealingModule != 'jdk.vm.ci':
                    for package in packages:
                        addedExports.setdefault(concealingModule + '/' + package, set()).add(deployedModule.name)

        for export, targets in addedExports.iteritems():
            argsPrefix.append('-XaddExports:' + export + '=' + ','.join(sorted(targets)))

        # Extend or set -modulepath argument
        mpUpdated = False
        for mpIndex in range(len(args)):
            if args[mpIndex] in ['-modulepath', '-mp']:
                assert mpIndex + 1 < len(args), 'VM option ' + args[mpIndex] + ' requires an argument'
                args[mpIndex + 1] = os.pathsep.join(_uniqify(args[mpIndex + 1].split(os.pathsep) + graalModulepath))
                mpUpdated = True
                break
        if not mpUpdated:
            argsPrefix.append('-modulepath')
            argsPrefix.append(os.pathsep.join(graalModulepath))

    # Set the JVMCI compiler to Graal
    argsPrefix.append('-Djvmci.Compiler=graal')

    if '-version' in args:
        ignoredArgs = args[args.index('-version') + 1:]
        if  len(ignoredArgs) > 0:
            mx.log("Warning: The following options will be ignored by the VM because they come after the '-version' argument: " + ' '.join(ignoredArgs))

    return jdk.processArgs(argsPrefix + args, addDefaultArgs=addDefaultArgs)
Exemplo n.º 6
0
def _unittest_config_participant(config):
    vmArgs, mainClass, mainClassArgs = config
    cpIndex, cp = mx.find_classpath_arg(vmArgs)
    if cp:
        cp = _uniqify(cp.split(os.pathsep))
        if isJDK8:
            # Remove entries from class path that are in Graal or on the boot class path
            redundantClasspathEntries = set()
            for dist in [entry.dist() for entry in _jvmci_classpath]:
                redundantClasspathEntries.update((d.output_dir() for d in dist.archived_deps() if d.isJavaProject()))
                redundantClasspathEntries.add(dist.path)
            cp = os.pathsep.join([e for e in cp if e not in redundantClasspathEntries])
            vmArgs[cpIndex] = cp
        else:
            deployedModules = []
            redundantClasspathEntries = set()
            for dist in [entry.dist() for entry in _jvmci_classpath] + _bootclasspath_appends:
                deployedModule = as_java_module(dist, jdk)
                deployedModules.append(deployedModule)
                redundantClasspathEntries.update(mx.classpath(dist, preferProjects=False, jdk=jdk).split(os.pathsep))
                redundantClasspathEntries.update(mx.classpath(dist, preferProjects=True, jdk=jdk).split(os.pathsep))

            # Remove entries from the class path that are in the deployed modules
            cp = [classpathEntry for classpathEntry in cp if classpathEntry not in redundantClasspathEntries]
            vmArgs[cpIndex] = os.pathsep.join(cp)

            # Junit libraries are made into automatic modules so that they are visible to tests
            # patched into modules. These automatic modules must be declared to be read by
            # Graal which means they must also be made root modules (i.e., ``-addmods``)
            # since ``-XaddReads`` can only be applied to root modules.
            junitCp = [e.classpath_repr() for e in mx.classpath_entries(['JUNIT'])]
            junitModules = [_automatic_module_name(e) for e in junitCp]
            vmArgs.extend(['-modulepath', os.pathsep.join(junitCp)])
            vmArgs.extend(['-addmods', ','.join(junitModules + [m.name for m in deployedModules])])
            for deployedModule in deployedModules:
                vmArgs.append('-XaddReads:' + deployedModule.name + '=' + ','.join(junitModules))

            # Explicitly export concealed JVMCI packages required by Graal. Even though
            # normally exported via jdk.vm.ci.services.Services.exportJVMCITo(), the
            # Junit harness wants to access JVMCI classes (e.g., when loading classes
            # to find test methods) without going through that entry point.
            addedExports = {}
            for deployedModule in deployedModules:
                for concealingModule, packages in deployedModule.concealedRequires.iteritems():
                    if concealingModule == 'jdk.vm.ci':
                        for package in packages:
                            addedExports.setdefault(concealingModule + '/' + package, set()).add(deployedModule.name)

            patches = {}
            pathToProject = {p.output_dir() : p for p in mx.projects() if p.isJavaProject()}
            for classpathEntry in cp:
                # Export concealed packages used by the class path entry
                _add_exports_for_concealed_packages(classpathEntry, pathToProject, addedExports, 'ALL-UNNAMED', deployedModules)

                for deployedModule in deployedModules:
                    assert deployedModule.dist.path != classpathEntry, deployedModule.dist.path + ' should no longer be on the class path'
                    # Patch the class path entry into a module if it defines packages already defined by the module.
                    # Packages definitions cannot be split between modules.
                    classpathEntryPackages = frozenset(_defined_packages(classpathEntry))
                    if not classpathEntryPackages.isdisjoint(deployedModule.packages):
                        patches.setdefault(deployedModule.name, []).append(classpathEntry)
                        extraPackages = classpathEntryPackages - frozenset(deployedModule.exports.iterkeys())
                        if extraPackages:
                            # From http://openjdk.java.net/jeps/261:
                            # If a package found in a module definition on a patch path is not already exported
                            # by that module then it will, still, not be exported. It can be exported explicitly
                            # via either the reflection API or the -XaddExports option.
                            for package in extraPackages:
                                addedExports.setdefault(deployedModule.name + '/' + package, set()).update(junitModules + ['ALL-UNNAMED'])

            for moduleName, cpEntries in patches.iteritems():
                vmArgs.append('-Xpatch:' + moduleName + '=' + os.pathsep.join(cpEntries))

            vmArgs.extend(['-XaddExports:' + export + '=' + ','.join(sorted(targets)) for export, targets in addedExports.iteritems()])

    if isJDK8:
        # Run the VM in a mode where application/test classes can
        # access JVMCI loaded classes.
        vmArgs.append('-XX:-UseJVMCIClassLoader')

    return (vmArgs, mainClass, mainClassArgs)
Exemplo n.º 7
0
def _unittest_config_participant(config):
    vmArgs, mainClass, mainClassArgs = config
    cpIndex, cp = mx.find_classpath_arg(vmArgs)
    if cp:
        cp = _uniqify(cp.split(os.pathsep))
        if isJDK8:
            # Remove entries from class path that are in Graal
            excluded = set()
            for entry in _jvmci_classpath:
                dist = entry.dist()
                excluded.update((d.output_dir() for d in dist.archived_deps() if d.isJavaProject()))
                excluded.add(dist.path)
            cp = os.pathsep.join([e for e in cp if e not in excluded])
            vmArgs[cpIndex] = cp
        else:
            # Remove entries from class path that are in the Graal
            assert _graal_module_descriptor is not None
            module = as_java_module(_graal_module_descriptor.dist(), jdk)

            junitCp = [e.classpath_repr() for e in mx.classpath_entries(['JUNIT'])]
            excluded = frozenset([classpathEntry.classpath_repr() for classpathEntry in get_module_deps(module.dist)] + junitCp)

            cp = [classpathEntry for classpathEntry in cp if classpathEntry not in excluded]

            vmArgs[cpIndex] = os.pathsep.join(cp)

            # Junit libraries are made into automatic modules so that they are visible to tests
            # patched into Graal. These automatic modules must be declared to be read by
            # Graal which means they must also be made root modules (i.e., ``-addmods``)
            # since ``-XaddReads`` can only be applied to root modules.
            junitModules = [_automatic_module_name(e) for e in junitCp]
            vmArgs.extend(['-modulepath', os.pathsep.join(junitCp)])
            vmArgs.extend(['-addmods', ','.join(junitModules)])
            vmArgs.extend(['-XaddReads:' + module.name + '=' + ','.join(junitModules)])

            # Explicitly export JVMCI to Graal
            addedExports = {}
            for concealingModule, packages in module.concealedRequires.iteritems():
                if concealingModule == 'jdk.vm.ci':
                    for package in packages:
                        addedExports.setdefault(concealingModule + '/' + package, set()).add(module.name)

            patches = []
            graalConcealedPackages = list(module.conceals)
            pathToProject = {p.output_dir() : p for p in mx.projects() if p.isJavaProject()}
            for classpathEntry in cp:
                # Export concealed JDK packages used by the class path entry
                _add_exports_for_concealed_packages(classpathEntry, pathToProject, addedExports, 'ALL-UNNAMED')

                # Patch the class path entry into Graal if it defines packages already defined by Graal.
                # Packages definitions cannot be split between modules.
                packages = frozenset(_defined_packages(classpathEntry))
                if not packages.isdisjoint(module.packages):
                    patches.append(classpathEntry)
                    extraPackages = packages - module.packages
                    if extraPackages:
                        # From http://openjdk.java.net/jeps/261:
                        # If a package found in a module definition on a patch path is not already exported
                        # by that module then it will, still, not be exported. It can be exported explicitly
                        # via either the reflection API or the -XaddExports option.
                        graalConcealedPackages.extend(extraPackages)

            if patches:
                vmArgs.append('-Xpatch:' + module.name + '=' + os.pathsep.join(patches))

            # Export all Graal packages to make them available to test classes
            for package in graalConcealedPackages:
                addedExports.setdefault(module.name + '/' + package, set()).update(junitModules + ['ALL-UNNAMED'])

            vmArgs.extend(['-XaddExports:' + export + '=' + ','.join(sorted(targets)) for export, targets in addedExports.iteritems()])

    if isJDK8:
        # Run the VM in a mode where application/test classes can
        # access JVMCI loaded classes.
        vmArgs.append('-XX:-UseJVMCIClassLoader')

    return (vmArgs, mainClass, mainClassArgs)
Exemplo n.º 8
0
def jlink_new_jdk(jdk, dst_jdk_dir, module_dists,
                  root_module_names=None,
                  missing_export_target_action='create',
                  with_source=lambda x: True,
                  vendor_info=None,
                  dedup_legal_notices=True):
    """
    Uses jlink from `jdk` to create a new JDK image in `dst_jdk_dir` with `module_dists` and
    their dependencies added to the JDK image, replacing any existing modules of the same name.

    :param JDKConfig jdk: source JDK
    :param str dst_jdk_dir: path to use for the jlink --output option
    :param list module_dists: list of distributions defining modules
    :param list root_module_names: list of strings naming the module root set for the new JDK image.
                     The named modules must either be in `module_dists` or in `jdk`. If None, then
                     the root set will be all the modules in ``module_dists` and `jdk`.
    :param str missing_export_target_action: the action to perform for a qualifed export target that
                     is not present in `module_dists` and does not have a hash stored in java.base.
                     The choices are:
                       "create" - an empty module is created
                        "error" - raise an error
                           None - do nothing
    :param lambda with_source: returns True if the sources of a module distribution must be included in the new JDK
    :param dict vendor_info: values for the jlink vendor options added by JDK-8232080
    """
    assert callable(with_source)

    if jdk.javaCompliance < '9':
        mx.abort('Cannot derive a new JDK from ' + jdk.home + ' with jlink since it is not JDK 9 or later')

    exploded_java_base_module = join(jdk.home, 'modules', 'java.base')
    if exists(exploded_java_base_module):
        mx.abort('Cannot derive a new JDK from ' + jdk.home + ' since it appears to be a developer build with exploded modules')

    jimage = join(jdk.home, 'lib', 'modules')
    jmods_dir = join(jdk.home, 'jmods')
    if not isfile(jimage):
        mx.abort('Cannot derive a new JDK from ' + jdk.home + ' since ' + jimage + ' is missing or is not an ordinary file')
    if not isdir(jmods_dir):
        mx.abort('Cannot derive a new JDK from ' + jdk.home + ' since ' + jmods_dir + ' is missing or is not a directory')

    # Exclude jdk.aot due to GR-10545 and JDK-8255616
    jdk_modules = {jmd.name: jmd for jmd in jdk.get_modules() if jmd.name != 'jdk.aot'}
    modules = [as_java_module(dist, jdk) for dist in module_dists]
    all_module_names = frozenset(list(jdk_modules.keys()) + [m.name for m in modules])

    # Read hashes stored in java.base (the only module in the JDK where hashes are stored)
    out = mx.LinesOutputCapture()
    mx.run([jdk.exe_path('jmod'), 'describe', jdk_modules['java.base'].get_jmod_path()], out=out)
    lines = out.lines
    hashes = {}
    for line in lines:
        if line.startswith('hashes'):
            parts = line.split()
            assert len(parts) == 4, 'expected hashes line to have 4 fields, got {} fields: {}'.format(len(parts), line)
            _, module_name, algorithm, hash_value = parts
            hashes[module_name] = (algorithm, hash_value)

    build_dir = mx.ensure_dir_exists(join(dst_jdk_dir + ".build"))
    try:
        # Handle targets of qualified exports that are not present in `modules`
        target_requires = {}
        for jmd in modules:
            for targets in jmd.exports.values():
                for target in targets:
                    if target not in all_module_names and target not in hashes:
                        target_requires.setdefault(target, set()).add(jmd.name)
        if target_requires and missing_export_target_action is not None:
            if missing_export_target_action == 'error':
                mx.abort('Target(s) of qualified exports cannot be resolved: ' + '.'.join(target_requires.keys()))
            assert missing_export_target_action == 'create', 'invalid value for missing_export_target_action: ' + str(missing_export_target_action)

            extra_modules = []
            for name, requires in target_requires.items():
                module_jar = join(build_dir, name + '.jar')
                jmd = JavaModuleDescriptor(name, {}, requires={module: [] for module in requires}, uses=set(), provides={}, jarpath=module_jar)
                extra_modules.append(jmd)
                module_build_dir = mx.ensure_dir_exists(join(build_dir, name))
                module_info_java = join(module_build_dir, 'module-info.java')
                module_info_class = join(module_build_dir, 'module-info.class')
                with open(module_info_java, 'w') as fp:
                    print(jmd.as_module_info(), file=fp)
                mx.run([jdk.javac, '-d', module_build_dir,
                        '--limit-modules=java.base,' + ','.join(jmd.requires.keys()),
                        '--module-path=' + os.pathsep.join((m.jarpath for m in modules)),
                        module_info_java])
                with ZipFile(module_jar, 'w') as zf:
                    zf.write(module_info_class, basename(module_info_class))
                if exists(jmd.get_jmod_path()):
                    os.remove(jmd.get_jmod_path())
                mx.run([jdk.javac.replace('javac', 'jmod'), 'create', '--class-path=' + module_build_dir, jmd.get_jmod_path()])

            modules.extend(extra_modules)
            all_module_names = frozenset(list(jdk_modules.keys()) + [m.name for m in modules])

        # Extract src.zip from source JDK
        jdk_src_zip = join(jdk.home, 'lib', 'src.zip')
        dst_src_zip_contents = {}
        if isfile(jdk_src_zip):
            mx.logv('[Extracting ' + jdk_src_zip + ']')
            with ZipFile(jdk_src_zip, 'r') as zf:
                for name in zf.namelist():
                    if not name.endswith('/'):
                        dst_src_zip_contents[name] = zf.read(name)
        else:
            mx.warn("'{}' does not exist or is not a file".format(jdk_src_zip))

        # Edit lib/security/default.policy in java.base
        patched_java_base = join(build_dir, 'java.base.jmod')
        with open(join(jmods_dir, 'java.base.jmod'), 'rb') as src_f, open(patched_java_base, 'wb') as dst_f:
            jmod_header = src_f.read(4)
            if len(jmod_header) != 4 or jmod_header != b'JM\x01\x00':
                raise mx.abort("Unexpected jmod header: " + b2a_hex(jmod_header).decode('ascii'))
            dst_f.write(jmod_header)
            policy_result = 'not found'
            with ZipFile(src_f, 'r') as src_zip, ZipFile(dst_f, 'w', src_zip.compression) as dst_zip:
                for i in src_zip.infolist():
                    if i.filename[-1] == '/':
                        continue
                    src_member = src_zip.read(i)
                    if i.filename == 'lib/security/default.policy':
                        policy_result = 'unmodified'
                        if 'grant codeBase "jrt:/com.oracle.graal.graal_enterprise"'.encode('utf-8') not in src_member:
                            policy_result = 'modified'
                            src_member += """
grant codeBase "jrt:/com.oracle.graal.graal_enterprise" {
    permission java.security.AllPermission;
};
""".encode('utf-8')
                        if 'grant codeBase "jrt:/org.graalvm.truffle"'.encode('utf-8') not in src_member:
                            policy_result = 'modified'
                            src_member += """
grant codeBase "jrt:/org.graalvm.truffle" {
    permission java.security.AllPermission;
};

grant codeBase "jrt:/org.graalvm.sdk" {
    permission java.security.AllPermission;
};

grant codeBase "jrt:/org.graalvm.locator" {
  permission java.io.FilePermission "<<ALL FILES>>", "read";
  permission java.util.PropertyPermission "*", "read,write";
  permission java.lang.RuntimePermission "createClassLoader";
  permission java.lang.RuntimePermission "getClassLoader";
  permission java.lang.RuntimePermission "getenv.*";
};

grant codeBase "file:${java.home}/languages/-" {
    permission java.security.AllPermission;
};
""".encode('utf-8')
                    dst_zip.writestr(i, src_member)
            if policy_result == 'not found':
                raise mx.abort("Couldn't find `lib/security/default.policy` in " + join(jmods_dir, 'java.base.jmod'))

        for jmd in modules:
            # Remove existing sources for all the modules that we include
            dst_src_zip_contents = {key: dst_src_zip_contents[key] for key in dst_src_zip_contents if not key.startswith(jmd.name)}

            if with_source(jmd.dist):
                # Add the sources that we can share.
                # Extract module sources
                jmd_src_zip = jmd.jarpath[0:-len('.jar')] + '.src.zip'
                if isfile(jmd_src_zip):
                    mx.logv('[Extracting ' + jmd_src_zip + ']')
                    with ZipFile(jmd_src_zip, 'r') as zf:
                        for name in zf.namelist():
                            if not name.endswith('/'):
                                dst_src_zip_contents[jmd.name + '/' + name] = zf.read(name)

                # Add module-info.java to sources
                dst_src_zip_contents[jmd.name + '/module-info.java'] = jmd.as_module_info(extras_as_comments=False)

        # Now build the new JDK image with jlink
        jlink = [jdk.javac.replace('javac', 'jlink')]

        if jdk_enables_jvmci_by_default(jdk):
            # On JDK 9+, +EnableJVMCI forces jdk.internal.vm.ci to be in the root set
            jlink += ['-J-XX:-EnableJVMCI', '-J-XX:-UseJVMCICompiler']
        if root_module_names is not None:
            missing = frozenset(root_module_names) - all_module_names
            if missing:
                mx.abort('Invalid module(s): {}.\nAvailable modules: {}'.format(','.join(missing), ','.join(sorted(all_module_names))))
            jlink.append('--add-modules=' + ','.join(root_module_names))
        else:
            jlink.append('--add-modules=' + ','.join(sorted(all_module_names)))

        module_path = patched_java_base + os.pathsep + jmods_dir
        if modules:
            module_path = os.pathsep.join((m.get_jmod_path(respect_stripping=True) for m in modules)) + os.pathsep + module_path
        jlink.append('--module-path=' + module_path)
        jlink.append('--output=' + dst_jdk_dir)

        # These options are derived from how OpenJDK runs jlink to produce the final runtime image.
        jlink.extend(['-J-XX:+UseSerialGC', '-J-Xms32M', '-J-Xmx512M', '-J-XX:TieredStopAtLevel=1'])
        jlink.append('-J-Dlink.debug=true')
        if dedup_legal_notices:
            jlink.append('--dedup-legal-notices=error-if-not-same-content')
        jlink.append('--keep-packaged-modules=' + join(dst_jdk_dir, 'jmods'))

        if jdk_has_new_jlink_options(jdk):
            if jdk_omits_warning_for_jlink_set_ThreadPriorityPolicy(jdk):
                thread_priority_policy_option = ' -XX:ThreadPriorityPolicy=1'
            else:
                mx.logv('[Creating JDK without -XX:ThreadPriorityPolicy=1]')
                thread_priority_policy_option = ''

            if jdk_supports_enablejvmciproduct(jdk):
                if any((m.name == 'jdk.internal.vm.compiler' for m in modules)):
                    jlink.append('--add-options=-XX:+UnlockExperimentalVMOptions -XX:+EnableJVMCIProduct -XX:-UnlockExperimentalVMOptions' + thread_priority_policy_option)
                else:
                    # Don't default to using JVMCI as JIT unless Graal is being updated in the image.
                    # This avoids unexpected issues with using the out-of-date Graal compiler in
                    # the JDK itself.
                    jlink.append('--add-options=-XX:+UnlockExperimentalVMOptions -XX:+EnableJVMCIProduct -XX:-UseJVMCICompiler -XX:-UnlockExperimentalVMOptions' + thread_priority_policy_option)
            else:
                mx.logv('[Creating JDK without -XX:+EnableJVMCIProduct]')
                if thread_priority_policy_option:
                    jlink.append('--add-options=' + thread_priority_policy_option.strip())
            if vendor_info is not None:
                for name, value in vendor_info.items():
                    jlink.append('--' + name + '=' + value)

        release_file = join(jdk.home, 'release')
        if isfile(release_file):
            jlink.append('--release-info=' + release_file)

        # TODO: investigate the options below used by OpenJDK to see if they should be used:
        # --order-resources: specifies order of resources in generated lib/modules file.
        #       This is apparently not so important if a CDS archive is available.
        # --generate-jli-classes: pre-generates a set of java.lang.invoke classes.
        #       See https://github.com/openjdk/jdk/blob/master/make/GenerateLinkOptData.gmk
        mx.logv('[Creating JDK image in {}]'.format(dst_jdk_dir))
        mx.run(jlink)

        dst_src_zip = join(dst_jdk_dir, 'lib', 'src.zip')
        mx.logv('[Creating ' + dst_src_zip + ']')
        with ZipFile(dst_src_zip, 'w', compression=ZIP_DEFLATED, allowZip64=True) as zf:
            for name, contents in sorted(dst_src_zip_contents.items()):
                zf.writestr(name, contents)

        mx.logv('[Copying static libraries]')
        lib_directory = join(jdk.home, 'lib', 'static')
        if exists(lib_directory):
            dst_lib_directory = join(dst_jdk_dir, 'lib', 'static')
            try:
                mx.copytree(lib_directory, dst_lib_directory)
            except shutil.Error as e:
                # On AArch64, there can be a problem in the copystat part
                # of copytree which occurs after file and directory copying
                # has successfully completed. Since the metadata doesn't
                # matter in this case, just ensure that the content was copied.
                for root, _, lib_files in os.walk(lib_directory):
                    relative_root = os.path.relpath(root, dst_lib_directory)
                    for lib in lib_files:
                        src_lib_path = join(root, lib)
                        dst_lib_path = join(dst_lib_directory, relative_root, lib)
                        if not exists(dst_lib_path):
                            mx.abort('Error copying static libraries: {} missing in {}{}Original copytree error: {}'.format(
                                join(relative_root, lib), dst_lib_directory, os.linesep, e))
                        src_lib_hash = mx.sha1OfFile(src_lib_path)
                        dst_lib_hash = mx.sha1OfFile(dst_lib_path)
                        if src_lib_hash != dst_lib_hash:
                            mx.abort('Error copying static libraries: {} (hash={}) and {} (hash={}) differ{}Original copytree error: {}'.format(
                                src_lib_path, src_lib_hash,
                                dst_lib_path, dst_lib_hash,
                                os.linesep, e))
        # Allow older JDK versions to work
        else:
            lib_prefix = mx.add_lib_prefix('')
            lib_suffix = mx.add_static_lib_suffix('')
            lib_directory = join(jdk.home, 'lib')
            dst_lib_directory = join(dst_jdk_dir, 'lib')
            for f in os.listdir(lib_directory):
                if f.startswith(lib_prefix) and f.endswith(lib_suffix):
                    lib_path = join(lib_directory, f)
                    if isfile(lib_path):
                        shutil.copy2(lib_path, dst_lib_directory)
    finally:
        if not mx.get_opts().verbose:
            # Preserve build directory so that javac command can be re-executed
            # by cutting and pasting verbose output.
            shutil.rmtree(build_dir)

    # Create CDS archive (https://openjdk.java.net/jeps/341).
    out = mx.OutputCapture()
    mx.logv('[Creating CDS shared archive]')
    if mx.run([mx.exe_suffix(join(dst_jdk_dir, 'bin', 'java')), '-Xshare:dump', '-Xmx128M', '-Xms128M'], out=out, err=out, nonZeroIsFatal=False) != 0:
        mx.log(out.data)
        mx.abort('Error generating CDS shared archive')
Exemplo n.º 9
0
def _parseVmArgs(args, addDefaultArgs=True):
    args = mx.expand_project_in_args(args, insitu=False)

    argsPrefix = []
    jacocoArgs = mx_gate.get_jacoco_agent_args()
    if jacocoArgs:
        argsPrefix.extend(jacocoArgs)

    # add default graal.options.file
    options_file = join(mx.primary_suite().dir, 'graal.options')
    if exists(options_file):
        argsPrefix.append('-Dgraal.options.file=' + options_file)

    if isJDK8:
        argsPrefix.append('-Djvmci.class.path.append=' + os.pathsep.join((e.get_path() for e in _jvmci_classpath)))
        argsPrefix.append('-Xbootclasspath/a:' + os.pathsep.join([dep.classpath_repr() for dep in _bootclasspath_appends]))
    else:
        deployedDists = [entry.dist() for entry in _jvmci_classpath] + \
                        [e for e in _bootclasspath_appends if e.isJARDistribution()]
        deployedModules = [as_java_module(dist, jdk) for dist in deployedDists]

        # Set or update module path to include Graal and its dependencies as modules
        graalModulepath = []
        for deployedModule in deployedModules:
            graalModulepath.extend([jmd.jarpath for jmd in deployedModule.modulepath if jmd.jarpath])
            graalModulepath.append(deployedModule.jarpath)
        graalModulepath = _uniqify(graalModulepath)

        # Update added exports to include concealed JDK packages required by Graal
        addedExports = {}
        args = _extract_added_exports(args, addedExports)
        for deployedModule in deployedModules:
            for concealingModule, packages in deployedModule.concealedRequires.iteritems():
                # No need to explicitly export JVMCI - it's exported via reflection
                if concealingModule != 'jdk.vm.ci':
                    for package in packages:
                        addedExports.setdefault(concealingModule + '/' + package, set()).add(deployedModule.name)

        for export, targets in addedExports.iteritems():
            argsPrefix.append('--add-exports=' + export + '=' + ','.join(sorted(targets)))

        # Extend or set --module-path argument
        mpUpdated = False
        for mpIndex in range(len(args)):
            if args[mpIndex] == '--module-path':
                assert mpIndex + 1 < len(args), 'VM option ' + args[mpIndex] + ' requires an argument'
                args[mpIndex + 1] = os.pathsep.join(_uniqify(args[mpIndex + 1].split(os.pathsep) + graalModulepath))
                mpUpdated = True
                break
            elif args[mpIndex].startswith('--module-path='):
                mp = args[mpIndex][len('--module-path='):]
                args[mpIndex] = '--module-path=' + os.pathsep.join(_uniqify(mp.split(os.pathsep) + graalModulepath))
                mpUpdated = True
                break
        if not mpUpdated:
            argsPrefix.append('--module-path=' + os.pathsep.join(graalModulepath))

    if '-version' in args:
        ignoredArgs = args[args.index('-version') + 1:]
        if  len(ignoredArgs) > 0:
            mx.log("Warning: The following options will be ignored by the VM because they come after the '-version' argument: " + ' '.join(ignoredArgs))

    return jdk.processArgs(argsPrefix + args, addDefaultArgs=addDefaultArgs)
Exemplo n.º 10
0
def _unittest_config_participant(config):
    vmArgs, mainClass, mainClassArgs = config
    cpIndex, cp = mx.find_classpath_arg(vmArgs)
    if cp:
        cp = _uniqify(cp.split(os.pathsep))
        if isJDK8:
            # Remove entries from class path that are in Graal or on the boot class path
            redundantClasspathEntries = set()
            for dist in [entry.dist() for entry in _jvmci_classpath]:
                redundantClasspathEntries.update((d.output_dir() for d in dist.archived_deps() if d.isJavaProject()))
                redundantClasspathEntries.add(dist.path)
            cp = os.pathsep.join([e for e in cp if e not in redundantClasspathEntries])
            vmArgs[cpIndex] = cp
        else:
            deployedModules = []
            redundantClasspathEntries = set()
            for dist in [entry.dist() for entry in _jvmci_classpath] + _bootclasspath_appends:
                deployedModule = as_java_module(dist, jdk)
                deployedModules.append(deployedModule)
                redundantClasspathEntries.update(mx.classpath(dist, preferProjects=False, jdk=jdk).split(os.pathsep))
                redundantClasspathEntries.update(mx.classpath(dist, preferProjects=True, jdk=jdk).split(os.pathsep))
                if hasattr(dist, 'overlaps'):
                    for o in dist.overlaps:
                        path = mx.distribution(o).classpath_repr()
                        if path:
                            redundantClasspathEntries.add(path)

            # Remove entries from the class path that are in the deployed modules
            cp = [classpathEntry for classpathEntry in cp if classpathEntry not in redundantClasspathEntries]
            vmArgs[cpIndex] = os.pathsep.join(cp)

            # Junit libraries are made into automatic modules so that they are visible to tests
            # patched into modules. These automatic modules must be declared to be read by
            # Graal which means they must also be made root modules (i.e., ``--add-modules``)
            # since ``--add-reads`` can only be applied to root modules.
            junitCp = [e.classpath_repr() for e in mx.classpath_entries(['JUNIT'])]
            junitModules = [_automatic_module_name(e) for e in junitCp]
            vmArgs.append('--module-path=' + os.pathsep.join(junitCp))
            vmArgs.append('--add-modules=' + ','.join(junitModules + [m.name for m in deployedModules]))
            for deployedModule in deployedModules:
                vmArgs.append('--add-reads=' + deployedModule.name + '=' + ','.join(junitModules))

            # Explicitly export concealed JVMCI packages required by Graal. Even though
            # normally exported via jdk.vm.ci.services.Services.exportJVMCITo(), the
            # Junit harness wants to access JVMCI classes (e.g., when loading classes
            # to find test methods) without going through that entry point.
            addedExports = {}
            for deployedModule in deployedModules:
                for concealingModule, packages in deployedModule.concealedRequires.iteritems():
                    if concealingModule == 'jdk.vm.ci':
                        for package in packages:
                            addedExports.setdefault(concealingModule + '/' + package, set()).add(deployedModule.name)

            pathToDep = {p.output_dir() if p.isJavaProject() else p.path: p for p in mx.dependencies() if p.isJavaProject() or p.isJARDistribution()}
            for classpathEntry in cp:
                # Export concealed packages used by the class path entry
                _add_exports_for_concealed_packages(classpathEntry, pathToDep, addedExports, 'ALL-UNNAMED', deployedModules)

                for deployedModule in deployedModules:
                    assert deployedModule.dist.path != classpathEntry, deployedModule.dist.path + ' should no longer be on the class path'
                    # Ensure the class path entry does not define packages already defined by the module.
                    # Packages definitions cannot be split between modules.
                    classpathEntryPackages = frozenset(_defined_packages(classpathEntry))
                    intersection = classpathEntryPackages.intersection(deployedModule.packages)
                    if intersection:
                        mx.abort(classpathEntry + ' cannot extend package(s) defined in the module ' + deployedModule.name + ': ' + ', '.join(intersection))

            vmArgs.extend(['--add-exports=' + export + '=' + ','.join(sorted(targets)) for export, targets in addedExports.iteritems()])

    if isJDK8:
        # Run the VM in a mode where application/test classes can
        # access JVMCI loaded classes.
        vmArgs.append('-XX:-UseJVMCIClassLoader')

    return (vmArgs, mainClass, mainClassArgs)
Exemplo n.º 11
0
def _parseVmArgs(args, addDefaultArgs=True):
    args = mx.expand_project_in_args(args, insitu=False)

    argsPrefix = []
    jacocoArgs = mx_gate.get_jacoco_agent_args()
    if jacocoArgs:
        argsPrefix.extend(jacocoArgs)

    # add default graal.options.file
    options_file = join(mx.primary_suite().dir, 'graal.options')
    if exists(options_file):
        argsPrefix.append('-Dgraal.options.file=' + options_file)

    if isJDK8:
        argsPrefix.append('-Djvmci.class.path.append=' +
                          os.pathsep.join((e.get_path()
                                           for e in _jvmci_classpath)))
        argsPrefix.append('-Xbootclasspath/a:' + os.pathsep.join(
            [dep.classpath_repr() for dep in _bootclasspath_appends]))
    else:
        deployedDists = [entry.dist() for entry in _jvmci_classpath] + \
                        [e for e in _bootclasspath_appends if e.isJARDistribution()]
        deployedModules = [as_java_module(dist, jdk) for dist in deployedDists]

        # Set or update module path to include Graal and its dependencies as modules
        jdkModuleNames = frozenset([m.name for m in jdk.get_modules()])
        graalModulepath = []
        graalUpgrademodulepath = []

        def _addToModulepath(modules):
            for m in modules:
                if m.jarpath:
                    modulepath = graalModulepath if m.name not in jdkModuleNames else graalUpgrademodulepath
                    if m not in modulepath:
                        modulepath.append(m)

        for deployedModule in deployedModules:
            _addToModulepath(deployedModule.modulepath)
            _addToModulepath([deployedModule])

        # Extend or set --module-path argument
        mpUpdated = False
        for mpIndex in range(len(args)):
            assert not args[mpIndex].startswith('--upgrade-module-path')
            if args[mpIndex] == '--module-path':
                assert mpIndex + 1 < len(args), 'VM option ' + args[
                    mpIndex] + ' requires an argument'
                args[mpIndex + 1] = os.pathsep.join(
                    _uniqify(args[mpIndex + 1].split(os.pathsep) +
                             [m.jarpath for m in graalModulepath]))
                mpUpdated = True
                break
            elif args[mpIndex].startswith('--module-path='):
                mp = args[mpIndex][len('--module-path='):]
                args[mpIndex] = '--module-path=' + os.pathsep.join(
                    _uniqify(
                        mp.split(os.pathsep) +
                        [m.jarpath for m in graalModulepath]))
                mpUpdated = True
                break
        if not mpUpdated:
            argsPrefix.append(
                '--module-path=' +
                os.pathsep.join([m.jarpath for m in graalModulepath]))

        if graalUpgrademodulepath:
            argsPrefix.append(
                '--upgrade-module-path=' +
                os.pathsep.join([m.jarpath for m in graalUpgrademodulepath]))

    if '-version' in args:
        ignoredArgs = args[args.index('-version') + 1:]
        if len(ignoredArgs) > 0:
            mx.log(
                "Warning: The following options will be ignored by the VM because they come after the '-version' argument: "
                + ' '.join(ignoredArgs))

    return jdk.processArgs(argsPrefix + args, addDefaultArgs=addDefaultArgs)
Exemplo n.º 12
0
def _parseVmArgs(args, addDefaultArgs=True):
    args = mx.expand_project_in_args(args, insitu=False)

    argsPrefix = []
    jacocoArgs = mx_gate.get_jacoco_agent_args()
    if jacocoArgs:
        argsPrefix.extend(jacocoArgs)

    # add default graal.options.file
    options_file = join(mx.primary_suite().dir, 'graal.options')
    if exists(options_file):
        argsPrefix.append('-Dgraal.options.file=' + options_file)

    if isJDK8:
        argsPrefix.append('-Djvmci.class.path.append=' +
                          os.pathsep.join((e.get_path()
                                           for e in _jvmci_classpath)))
        argsPrefix.append('-Xbootclasspath/a:' + os.pathsep.join(
            [dep.classpath_repr() for dep in _bootclasspath_appends]))
    else:
        deployedDists = [entry.dist() for entry in _jvmci_classpath] + \
                        [e for e in _bootclasspath_appends if e.isJARDistribution()]
        deployedModules = [as_java_module(dist, jdk) for dist in deployedDists]

        # Set or update module path to include Graal and its dependencies as modules
        graalModulepath = []
        for deployedModule in deployedModules:
            graalModulepath.extend([
                jmd.jarpath for jmd in deployedModule.modulepath if jmd.jarpath
            ])
            graalModulepath.append(deployedModule.jarpath)
        graalModulepath = _uniqify(graalModulepath)

        # Update added exports to include concealed JDK packages required by Graal
        addedExports = {}
        args = _extract_added_exports(args, addedExports)
        for deployedModule in deployedModules:
            for concealingModule, packages in deployedModule.concealedRequires.iteritems(
            ):
                # No need to explicitly export JVMCI - it's exported via reflection
                if concealingModule != 'jdk.vm.ci':
                    for package in packages:
                        addedExports.setdefault(
                            concealingModule + '/' + package,
                            set()).add(deployedModule.name)

        for export, targets in addedExports.iteritems():
            argsPrefix.append('--add-exports=' + export + '=' +
                              ','.join(sorted(targets)))

        # Extend or set --module-path argument
        mpUpdated = False
        for mpIndex in range(len(args)):
            if args[mpIndex] == '--module-path':
                assert mpIndex + 1 < len(args), 'VM option ' + args[
                    mpIndex] + ' requires an argument'
                args[mpIndex + 1] = os.pathsep.join(
                    _uniqify(args[mpIndex + 1].split(os.pathsep) +
                             graalModulepath))
                mpUpdated = True
                break
            elif args[mpIndex].startswith('--module-path='):
                mp = args[mpIndex][len('--module-path='):]
                args[mpIndex] = '--module-path=' + os.pathsep.join(
                    _uniqify(mp.split(os.pathsep) + graalModulepath))
                mpUpdated = True
                break
        if not mpUpdated:
            argsPrefix.append('--module-path=' +
                              os.pathsep.join(graalModulepath))

    if '-version' in args:
        ignoredArgs = args[args.index('-version') + 1:]
        if len(ignoredArgs) > 0:
            mx.log(
                "Warning: The following options will be ignored by the VM because they come after the '-version' argument: "
                + ' '.join(ignoredArgs))

    return jdk.processArgs(argsPrefix + args, addDefaultArgs=addDefaultArgs)
Exemplo n.º 13
0
def _unittest_config_participant(config):
    vmArgs, mainClass, mainClassArgs = config
    cpIndex, cp = mx.find_classpath_arg(vmArgs)
    if cp:
        cp = _uniqify(cp.split(os.pathsep))
        if isJDK8:
            # Remove entries from class path that are in Graal or on the boot class path
            redundantClasspathEntries = set()
            for dist in [entry.dist() for entry in _jvmci_classpath]:
                redundantClasspathEntries.update(
                    (d.output_dir() for d in dist.archived_deps()
                     if d.isJavaProject()))
                redundantClasspathEntries.add(dist.path)
            cp = os.pathsep.join(
                [e for e in cp if e not in redundantClasspathEntries])
            vmArgs[cpIndex] = cp
        else:
            deployedModules = []
            redundantClasspathEntries = set()
            for dist in [entry.dist() for entry in _jvmci_classpath
                         ] + _bootclasspath_appends:
                deployedModule = as_java_module(dist, jdk)
                deployedModules.append(deployedModule)
                redundantClasspathEntries.update(
                    mx.classpath(dist, preferProjects=False,
                                 jdk=jdk).split(os.pathsep))
                redundantClasspathEntries.update(
                    mx.classpath(dist, preferProjects=True,
                                 jdk=jdk).split(os.pathsep))
                if hasattr(dist, 'overlaps'):
                    for o in dist.overlaps:
                        path = mx.distribution(o).classpath_repr()
                        if path:
                            redundantClasspathEntries.add(path)

            # Remove entries from the class path that are in the deployed modules
            cp = [
                classpathEntry for classpathEntry in cp
                if classpathEntry not in redundantClasspathEntries
            ]
            vmArgs[cpIndex] = os.pathsep.join(cp)

            # Junit libraries are made into automatic modules so that they are visible to tests
            # patched into modules. These automatic modules must be declared to be read by
            # Graal which means they must also be made root modules (i.e., ``--add-modules``)
            # since ``--add-reads`` can only be applied to root modules.
            junitCp = [
                e.classpath_repr() for e in mx.classpath_entries(['JUNIT'])
            ]
            junitModules = [_automatic_module_name(e) for e in junitCp]
            vmArgs.append('--module-path=' + os.pathsep.join(junitCp))
            vmArgs.append('--add-modules=' +
                          ','.join(junitModules +
                                   [m.name for m in deployedModules]))
            for deployedModule in deployedModules:
                vmArgs.append('--add-reads=' + deployedModule.name + '=' +
                              ','.join(junitModules))

            # Explicitly export concealed JVMCI packages required by Graal. Even though
            # normally exported via jdk.vm.ci.services.Services.exportJVMCITo(), the
            # Junit harness wants to access JVMCI classes (e.g., when loading classes
            # to find test methods) without going through that entry point.
            addedExports = {}
            for deployedModule in deployedModules:
                for concealingModule, packages in deployedModule.concealedRequires.iteritems(
                ):
                    if concealingModule == 'jdk.vm.ci':
                        for package in packages:
                            addedExports.setdefault(
                                concealingModule + '/' + package,
                                set()).add(deployedModule.name)

            pathToDep = {
                p.output_dir() if p.isJavaProject() else p.path: p
                for p in mx.dependencies()
                if p.isJavaProject() or p.isJARDistribution()
            }
            for classpathEntry in cp:
                # Export concealed packages used by the class path entry
                _add_exports_for_concealed_packages(classpathEntry, pathToDep,
                                                    addedExports,
                                                    'ALL-UNNAMED',
                                                    deployedModules)

                for deployedModule in deployedModules:
                    assert deployedModule.dist.path != classpathEntry, deployedModule.dist.path + ' should no longer be on the class path'
                    # Ensure the class path entry does not define packages already defined by the module.
                    # Packages definitions cannot be split between modules.
                    classpathEntryPackages = frozenset(
                        _defined_packages(classpathEntry))
                    intersection = classpathEntryPackages.intersection(
                        deployedModule.packages)
                    if intersection:
                        mx.abort(
                            classpathEntry +
                            ' cannot extend package(s) defined in the module '
                            + deployedModule.name + ': ' +
                            ', '.join(intersection))

            vmArgs.extend([
                '--add-exports=' + export + '=' + ','.join(sorted(targets))
                for export, targets in addedExports.iteritems()
            ])

    if isJDK8:
        # Run the VM in a mode where application/test classes can
        # access JVMCI loaded classes.
        vmArgs.append('-XX:-UseJVMCIClassLoader')

    return (vmArgs, mainClass, mainClassArgs)
Exemplo n.º 14
0
def jlink_new_jdk(jdk, dst_jdk_dir, module_dists, ignore_dists,
                  root_module_names=None,
                  missing_export_target_action='create',
                  with_source=lambda x: True,
                  vendor_info=None,
                  dedup_legal_notices=True,
                  use_upgrade_module_path=False):
    """
    Uses jlink from `jdk` to create a new JDK image in `dst_jdk_dir` with `module_dists` and
    their dependencies added to the JDK image, replacing any existing modules of the same name.

    :param JDKConfig jdk: source JDK
    :param str dst_jdk_dir: path to use for the jlink --output option
    :param list module_dists: list of distributions defining modules
    :param list ignore_dists: list of distributions that should be ignored for missing_export_target_action
    :param list root_module_names: list of strings naming the module root set for the new JDK image.
                     The named modules must either be in `module_dists` or in `jdk`. If None, then
                     the root set will be all the modules in ``module_dists` and `jdk`.
    :param str missing_export_target_action: the action to perform for a qualified export target that
                     is not present in `module_dists` and does not have a hash stored in java.base.
                     The choices are:
                       "create" - an empty module is created
                        "error" - raise an error
                           None - do nothing
    :param lambda with_source: returns True if the sources of a module distribution must be included in the new JDK
    :param dict vendor_info: values for the jlink vendor options added by JDK-8232080
    :param bool use_upgrade_module_path: if True, then instead of linking `module_dists` into the image, resolve
                     them via --upgrade-module-path at image runtime
    :return bool: False if use_upgrade_module_path == True and the existing image is up to date otherwise True
    """
    assert callable(with_source)

    if jdk.javaCompliance < '9':
        mx.abort('Cannot derive a new JDK from ' + jdk.home + ' with jlink since it is not JDK 9 or later')

    exploded_java_base_module = join(jdk.home, 'modules', 'java.base')
    if exists(exploded_java_base_module):
        mx.abort('Cannot derive a new JDK from ' + jdk.home + ' since it appears to be a developer build with exploded modules')

    jimage = join(jdk.home, 'lib', 'modules')
    jmods_dir = join(jdk.home, 'jmods')
    if not isfile(jimage):
        mx.abort('Cannot derive a new JDK from ' + jdk.home + ' since ' + jimage + ' is missing or is not an ordinary file')
    if not isdir(jmods_dir):
        mx.abort('Cannot derive a new JDK from ' + jdk.home + ' since ' + jmods_dir + ' is missing or is not a directory')

    # Exclude jdk.aot due to GR-10545 and JDK-8255616
    jdk_modules = {jmd.name: jmd for jmd in jdk.get_modules() if jmd.name != 'jdk.aot'}
    modules = [as_java_module(dist, jdk) for dist in module_dists]
    module_names = frozenset((m.name for m in modules))
    all_module_names = frozenset(list(jdk_modules.keys())) | module_names

    # Read hashes stored in java.base (the only module in the JDK where hashes are stored)
    hashes = _read_java_base_hashes(jdk)

    build_dir = mx.ensure_dir_exists(join(dst_jdk_dir + ".build"))

    # Directory under dst_jdk_dir for artifacts related to use_upgrade_module_path
    upgrade_dir = join(dst_jdk_dir, 'upgrade_modules_support')

    # Map from JavaModuleDescriptors to post-jlink jar location.
    synthetic_modules = OrderedDict()
    try:
        ignore_module_names = set(mx_javamodules.get_module_name(mx.dependency(ignore_dist)) for ignore_dist in ignore_dists)
        # Synthesize modules for targets of qualified exports that are not present in `modules`.
        # Without this, runtime module resolution will fail due to missing modules.
        target_requires = {}
        for jmd in modules:
            for targets in jmd.exports.values():
                for target in targets:
                    if target not in all_module_names and target not in ignore_module_names and target not in hashes:
                        target_requires.setdefault(target, set()).add(jmd.name)
        if target_requires and missing_export_target_action is not None:
            if missing_export_target_action == 'error':
                mx.abort('Target(s) of qualified exports cannot be resolved: ' + '.'.join(target_requires.keys()))
            assert missing_export_target_action == 'create', 'invalid value for missing_export_target_action: ' + str(missing_export_target_action)

            for name, requires in sorted(target_requires.items()):
                module_jar = join(build_dir, name + '.jar')
                jmd = JavaModuleDescriptor(name, {}, requires={module: [] for module in requires}, uses=set(), provides={}, jarpath=module_jar)
                module_build_dir = mx.ensure_dir_exists(join(build_dir, name))
                module_info = jmd.as_module_info()
                module_info_java = join(module_build_dir, 'module-info.java')
                module_info_class = join(module_build_dir, 'module-info.class')
                dst_module_jar = join(upgrade_dir, name + '.jar')
                synthetic_modules[jmd] = dst_module_jar
                if use_upgrade_module_path and exists(dst_module_jar):
                    with ZipFile(dst_module_jar, 'r') as zf:
                        previous_module_info = zf.read('module-info.java').decode()
                    if previous_module_info == module_info:
                        mx.logv('[Reusing synthetic module {}]'.format(name))
                        os.rename(dst_module_jar, module_jar)
                        continue
                    mx.logv('[Rebuilding synthetic module {} as module descriptor changed]'.format(name))

                with open(module_info_java, 'w') as fp:
                    fp.write(module_info)
                mx.run([jdk.javac, '-d', module_build_dir,
                        '--limit-modules=java.base,' + ','.join(jmd.requires.keys()),
                        '--module-path=' + os.pathsep.join((m.jarpath for m in modules)),
                        module_info_java])
                with ZipFile(module_jar, 'w') as zf:
                    zf.write(module_info_java, 'module-info.java')
                    zf.write(module_info_class, 'module-info.class')
                if exists(jmd.get_jmod_path()):
                    os.remove(jmd.get_jmod_path())
                if not use_upgrade_module_path:
                    mx.run([jdk.javac.replace('javac', 'jmod'), 'create', '--class-path=' + module_build_dir, jmd.get_jmod_path()])

            modules.extend(synthetic_modules.keys())
            module_names = frozenset((m.name for m in modules))
            all_module_names = frozenset(list(jdk_modules.keys())) | module_names

        # Edit lib/security/default.policy in java.base
        patched_java_base = _patch_default_security_policy(build_dir, jmods_dir, dst_jdk_dir)

        # Now build the new JDK image with jlink
        jlink = [jdk.javac.replace('javac', 'jlink')]

        if jdk_enables_jvmci_by_default(jdk):
            # On JDK 9+, +EnableJVMCI forces jdk.internal.vm.ci to be in the root set
            jlink += ['-J-XX:-EnableJVMCI', '-J-XX:-UseJVMCICompiler']

        jlink.append('--add-modules=' + ','.join(_get_image_root_modules(root_module_names, module_names, jdk_modules.keys(), use_upgrade_module_path)))

        module_path = patched_java_base + os.pathsep + jmods_dir
        if modules and not use_upgrade_module_path:
            module_path = os.pathsep.join((m.get_jmod_path(respect_stripping=True) for m in modules)) + os.pathsep + module_path
        jlink.append('--module-path=' + module_path)
        jlink.append('--output=' + dst_jdk_dir)

        # These options are derived from how OpenJDK runs jlink to produce the final runtime image.
        jlink.extend(['-J-XX:+UseSerialGC', '-J-Xms32M', '-J-Xmx512M', '-J-XX:TieredStopAtLevel=1'])
        jlink.append('-J-Dlink.debug=true')
        if dedup_legal_notices:
            jlink.append('--dedup-legal-notices=error-if-not-same-content')
        jlink.append('--keep-packaged-modules=' + join(dst_jdk_dir, 'jmods'))

        vm_options_path = join(upgrade_dir, 'vm_options')
        vm_options = _get_image_vm_options(jdk, use_upgrade_module_path, modules, synthetic_modules)
        if vm_options:
            jlink.append('--add-options=' + ' '.join(vm_options))

        if jdk_has_new_jlink_options(jdk) and vendor_info is not None:
            for name, value in vendor_info.items():
                jlink.append('--' + name + '=' + value)

        release_file = join(jdk.home, 'release')
        if isfile(release_file):
            jlink.append('--release-info=' + release_file)

        if exists(dst_jdk_dir):
            if use_upgrade_module_path and _vm_options_match(vm_options, vm_options_path):
                mx.logv('[Existing JDK image {} is up to date]'.format(dst_jdk_dir))
                return False
            mx.rmtree(dst_jdk_dir)

        # TODO: investigate the options below used by OpenJDK to see if they should be used:
        # --order-resources: specifies order of resources in generated lib/modules file.
        #       This is apparently not so important if a CDS archive is available.
        # --generate-jli-classes: pre-generates a set of java.lang.invoke classes.
        #       See https://github.com/openjdk/jdk/blob/master/make/GenerateLinkOptData.gmk
        mx.logv('[Creating JDK image in {}]'.format(dst_jdk_dir))
        mx.run(jlink)

        if use_upgrade_module_path:
            # Move synthetic upgrade modules into final location
            for jmd, jarpath in synthetic_modules.items():
                mx.ensure_dir_exists(dirname(jarpath))
                os.rename(jmd.jarpath, jarpath)
            # Persist VM options cooked into image to be able to skip a subsequent
            # jlink execution if the options do not change.
            with open(vm_options_path, 'w') as fp:
                fp.write(os.linesep.join(vm_options))

        # Create src.zip in new JDK image
        _copy_src_zip(jdk.home, dst_jdk_dir, modules, lambda jmd: not use_upgrade_module_path and with_source(jmd.dist))

        mx.logv('[Copying static libraries]')
        lib_directory = join(jdk.home, 'lib', 'static')
        if exists(lib_directory):
            dst_lib_directory = join(dst_jdk_dir, 'lib', 'static')
            try:
                mx.copytree(lib_directory, dst_lib_directory)
            except shutil.Error as e:
                # On AArch64, there can be a problem in the copystat part
                # of copytree which occurs after file and directory copying
                # has successfully completed. Since the metadata doesn't
                # matter in this case, just ensure that the content was copied.
                for root, _, lib_files in os.walk(lib_directory):
                    relative_root = os.path.relpath(root, dst_lib_directory)
                    for lib in lib_files:
                        src_lib_path = join(root, lib)
                        dst_lib_path = join(dst_lib_directory, relative_root, lib)
                        if not exists(dst_lib_path):
                            mx.abort('Error copying static libraries: {} missing in {}{}Original copytree error: {}'.format(
                                join(relative_root, lib), dst_lib_directory, os.linesep, e))
                        src_lib_hash = mx.sha1OfFile(src_lib_path)
                        dst_lib_hash = mx.sha1OfFile(dst_lib_path)
                        if src_lib_hash != dst_lib_hash:
                            mx.abort('Error copying static libraries: {} (hash={}) and {} (hash={}) differ{}Original copytree error: {}'.format(
                                src_lib_path, src_lib_hash,
                                dst_lib_path, dst_lib_hash,
                                os.linesep, e))
        # Allow older JDK versions to work
        else:
            lib_prefix = mx.add_lib_prefix('')
            lib_suffix = mx.add_static_lib_suffix('')
            lib_directory = join(jdk.home, 'lib')
            dst_lib_directory = join(dst_jdk_dir, 'lib')
            for f in os.listdir(lib_directory):
                if f.startswith(lib_prefix) and f.endswith(lib_suffix):
                    lib_path = join(lib_directory, f)
                    if isfile(lib_path):
                        shutil.copy2(lib_path, dst_lib_directory)
    finally:
        if not mx.get_opts().verbose:
            # Preserve build directory so that javac command can be re-executed
            # by cutting and pasting verbose output.
            shutil.rmtree(build_dir)

    if not use_upgrade_module_path:
        # Create CDS archive (https://openjdk.java.net/jeps/341).
        out = mx.OutputCapture()
        mx.logv('[Creating CDS shared archive]')
        if mx.run([mx.exe_suffix(join(dst_jdk_dir, 'bin', 'java')), '-Xshare:dump', '-Xmx128M', '-Xms128M'], out=out, err=out, nonZeroIsFatal=False) != 0:
            mx.log(out.data)
            mx.abort('Error generating CDS shared archive')
    else:
        # -Xshare is incompatible with --upgrade-module-path
        pass
    return True