def _path_args(depNames=None): """ Gets the VM args for putting the dependencies named in `depNames` on the class path and module path (if running on JDK9 or later). :param names: a Dependency, str or list containing Dependency/str objects. If None, then all registered dependencies are used. """ jdk = mx.get_jdk() if jdk.javaCompliance >= '1.9': modules = [ as_java_module(dist, jdk) for dist in _suite.dists if get_java_module_info(dist) ] if modules: # Partition resources between the class path and module path modulepath = [] classpath = [] cpEntryToModule = {m.dist.path: m for m in modules} for e in mx.classpath(depNames).split(os.pathsep): if cpEntryToModule.has_key(e): modulepath.append(cpEntryToModule[e].jarpath) else: classpath.append(e) # The Truffle modules must be eagerly loaded as they could be referenced from # the main class hence the --add-modules argument return [ '--add-modules=' + ','.join([m.name for m in modules]), '--module-path=' + os.pathsep.join(modulepath), '-cp', os.pathsep.join(classpath) ] return ['-cp', mx.classpath(depNames)]
def _path_args(depNames=None): """ Gets the VM args for putting the dependencies named in `depNames` on the class path and module path (if running on JDK9 or later). :param names: a Dependency, str or list containing Dependency/str objects. If None, then all registered dependencies are used. """ jdk = mx.get_jdk() if jdk.javaCompliance >= '1.9': modules = [as_java_module(dist, jdk) for dist in _suite.dists if get_java_module_info(dist)] if modules: # Partition resources between the class path and module path modulepath = [] classpath = [] cpEntryToModule = {m.dist.path : m for m in modules} for e in mx.classpath(depNames).split(os.pathsep): if cpEntryToModule.has_key(e): modulepath.append(cpEntryToModule[e].jarpath) else: classpath.append(e) # The Truffle modules must be eagerly loaded as they could be referenced from # the main class hence the --add-modules argument return ['--add-modules=' + ','.join([m.name for m in modules]), '--module-path=' + os.pathsep.join(modulepath), '-cp', os.pathsep.join(classpath)] return ['-cp', mx.classpath(depNames)]
def get_module_jar(self): """ Gets the path to the module jar file. :rtype: str """ return as_java_module(self.dist(), jdk).jarpath
def jlink_new_jdk(jdk, dst_jdk_dir, module_dists, root_module_names=None, missing_export_target_action='create', with_source=lambda x: True): """ Uses jlink from `jdk` to create a new JDK image in `dst_jdk_dir` with `module_dists` and their dependencies added to the JDK image, replacing any existing modules of the same name. :param JDKConfig jdk: source JDK :param str dst_jdk_dir: path to use for the jlink --output option :param list module_dists: list of distributions defining modules :param list root_module_names: list of strings naming the module root set for the new JDK image. The named modules must either be in `module_dists` or in `jdk`. If None, then the root set will be all the modules in ``module_dists` and `jdk`. :param str missing_export_target_action: the action to perform for a qualifed export target that is not present in `module_dists` and does not have a hash stored in java.base. The choices are: "create" - an empty module is created "error" - raise an error None - do nothing :param lambda with_source: returns True if the sources of a module distribution must be included in the new JDK """ assert callable(with_source) if jdk.javaCompliance < '9': mx.abort('Cannot derive a new JDK from ' + jdk.home + ' with jlink since it is not JDK 9 or later') exploded_java_base_module = join(jdk.home, 'modules', 'java.base') if exists(exploded_java_base_module): mx.abort('Cannot derive a new JDK from ' + jdk.home + ' since it appears to be a developer build with exploded modules') jimage = join(jdk.home, 'lib', 'modules') jmods_dir = join(jdk.home, 'jmods') if not isfile(jimage): mx.abort('Cannot derive a new JDK from ' + jdk.home + ' since ' + jimage + ' is missing or is not an ordinary file') if not isdir(jmods_dir): mx.abort('Cannot derive a new JDK from ' + jdk.home + ' since ' + jmods_dir + ' is missing or is not a directory') jdk_modules = {jmd.name : jmd for jmd in jdk.get_modules()} modules = [as_java_module(dist, jdk) for dist in module_dists] all_module_names = frozenset(list(jdk_modules.keys()) + [m.name for m in modules]) # Read hashes stored in java.base (the only module in the JDK where hashes are stored) out = mx.LinesOutputCapture() mx.run([jdk.exe_path('jmod'), 'describe', jdk_modules['java.base'].get_jmod_path()], out=out) lines = out.lines hashes = {} for line in lines: if line.startswith('hashes'): parts = line.split() assert len(parts) == 4, 'expected hashes line to have 4 fields, got {} fields: {}'.format(len(parts), line) _, module_name, algorithm, hash_value = parts hashes[module_name] = (algorithm, hash_value) build_dir = mx.ensure_dir_exists(join(dst_jdk_dir + ".build")) try: # Handle targets of qualified exports that are not present in `modules` target_requires = {} for jmd in modules: for targets in jmd.exports.values(): for target in targets: if target not in all_module_names and target not in hashes: target_requires.setdefault(target, set()).add(jmd.name) if target_requires and missing_export_target_action is not None: if missing_export_target_action == 'error': mx.abort('Target(s) of qualified exports cannot be resolved: ' + '.'.join(target_requires.keys())) assert missing_export_target_action == 'create', 'invalid value for missing_export_target_action: ' + str(missing_export_target_action) extra_modules = [] for name, requires in target_requires.items(): module_jar = join(build_dir, name + '.jar') jmd = JavaModuleDescriptor(name, {}, requires={module : [] for module in requires}, uses=set(), provides={}, jarpath=module_jar) extra_modules.append(jmd) module_build_dir = mx.ensure_dir_exists(join(build_dir, name)) module_info_java = join(module_build_dir, 'module-info.java') module_info_class = join(module_build_dir, 'module-info.class') with open(module_info_java, 'w') as fp: print(jmd.as_module_info(), file=fp) mx.run([jdk.javac, '-d', module_build_dir, \ '--limit-modules=java.base,' + ','.join(jmd.requires.keys()), \ '--module-path=' + os.pathsep.join((m.jarpath for m in modules)), \ module_info_java]) with ZipFile(module_jar, 'w') as zf: zf.write(module_info_class, basename(module_info_class)) if exists(jmd.get_jmod_path()): os.remove(jmd.get_jmod_path()) mx.run([jdk.javac.replace('javac', 'jmod'), 'create', '--class-path=' + module_build_dir, jmd.get_jmod_path()]) modules.extend(extra_modules) all_module_names = frozenset(list(jdk_modules.keys()) + [m.name for m in modules]) # Extract src.zip from source JDK jdk_src_zip = join(jdk.home, 'lib', 'src.zip') dst_src_zip_contents = {} if isfile(jdk_src_zip): mx.logv('[Extracting ' + jdk_src_zip + ']') with ZipFile(jdk_src_zip, 'r') as zf: for name in zf.namelist(): if not name.endswith('/'): dst_src_zip_contents[name] = zf.read(name) else: mx.warn("'{}' does not exist or is not a file".format(jdk_src_zip)) for jmd in modules: # Remove existing sources for all the modules that we include dst_src_zip_contents = {key : dst_src_zip_contents[key] for key in dst_src_zip_contents if not key.startswith(jmd.name)} if with_source(jmd.dist): # Add the sources that we can share. # Extract module sources jmd_src_zip = jmd.jarpath[0:-len('.jar')] + '.src.zip' if isfile(jmd_src_zip): mx.logv('[Extracting ' + jmd_src_zip + ']') with ZipFile(jmd_src_zip, 'r') as zf: for name in zf.namelist(): if not name.endswith('/'): dst_src_zip_contents[jmd.name + '/' + name] = zf.read(name) # Add module-info.java to sources dst_src_zip_contents[jmd.name + '/module-info.java'] = jmd.as_module_info(extras_as_comments=False) # Now build the new JDK image with jlink jlink = [jdk.javac.replace('javac', 'jlink')] if jdk_enables_jvmci_by_default(jdk): # On JDK 9+, +EnableJVMCI forces jdk.internal.vm.ci to be in the root set jlink.append('-J-XX:-EnableJVMCI') if root_module_names is not None: missing = frozenset(root_module_names) - all_module_names if missing: mx.abort('Invalid module(s): {}.\nAvailable modules: {}'.format(','.join(missing), ','.join(sorted(all_module_names)))) jlink.append('--add-modules=' + ','.join(root_module_names)) else: jlink.append('--add-modules=' + ','.join(sorted(all_module_names))) module_path = jmods_dir if modules: module_path = os.pathsep.join((m.get_jmod_path(respect_stripping=True) for m in modules)) + os.pathsep + module_path jlink.append('--module-path=' + module_path) jlink.append('--output=' + dst_jdk_dir) # These options are inspired by how OpenJDK runs jlink to produce the final runtime image. jlink.extend(['-J-XX:+UseSerialGC', '-J-Xms32M', '-J-Xmx512M', '-J-XX:TieredStopAtLevel=1']) jlink.append('-J-Dlink.debug=true') jlink.append('--dedup-legal-notices=error-if-not-same-content') jlink.append('--keep-packaged-modules=' + join(dst_jdk_dir, 'jmods')) # TODO: investigate the options below used by OpenJDK to see if they should be used: # --release-info: this allow extra properties to be written to the <jdk>/release file # --order-resources: specifies order of resources in generated lib/modules file. # This is apparently not so important if a CDS archive is available. # --generate-jli-classes: pre-generates a set of java.lang.invoke classes. # See https://github.com/openjdk/jdk/blob/master/make/GenerateLinkOptData.gmk mx.logv('[Creating JDK image]') mx.run(jlink) dst_src_zip = join(dst_jdk_dir, 'lib', 'src.zip') mx.logv('[Creating ' + dst_src_zip + ']') with ZipFile(dst_src_zip, 'w', compression=ZIP_DEFLATED, allowZip64=True) as zf: for name, contents in sorted(dst_src_zip_contents.items()): zf.writestr(name, contents) mx.logv('[Copying static libraries]') lib_prefix = mx.add_lib_prefix('') lib_suffix = '.lib' if mx.is_windows() else '.a' lib_directory = join(jdk.home, 'lib') dst_lib_directory = join(dst_jdk_dir, 'lib') for f in os.listdir(lib_directory): if f.startswith(lib_prefix) and f.endswith(lib_suffix): lib_path = join(lib_directory, f) if isfile(lib_path): shutil.copy2(lib_path, dst_lib_directory) # Build the list of modules whose classes might have annotations # to be processed by native-image (GR-15192). with open(join(dst_jdk_dir, 'lib', 'native-image-modules.list'), 'w') as fp: print('# Modules whose classes might have annotations processed by native-image', file=fp) for m in modules: print(m.name, file=fp) finally: if not mx.get_opts().verbose: # Preserve build directory so that javac command can be re-executed # by cutting and pasting verbose output. shutil.rmtree(build_dir) # Create CDS archive (https://openjdk.java.net/jeps/341). out = mx.OutputCapture() mx.logv('[Creating CDS shared archive]') if mx.run([mx.exe_suffix(join(dst_jdk_dir, 'bin', 'java')), '-Xshare:dump', '-Xmx128M', '-Xms128M'], out=out, err=out, nonZeroIsFatal=False) != 0: mx.log(out.data) mx.abort('Error generating CDS shared archive')
def _parseVmArgs(args, addDefaultArgs=True): args = mx.expand_project_in_args(args, insitu=False) argsPrefix = [] jacocoArgs = mx_gate.get_jacoco_agent_args() if jacocoArgs: argsPrefix.extend(jacocoArgs) # Check for -G: options def checkGOption(arg): if arg.startswith('-G:+'): if '=' in arg: mx.abort('Mixing + and = in -G: option specification: ' + arg) translation = '-Dgraal.' + arg[len('-G:+'):] + '=true' elif arg.startswith('-G:-'): if '=' in arg: mx.abort('Mixing - and = in -G: option specification: ' + arg) translation = '-Dgraal.' + arg[len('-G:+'):] + '=false' elif arg.startswith('-G:'): if '=' not in arg: mx.abort('Missing "=" in non-boolean -G: option specification: ' + arg) translation = '-Dgraal.' + arg[len('-G:'):] else: return arg mx.warn('Support for -G options is deprecated and will soon be removed. Replace "' + arg + '" with "' + translation + '"') return translation # add default graal.options.file options_file = join(mx.primary_suite().dir, 'graal.options') if exists(options_file): argsPrefix.append('-Dgraal.options.file=' + options_file) args = [checkGOption(a) for a in args] if '-Dgraal.PrintFlags=true' in args and '-Xcomp' not in args: mx.warn('Using -Dgraal.PrintFlags=true may have no effect without -Xcomp as Graal initialization is lazy') if isJDK8: argsPrefix.append('-Djvmci.class.path.append=' + os.pathsep.join((e.get_path() for e in _jvmci_classpath))) argsPrefix.append('-Xbootclasspath/a:' + os.pathsep.join([dep.classpath_repr() for dep in _bootclasspath_appends])) else: deployedDists = [entry.dist() for entry in _jvmci_classpath] + \ [e for e in _bootclasspath_appends if e.isJARDistribution()] deployedModules = [as_java_module(dist, jdk) for dist in deployedDists] # Set or update module path to include Graal and its dependencies as modules graalModulepath = [] for deployedModule in deployedModules: graalModulepath.extend([jmd.jarpath for jmd in deployedModule.modulepath if jmd.jarpath]) graalModulepath.append(deployedModule.jarpath) graalModulepath = _uniqify(graalModulepath) # Update added exports to include concealed JDK packages required by Graal addedExports = {} args = _extract_added_exports(args, addedExports) for deployedModule in deployedModules: for concealingModule, packages in deployedModule.concealedRequires.iteritems(): # No need to explicitly export JVMCI - it's exported via reflection if concealingModule != 'jdk.vm.ci': for package in packages: addedExports.setdefault(concealingModule + '/' + package, set()).add(deployedModule.name) for export, targets in addedExports.iteritems(): argsPrefix.append('-XaddExports:' + export + '=' + ','.join(sorted(targets))) # Extend or set -modulepath argument mpUpdated = False for mpIndex in range(len(args)): if args[mpIndex] in ['-modulepath', '-mp']: assert mpIndex + 1 < len(args), 'VM option ' + args[mpIndex] + ' requires an argument' args[mpIndex + 1] = os.pathsep.join(_uniqify(args[mpIndex + 1].split(os.pathsep) + graalModulepath)) mpUpdated = True break if not mpUpdated: argsPrefix.append('-modulepath') argsPrefix.append(os.pathsep.join(graalModulepath)) # Set the JVMCI compiler to Graal argsPrefix.append('-Djvmci.Compiler=graal') if '-version' in args: ignoredArgs = args[args.index('-version') + 1:] if len(ignoredArgs) > 0: mx.log("Warning: The following options will be ignored by the VM because they come after the '-version' argument: " + ' '.join(ignoredArgs)) return jdk.processArgs(argsPrefix + args, addDefaultArgs=addDefaultArgs)
def _unittest_config_participant(config): vmArgs, mainClass, mainClassArgs = config cpIndex, cp = mx.find_classpath_arg(vmArgs) if cp: cp = _uniqify(cp.split(os.pathsep)) if isJDK8: # Remove entries from class path that are in Graal or on the boot class path redundantClasspathEntries = set() for dist in [entry.dist() for entry in _jvmci_classpath]: redundantClasspathEntries.update((d.output_dir() for d in dist.archived_deps() if d.isJavaProject())) redundantClasspathEntries.add(dist.path) cp = os.pathsep.join([e for e in cp if e not in redundantClasspathEntries]) vmArgs[cpIndex] = cp else: deployedModules = [] redundantClasspathEntries = set() for dist in [entry.dist() for entry in _jvmci_classpath] + _bootclasspath_appends: deployedModule = as_java_module(dist, jdk) deployedModules.append(deployedModule) redundantClasspathEntries.update(mx.classpath(dist, preferProjects=False, jdk=jdk).split(os.pathsep)) redundantClasspathEntries.update(mx.classpath(dist, preferProjects=True, jdk=jdk).split(os.pathsep)) # Remove entries from the class path that are in the deployed modules cp = [classpathEntry for classpathEntry in cp if classpathEntry not in redundantClasspathEntries] vmArgs[cpIndex] = os.pathsep.join(cp) # Junit libraries are made into automatic modules so that they are visible to tests # patched into modules. These automatic modules must be declared to be read by # Graal which means they must also be made root modules (i.e., ``-addmods``) # since ``-XaddReads`` can only be applied to root modules. junitCp = [e.classpath_repr() for e in mx.classpath_entries(['JUNIT'])] junitModules = [_automatic_module_name(e) for e in junitCp] vmArgs.extend(['-modulepath', os.pathsep.join(junitCp)]) vmArgs.extend(['-addmods', ','.join(junitModules + [m.name for m in deployedModules])]) for deployedModule in deployedModules: vmArgs.append('-XaddReads:' + deployedModule.name + '=' + ','.join(junitModules)) # Explicitly export concealed JVMCI packages required by Graal. Even though # normally exported via jdk.vm.ci.services.Services.exportJVMCITo(), the # Junit harness wants to access JVMCI classes (e.g., when loading classes # to find test methods) without going through that entry point. addedExports = {} for deployedModule in deployedModules: for concealingModule, packages in deployedModule.concealedRequires.iteritems(): if concealingModule == 'jdk.vm.ci': for package in packages: addedExports.setdefault(concealingModule + '/' + package, set()).add(deployedModule.name) patches = {} pathToProject = {p.output_dir() : p for p in mx.projects() if p.isJavaProject()} for classpathEntry in cp: # Export concealed packages used by the class path entry _add_exports_for_concealed_packages(classpathEntry, pathToProject, addedExports, 'ALL-UNNAMED', deployedModules) for deployedModule in deployedModules: assert deployedModule.dist.path != classpathEntry, deployedModule.dist.path + ' should no longer be on the class path' # Patch the class path entry into a module if it defines packages already defined by the module. # Packages definitions cannot be split between modules. classpathEntryPackages = frozenset(_defined_packages(classpathEntry)) if not classpathEntryPackages.isdisjoint(deployedModule.packages): patches.setdefault(deployedModule.name, []).append(classpathEntry) extraPackages = classpathEntryPackages - frozenset(deployedModule.exports.iterkeys()) if extraPackages: # From http://openjdk.java.net/jeps/261: # If a package found in a module definition on a patch path is not already exported # by that module then it will, still, not be exported. It can be exported explicitly # via either the reflection API or the -XaddExports option. for package in extraPackages: addedExports.setdefault(deployedModule.name + '/' + package, set()).update(junitModules + ['ALL-UNNAMED']) for moduleName, cpEntries in patches.iteritems(): vmArgs.append('-Xpatch:' + moduleName + '=' + os.pathsep.join(cpEntries)) vmArgs.extend(['-XaddExports:' + export + '=' + ','.join(sorted(targets)) for export, targets in addedExports.iteritems()]) if isJDK8: # Run the VM in a mode where application/test classes can # access JVMCI loaded classes. vmArgs.append('-XX:-UseJVMCIClassLoader') return (vmArgs, mainClass, mainClassArgs)
def _unittest_config_participant(config): vmArgs, mainClass, mainClassArgs = config cpIndex, cp = mx.find_classpath_arg(vmArgs) if cp: cp = _uniqify(cp.split(os.pathsep)) if isJDK8: # Remove entries from class path that are in Graal excluded = set() for entry in _jvmci_classpath: dist = entry.dist() excluded.update((d.output_dir() for d in dist.archived_deps() if d.isJavaProject())) excluded.add(dist.path) cp = os.pathsep.join([e for e in cp if e not in excluded]) vmArgs[cpIndex] = cp else: # Remove entries from class path that are in the Graal assert _graal_module_descriptor is not None module = as_java_module(_graal_module_descriptor.dist(), jdk) junitCp = [e.classpath_repr() for e in mx.classpath_entries(['JUNIT'])] excluded = frozenset([classpathEntry.classpath_repr() for classpathEntry in get_module_deps(module.dist)] + junitCp) cp = [classpathEntry for classpathEntry in cp if classpathEntry not in excluded] vmArgs[cpIndex] = os.pathsep.join(cp) # Junit libraries are made into automatic modules so that they are visible to tests # patched into Graal. These automatic modules must be declared to be read by # Graal which means they must also be made root modules (i.e., ``-addmods``) # since ``-XaddReads`` can only be applied to root modules. junitModules = [_automatic_module_name(e) for e in junitCp] vmArgs.extend(['-modulepath', os.pathsep.join(junitCp)]) vmArgs.extend(['-addmods', ','.join(junitModules)]) vmArgs.extend(['-XaddReads:' + module.name + '=' + ','.join(junitModules)]) # Explicitly export JVMCI to Graal addedExports = {} for concealingModule, packages in module.concealedRequires.iteritems(): if concealingModule == 'jdk.vm.ci': for package in packages: addedExports.setdefault(concealingModule + '/' + package, set()).add(module.name) patches = [] graalConcealedPackages = list(module.conceals) pathToProject = {p.output_dir() : p for p in mx.projects() if p.isJavaProject()} for classpathEntry in cp: # Export concealed JDK packages used by the class path entry _add_exports_for_concealed_packages(classpathEntry, pathToProject, addedExports, 'ALL-UNNAMED') # Patch the class path entry into Graal if it defines packages already defined by Graal. # Packages definitions cannot be split between modules. packages = frozenset(_defined_packages(classpathEntry)) if not packages.isdisjoint(module.packages): patches.append(classpathEntry) extraPackages = packages - module.packages if extraPackages: # From http://openjdk.java.net/jeps/261: # If a package found in a module definition on a patch path is not already exported # by that module then it will, still, not be exported. It can be exported explicitly # via either the reflection API or the -XaddExports option. graalConcealedPackages.extend(extraPackages) if patches: vmArgs.append('-Xpatch:' + module.name + '=' + os.pathsep.join(patches)) # Export all Graal packages to make them available to test classes for package in graalConcealedPackages: addedExports.setdefault(module.name + '/' + package, set()).update(junitModules + ['ALL-UNNAMED']) vmArgs.extend(['-XaddExports:' + export + '=' + ','.join(sorted(targets)) for export, targets in addedExports.iteritems()]) if isJDK8: # Run the VM in a mode where application/test classes can # access JVMCI loaded classes. vmArgs.append('-XX:-UseJVMCIClassLoader') return (vmArgs, mainClass, mainClassArgs)
def jlink_new_jdk(jdk, dst_jdk_dir, module_dists, root_module_names=None, missing_export_target_action='create', with_source=lambda x: True, vendor_info=None, dedup_legal_notices=True): """ Uses jlink from `jdk` to create a new JDK image in `dst_jdk_dir` with `module_dists` and their dependencies added to the JDK image, replacing any existing modules of the same name. :param JDKConfig jdk: source JDK :param str dst_jdk_dir: path to use for the jlink --output option :param list module_dists: list of distributions defining modules :param list root_module_names: list of strings naming the module root set for the new JDK image. The named modules must either be in `module_dists` or in `jdk`. If None, then the root set will be all the modules in ``module_dists` and `jdk`. :param str missing_export_target_action: the action to perform for a qualifed export target that is not present in `module_dists` and does not have a hash stored in java.base. The choices are: "create" - an empty module is created "error" - raise an error None - do nothing :param lambda with_source: returns True if the sources of a module distribution must be included in the new JDK :param dict vendor_info: values for the jlink vendor options added by JDK-8232080 """ assert callable(with_source) if jdk.javaCompliance < '9': mx.abort('Cannot derive a new JDK from ' + jdk.home + ' with jlink since it is not JDK 9 or later') exploded_java_base_module = join(jdk.home, 'modules', 'java.base') if exists(exploded_java_base_module): mx.abort('Cannot derive a new JDK from ' + jdk.home + ' since it appears to be a developer build with exploded modules') jimage = join(jdk.home, 'lib', 'modules') jmods_dir = join(jdk.home, 'jmods') if not isfile(jimage): mx.abort('Cannot derive a new JDK from ' + jdk.home + ' since ' + jimage + ' is missing or is not an ordinary file') if not isdir(jmods_dir): mx.abort('Cannot derive a new JDK from ' + jdk.home + ' since ' + jmods_dir + ' is missing or is not a directory') # Exclude jdk.aot due to GR-10545 and JDK-8255616 jdk_modules = {jmd.name: jmd for jmd in jdk.get_modules() if jmd.name != 'jdk.aot'} modules = [as_java_module(dist, jdk) for dist in module_dists] all_module_names = frozenset(list(jdk_modules.keys()) + [m.name for m in modules]) # Read hashes stored in java.base (the only module in the JDK where hashes are stored) out = mx.LinesOutputCapture() mx.run([jdk.exe_path('jmod'), 'describe', jdk_modules['java.base'].get_jmod_path()], out=out) lines = out.lines hashes = {} for line in lines: if line.startswith('hashes'): parts = line.split() assert len(parts) == 4, 'expected hashes line to have 4 fields, got {} fields: {}'.format(len(parts), line) _, module_name, algorithm, hash_value = parts hashes[module_name] = (algorithm, hash_value) build_dir = mx.ensure_dir_exists(join(dst_jdk_dir + ".build")) try: # Handle targets of qualified exports that are not present in `modules` target_requires = {} for jmd in modules: for targets in jmd.exports.values(): for target in targets: if target not in all_module_names and target not in hashes: target_requires.setdefault(target, set()).add(jmd.name) if target_requires and missing_export_target_action is not None: if missing_export_target_action == 'error': mx.abort('Target(s) of qualified exports cannot be resolved: ' + '.'.join(target_requires.keys())) assert missing_export_target_action == 'create', 'invalid value for missing_export_target_action: ' + str(missing_export_target_action) extra_modules = [] for name, requires in target_requires.items(): module_jar = join(build_dir, name + '.jar') jmd = JavaModuleDescriptor(name, {}, requires={module: [] for module in requires}, uses=set(), provides={}, jarpath=module_jar) extra_modules.append(jmd) module_build_dir = mx.ensure_dir_exists(join(build_dir, name)) module_info_java = join(module_build_dir, 'module-info.java') module_info_class = join(module_build_dir, 'module-info.class') with open(module_info_java, 'w') as fp: print(jmd.as_module_info(), file=fp) mx.run([jdk.javac, '-d', module_build_dir, '--limit-modules=java.base,' + ','.join(jmd.requires.keys()), '--module-path=' + os.pathsep.join((m.jarpath for m in modules)), module_info_java]) with ZipFile(module_jar, 'w') as zf: zf.write(module_info_class, basename(module_info_class)) if exists(jmd.get_jmod_path()): os.remove(jmd.get_jmod_path()) mx.run([jdk.javac.replace('javac', 'jmod'), 'create', '--class-path=' + module_build_dir, jmd.get_jmod_path()]) modules.extend(extra_modules) all_module_names = frozenset(list(jdk_modules.keys()) + [m.name for m in modules]) # Extract src.zip from source JDK jdk_src_zip = join(jdk.home, 'lib', 'src.zip') dst_src_zip_contents = {} if isfile(jdk_src_zip): mx.logv('[Extracting ' + jdk_src_zip + ']') with ZipFile(jdk_src_zip, 'r') as zf: for name in zf.namelist(): if not name.endswith('/'): dst_src_zip_contents[name] = zf.read(name) else: mx.warn("'{}' does not exist or is not a file".format(jdk_src_zip)) # Edit lib/security/default.policy in java.base patched_java_base = join(build_dir, 'java.base.jmod') with open(join(jmods_dir, 'java.base.jmod'), 'rb') as src_f, open(patched_java_base, 'wb') as dst_f: jmod_header = src_f.read(4) if len(jmod_header) != 4 or jmod_header != b'JM\x01\x00': raise mx.abort("Unexpected jmod header: " + b2a_hex(jmod_header).decode('ascii')) dst_f.write(jmod_header) policy_result = 'not found' with ZipFile(src_f, 'r') as src_zip, ZipFile(dst_f, 'w', src_zip.compression) as dst_zip: for i in src_zip.infolist(): if i.filename[-1] == '/': continue src_member = src_zip.read(i) if i.filename == 'lib/security/default.policy': policy_result = 'unmodified' if 'grant codeBase "jrt:/com.oracle.graal.graal_enterprise"'.encode('utf-8') not in src_member: policy_result = 'modified' src_member += """ grant codeBase "jrt:/com.oracle.graal.graal_enterprise" { permission java.security.AllPermission; }; """.encode('utf-8') if 'grant codeBase "jrt:/org.graalvm.truffle"'.encode('utf-8') not in src_member: policy_result = 'modified' src_member += """ grant codeBase "jrt:/org.graalvm.truffle" { permission java.security.AllPermission; }; grant codeBase "jrt:/org.graalvm.sdk" { permission java.security.AllPermission; }; grant codeBase "jrt:/org.graalvm.locator" { permission java.io.FilePermission "<<ALL FILES>>", "read"; permission java.util.PropertyPermission "*", "read,write"; permission java.lang.RuntimePermission "createClassLoader"; permission java.lang.RuntimePermission "getClassLoader"; permission java.lang.RuntimePermission "getenv.*"; }; grant codeBase "file:${java.home}/languages/-" { permission java.security.AllPermission; }; """.encode('utf-8') dst_zip.writestr(i, src_member) if policy_result == 'not found': raise mx.abort("Couldn't find `lib/security/default.policy` in " + join(jmods_dir, 'java.base.jmod')) for jmd in modules: # Remove existing sources for all the modules that we include dst_src_zip_contents = {key: dst_src_zip_contents[key] for key in dst_src_zip_contents if not key.startswith(jmd.name)} if with_source(jmd.dist): # Add the sources that we can share. # Extract module sources jmd_src_zip = jmd.jarpath[0:-len('.jar')] + '.src.zip' if isfile(jmd_src_zip): mx.logv('[Extracting ' + jmd_src_zip + ']') with ZipFile(jmd_src_zip, 'r') as zf: for name in zf.namelist(): if not name.endswith('/'): dst_src_zip_contents[jmd.name + '/' + name] = zf.read(name) # Add module-info.java to sources dst_src_zip_contents[jmd.name + '/module-info.java'] = jmd.as_module_info(extras_as_comments=False) # Now build the new JDK image with jlink jlink = [jdk.javac.replace('javac', 'jlink')] if jdk_enables_jvmci_by_default(jdk): # On JDK 9+, +EnableJVMCI forces jdk.internal.vm.ci to be in the root set jlink += ['-J-XX:-EnableJVMCI', '-J-XX:-UseJVMCICompiler'] if root_module_names is not None: missing = frozenset(root_module_names) - all_module_names if missing: mx.abort('Invalid module(s): {}.\nAvailable modules: {}'.format(','.join(missing), ','.join(sorted(all_module_names)))) jlink.append('--add-modules=' + ','.join(root_module_names)) else: jlink.append('--add-modules=' + ','.join(sorted(all_module_names))) module_path = patched_java_base + os.pathsep + jmods_dir if modules: module_path = os.pathsep.join((m.get_jmod_path(respect_stripping=True) for m in modules)) + os.pathsep + module_path jlink.append('--module-path=' + module_path) jlink.append('--output=' + dst_jdk_dir) # These options are derived from how OpenJDK runs jlink to produce the final runtime image. jlink.extend(['-J-XX:+UseSerialGC', '-J-Xms32M', '-J-Xmx512M', '-J-XX:TieredStopAtLevel=1']) jlink.append('-J-Dlink.debug=true') if dedup_legal_notices: jlink.append('--dedup-legal-notices=error-if-not-same-content') jlink.append('--keep-packaged-modules=' + join(dst_jdk_dir, 'jmods')) if jdk_has_new_jlink_options(jdk): if jdk_omits_warning_for_jlink_set_ThreadPriorityPolicy(jdk): thread_priority_policy_option = ' -XX:ThreadPriorityPolicy=1' else: mx.logv('[Creating JDK without -XX:ThreadPriorityPolicy=1]') thread_priority_policy_option = '' if jdk_supports_enablejvmciproduct(jdk): if any((m.name == 'jdk.internal.vm.compiler' for m in modules)): jlink.append('--add-options=-XX:+UnlockExperimentalVMOptions -XX:+EnableJVMCIProduct -XX:-UnlockExperimentalVMOptions' + thread_priority_policy_option) else: # Don't default to using JVMCI as JIT unless Graal is being updated in the image. # This avoids unexpected issues with using the out-of-date Graal compiler in # the JDK itself. jlink.append('--add-options=-XX:+UnlockExperimentalVMOptions -XX:+EnableJVMCIProduct -XX:-UseJVMCICompiler -XX:-UnlockExperimentalVMOptions' + thread_priority_policy_option) else: mx.logv('[Creating JDK without -XX:+EnableJVMCIProduct]') if thread_priority_policy_option: jlink.append('--add-options=' + thread_priority_policy_option.strip()) if vendor_info is not None: for name, value in vendor_info.items(): jlink.append('--' + name + '=' + value) release_file = join(jdk.home, 'release') if isfile(release_file): jlink.append('--release-info=' + release_file) # TODO: investigate the options below used by OpenJDK to see if they should be used: # --order-resources: specifies order of resources in generated lib/modules file. # This is apparently not so important if a CDS archive is available. # --generate-jli-classes: pre-generates a set of java.lang.invoke classes. # See https://github.com/openjdk/jdk/blob/master/make/GenerateLinkOptData.gmk mx.logv('[Creating JDK image in {}]'.format(dst_jdk_dir)) mx.run(jlink) dst_src_zip = join(dst_jdk_dir, 'lib', 'src.zip') mx.logv('[Creating ' + dst_src_zip + ']') with ZipFile(dst_src_zip, 'w', compression=ZIP_DEFLATED, allowZip64=True) as zf: for name, contents in sorted(dst_src_zip_contents.items()): zf.writestr(name, contents) mx.logv('[Copying static libraries]') lib_directory = join(jdk.home, 'lib', 'static') if exists(lib_directory): dst_lib_directory = join(dst_jdk_dir, 'lib', 'static') try: mx.copytree(lib_directory, dst_lib_directory) except shutil.Error as e: # On AArch64, there can be a problem in the copystat part # of copytree which occurs after file and directory copying # has successfully completed. Since the metadata doesn't # matter in this case, just ensure that the content was copied. for root, _, lib_files in os.walk(lib_directory): relative_root = os.path.relpath(root, dst_lib_directory) for lib in lib_files: src_lib_path = join(root, lib) dst_lib_path = join(dst_lib_directory, relative_root, lib) if not exists(dst_lib_path): mx.abort('Error copying static libraries: {} missing in {}{}Original copytree error: {}'.format( join(relative_root, lib), dst_lib_directory, os.linesep, e)) src_lib_hash = mx.sha1OfFile(src_lib_path) dst_lib_hash = mx.sha1OfFile(dst_lib_path) if src_lib_hash != dst_lib_hash: mx.abort('Error copying static libraries: {} (hash={}) and {} (hash={}) differ{}Original copytree error: {}'.format( src_lib_path, src_lib_hash, dst_lib_path, dst_lib_hash, os.linesep, e)) # Allow older JDK versions to work else: lib_prefix = mx.add_lib_prefix('') lib_suffix = mx.add_static_lib_suffix('') lib_directory = join(jdk.home, 'lib') dst_lib_directory = join(dst_jdk_dir, 'lib') for f in os.listdir(lib_directory): if f.startswith(lib_prefix) and f.endswith(lib_suffix): lib_path = join(lib_directory, f) if isfile(lib_path): shutil.copy2(lib_path, dst_lib_directory) finally: if not mx.get_opts().verbose: # Preserve build directory so that javac command can be re-executed # by cutting and pasting verbose output. shutil.rmtree(build_dir) # Create CDS archive (https://openjdk.java.net/jeps/341). out = mx.OutputCapture() mx.logv('[Creating CDS shared archive]') if mx.run([mx.exe_suffix(join(dst_jdk_dir, 'bin', 'java')), '-Xshare:dump', '-Xmx128M', '-Xms128M'], out=out, err=out, nonZeroIsFatal=False) != 0: mx.log(out.data) mx.abort('Error generating CDS shared archive')
def _parseVmArgs(args, addDefaultArgs=True): args = mx.expand_project_in_args(args, insitu=False) argsPrefix = [] jacocoArgs = mx_gate.get_jacoco_agent_args() if jacocoArgs: argsPrefix.extend(jacocoArgs) # add default graal.options.file options_file = join(mx.primary_suite().dir, 'graal.options') if exists(options_file): argsPrefix.append('-Dgraal.options.file=' + options_file) if isJDK8: argsPrefix.append('-Djvmci.class.path.append=' + os.pathsep.join((e.get_path() for e in _jvmci_classpath))) argsPrefix.append('-Xbootclasspath/a:' + os.pathsep.join([dep.classpath_repr() for dep in _bootclasspath_appends])) else: deployedDists = [entry.dist() for entry in _jvmci_classpath] + \ [e for e in _bootclasspath_appends if e.isJARDistribution()] deployedModules = [as_java_module(dist, jdk) for dist in deployedDists] # Set or update module path to include Graal and its dependencies as modules graalModulepath = [] for deployedModule in deployedModules: graalModulepath.extend([jmd.jarpath for jmd in deployedModule.modulepath if jmd.jarpath]) graalModulepath.append(deployedModule.jarpath) graalModulepath = _uniqify(graalModulepath) # Update added exports to include concealed JDK packages required by Graal addedExports = {} args = _extract_added_exports(args, addedExports) for deployedModule in deployedModules: for concealingModule, packages in deployedModule.concealedRequires.iteritems(): # No need to explicitly export JVMCI - it's exported via reflection if concealingModule != 'jdk.vm.ci': for package in packages: addedExports.setdefault(concealingModule + '/' + package, set()).add(deployedModule.name) for export, targets in addedExports.iteritems(): argsPrefix.append('--add-exports=' + export + '=' + ','.join(sorted(targets))) # Extend or set --module-path argument mpUpdated = False for mpIndex in range(len(args)): if args[mpIndex] == '--module-path': assert mpIndex + 1 < len(args), 'VM option ' + args[mpIndex] + ' requires an argument' args[mpIndex + 1] = os.pathsep.join(_uniqify(args[mpIndex + 1].split(os.pathsep) + graalModulepath)) mpUpdated = True break elif args[mpIndex].startswith('--module-path='): mp = args[mpIndex][len('--module-path='):] args[mpIndex] = '--module-path=' + os.pathsep.join(_uniqify(mp.split(os.pathsep) + graalModulepath)) mpUpdated = True break if not mpUpdated: argsPrefix.append('--module-path=' + os.pathsep.join(graalModulepath)) if '-version' in args: ignoredArgs = args[args.index('-version') + 1:] if len(ignoredArgs) > 0: mx.log("Warning: The following options will be ignored by the VM because they come after the '-version' argument: " + ' '.join(ignoredArgs)) return jdk.processArgs(argsPrefix + args, addDefaultArgs=addDefaultArgs)
def _unittest_config_participant(config): vmArgs, mainClass, mainClassArgs = config cpIndex, cp = mx.find_classpath_arg(vmArgs) if cp: cp = _uniqify(cp.split(os.pathsep)) if isJDK8: # Remove entries from class path that are in Graal or on the boot class path redundantClasspathEntries = set() for dist in [entry.dist() for entry in _jvmci_classpath]: redundantClasspathEntries.update((d.output_dir() for d in dist.archived_deps() if d.isJavaProject())) redundantClasspathEntries.add(dist.path) cp = os.pathsep.join([e for e in cp if e not in redundantClasspathEntries]) vmArgs[cpIndex] = cp else: deployedModules = [] redundantClasspathEntries = set() for dist in [entry.dist() for entry in _jvmci_classpath] + _bootclasspath_appends: deployedModule = as_java_module(dist, jdk) deployedModules.append(deployedModule) redundantClasspathEntries.update(mx.classpath(dist, preferProjects=False, jdk=jdk).split(os.pathsep)) redundantClasspathEntries.update(mx.classpath(dist, preferProjects=True, jdk=jdk).split(os.pathsep)) if hasattr(dist, 'overlaps'): for o in dist.overlaps: path = mx.distribution(o).classpath_repr() if path: redundantClasspathEntries.add(path) # Remove entries from the class path that are in the deployed modules cp = [classpathEntry for classpathEntry in cp if classpathEntry not in redundantClasspathEntries] vmArgs[cpIndex] = os.pathsep.join(cp) # Junit libraries are made into automatic modules so that they are visible to tests # patched into modules. These automatic modules must be declared to be read by # Graal which means they must also be made root modules (i.e., ``--add-modules``) # since ``--add-reads`` can only be applied to root modules. junitCp = [e.classpath_repr() for e in mx.classpath_entries(['JUNIT'])] junitModules = [_automatic_module_name(e) for e in junitCp] vmArgs.append('--module-path=' + os.pathsep.join(junitCp)) vmArgs.append('--add-modules=' + ','.join(junitModules + [m.name for m in deployedModules])) for deployedModule in deployedModules: vmArgs.append('--add-reads=' + deployedModule.name + '=' + ','.join(junitModules)) # Explicitly export concealed JVMCI packages required by Graal. Even though # normally exported via jdk.vm.ci.services.Services.exportJVMCITo(), the # Junit harness wants to access JVMCI classes (e.g., when loading classes # to find test methods) without going through that entry point. addedExports = {} for deployedModule in deployedModules: for concealingModule, packages in deployedModule.concealedRequires.iteritems(): if concealingModule == 'jdk.vm.ci': for package in packages: addedExports.setdefault(concealingModule + '/' + package, set()).add(deployedModule.name) pathToDep = {p.output_dir() if p.isJavaProject() else p.path: p for p in mx.dependencies() if p.isJavaProject() or p.isJARDistribution()} for classpathEntry in cp: # Export concealed packages used by the class path entry _add_exports_for_concealed_packages(classpathEntry, pathToDep, addedExports, 'ALL-UNNAMED', deployedModules) for deployedModule in deployedModules: assert deployedModule.dist.path != classpathEntry, deployedModule.dist.path + ' should no longer be on the class path' # Ensure the class path entry does not define packages already defined by the module. # Packages definitions cannot be split between modules. classpathEntryPackages = frozenset(_defined_packages(classpathEntry)) intersection = classpathEntryPackages.intersection(deployedModule.packages) if intersection: mx.abort(classpathEntry + ' cannot extend package(s) defined in the module ' + deployedModule.name + ': ' + ', '.join(intersection)) vmArgs.extend(['--add-exports=' + export + '=' + ','.join(sorted(targets)) for export, targets in addedExports.iteritems()]) if isJDK8: # Run the VM in a mode where application/test classes can # access JVMCI loaded classes. vmArgs.append('-XX:-UseJVMCIClassLoader') return (vmArgs, mainClass, mainClassArgs)
def _parseVmArgs(args, addDefaultArgs=True): args = mx.expand_project_in_args(args, insitu=False) argsPrefix = [] jacocoArgs = mx_gate.get_jacoco_agent_args() if jacocoArgs: argsPrefix.extend(jacocoArgs) # add default graal.options.file options_file = join(mx.primary_suite().dir, 'graal.options') if exists(options_file): argsPrefix.append('-Dgraal.options.file=' + options_file) if isJDK8: argsPrefix.append('-Djvmci.class.path.append=' + os.pathsep.join((e.get_path() for e in _jvmci_classpath))) argsPrefix.append('-Xbootclasspath/a:' + os.pathsep.join( [dep.classpath_repr() for dep in _bootclasspath_appends])) else: deployedDists = [entry.dist() for entry in _jvmci_classpath] + \ [e for e in _bootclasspath_appends if e.isJARDistribution()] deployedModules = [as_java_module(dist, jdk) for dist in deployedDists] # Set or update module path to include Graal and its dependencies as modules jdkModuleNames = frozenset([m.name for m in jdk.get_modules()]) graalModulepath = [] graalUpgrademodulepath = [] def _addToModulepath(modules): for m in modules: if m.jarpath: modulepath = graalModulepath if m.name not in jdkModuleNames else graalUpgrademodulepath if m not in modulepath: modulepath.append(m) for deployedModule in deployedModules: _addToModulepath(deployedModule.modulepath) _addToModulepath([deployedModule]) # Extend or set --module-path argument mpUpdated = False for mpIndex in range(len(args)): assert not args[mpIndex].startswith('--upgrade-module-path') if args[mpIndex] == '--module-path': assert mpIndex + 1 < len(args), 'VM option ' + args[ mpIndex] + ' requires an argument' args[mpIndex + 1] = os.pathsep.join( _uniqify(args[mpIndex + 1].split(os.pathsep) + [m.jarpath for m in graalModulepath])) mpUpdated = True break elif args[mpIndex].startswith('--module-path='): mp = args[mpIndex][len('--module-path='):] args[mpIndex] = '--module-path=' + os.pathsep.join( _uniqify( mp.split(os.pathsep) + [m.jarpath for m in graalModulepath])) mpUpdated = True break if not mpUpdated: argsPrefix.append( '--module-path=' + os.pathsep.join([m.jarpath for m in graalModulepath])) if graalUpgrademodulepath: argsPrefix.append( '--upgrade-module-path=' + os.pathsep.join([m.jarpath for m in graalUpgrademodulepath])) if '-version' in args: ignoredArgs = args[args.index('-version') + 1:] if len(ignoredArgs) > 0: mx.log( "Warning: The following options will be ignored by the VM because they come after the '-version' argument: " + ' '.join(ignoredArgs)) return jdk.processArgs(argsPrefix + args, addDefaultArgs=addDefaultArgs)
def _parseVmArgs(args, addDefaultArgs=True): args = mx.expand_project_in_args(args, insitu=False) argsPrefix = [] jacocoArgs = mx_gate.get_jacoco_agent_args() if jacocoArgs: argsPrefix.extend(jacocoArgs) # add default graal.options.file options_file = join(mx.primary_suite().dir, 'graal.options') if exists(options_file): argsPrefix.append('-Dgraal.options.file=' + options_file) if isJDK8: argsPrefix.append('-Djvmci.class.path.append=' + os.pathsep.join((e.get_path() for e in _jvmci_classpath))) argsPrefix.append('-Xbootclasspath/a:' + os.pathsep.join( [dep.classpath_repr() for dep in _bootclasspath_appends])) else: deployedDists = [entry.dist() for entry in _jvmci_classpath] + \ [e for e in _bootclasspath_appends if e.isJARDistribution()] deployedModules = [as_java_module(dist, jdk) for dist in deployedDists] # Set or update module path to include Graal and its dependencies as modules graalModulepath = [] for deployedModule in deployedModules: graalModulepath.extend([ jmd.jarpath for jmd in deployedModule.modulepath if jmd.jarpath ]) graalModulepath.append(deployedModule.jarpath) graalModulepath = _uniqify(graalModulepath) # Update added exports to include concealed JDK packages required by Graal addedExports = {} args = _extract_added_exports(args, addedExports) for deployedModule in deployedModules: for concealingModule, packages in deployedModule.concealedRequires.iteritems( ): # No need to explicitly export JVMCI - it's exported via reflection if concealingModule != 'jdk.vm.ci': for package in packages: addedExports.setdefault( concealingModule + '/' + package, set()).add(deployedModule.name) for export, targets in addedExports.iteritems(): argsPrefix.append('--add-exports=' + export + '=' + ','.join(sorted(targets))) # Extend or set --module-path argument mpUpdated = False for mpIndex in range(len(args)): if args[mpIndex] == '--module-path': assert mpIndex + 1 < len(args), 'VM option ' + args[ mpIndex] + ' requires an argument' args[mpIndex + 1] = os.pathsep.join( _uniqify(args[mpIndex + 1].split(os.pathsep) + graalModulepath)) mpUpdated = True break elif args[mpIndex].startswith('--module-path='): mp = args[mpIndex][len('--module-path='):] args[mpIndex] = '--module-path=' + os.pathsep.join( _uniqify(mp.split(os.pathsep) + graalModulepath)) mpUpdated = True break if not mpUpdated: argsPrefix.append('--module-path=' + os.pathsep.join(graalModulepath)) if '-version' in args: ignoredArgs = args[args.index('-version') + 1:] if len(ignoredArgs) > 0: mx.log( "Warning: The following options will be ignored by the VM because they come after the '-version' argument: " + ' '.join(ignoredArgs)) return jdk.processArgs(argsPrefix + args, addDefaultArgs=addDefaultArgs)
def _unittest_config_participant(config): vmArgs, mainClass, mainClassArgs = config cpIndex, cp = mx.find_classpath_arg(vmArgs) if cp: cp = _uniqify(cp.split(os.pathsep)) if isJDK8: # Remove entries from class path that are in Graal or on the boot class path redundantClasspathEntries = set() for dist in [entry.dist() for entry in _jvmci_classpath]: redundantClasspathEntries.update( (d.output_dir() for d in dist.archived_deps() if d.isJavaProject())) redundantClasspathEntries.add(dist.path) cp = os.pathsep.join( [e for e in cp if e not in redundantClasspathEntries]) vmArgs[cpIndex] = cp else: deployedModules = [] redundantClasspathEntries = set() for dist in [entry.dist() for entry in _jvmci_classpath ] + _bootclasspath_appends: deployedModule = as_java_module(dist, jdk) deployedModules.append(deployedModule) redundantClasspathEntries.update( mx.classpath(dist, preferProjects=False, jdk=jdk).split(os.pathsep)) redundantClasspathEntries.update( mx.classpath(dist, preferProjects=True, jdk=jdk).split(os.pathsep)) if hasattr(dist, 'overlaps'): for o in dist.overlaps: path = mx.distribution(o).classpath_repr() if path: redundantClasspathEntries.add(path) # Remove entries from the class path that are in the deployed modules cp = [ classpathEntry for classpathEntry in cp if classpathEntry not in redundantClasspathEntries ] vmArgs[cpIndex] = os.pathsep.join(cp) # Junit libraries are made into automatic modules so that they are visible to tests # patched into modules. These automatic modules must be declared to be read by # Graal which means they must also be made root modules (i.e., ``--add-modules``) # since ``--add-reads`` can only be applied to root modules. junitCp = [ e.classpath_repr() for e in mx.classpath_entries(['JUNIT']) ] junitModules = [_automatic_module_name(e) for e in junitCp] vmArgs.append('--module-path=' + os.pathsep.join(junitCp)) vmArgs.append('--add-modules=' + ','.join(junitModules + [m.name for m in deployedModules])) for deployedModule in deployedModules: vmArgs.append('--add-reads=' + deployedModule.name + '=' + ','.join(junitModules)) # Explicitly export concealed JVMCI packages required by Graal. Even though # normally exported via jdk.vm.ci.services.Services.exportJVMCITo(), the # Junit harness wants to access JVMCI classes (e.g., when loading classes # to find test methods) without going through that entry point. addedExports = {} for deployedModule in deployedModules: for concealingModule, packages in deployedModule.concealedRequires.iteritems( ): if concealingModule == 'jdk.vm.ci': for package in packages: addedExports.setdefault( concealingModule + '/' + package, set()).add(deployedModule.name) pathToDep = { p.output_dir() if p.isJavaProject() else p.path: p for p in mx.dependencies() if p.isJavaProject() or p.isJARDistribution() } for classpathEntry in cp: # Export concealed packages used by the class path entry _add_exports_for_concealed_packages(classpathEntry, pathToDep, addedExports, 'ALL-UNNAMED', deployedModules) for deployedModule in deployedModules: assert deployedModule.dist.path != classpathEntry, deployedModule.dist.path + ' should no longer be on the class path' # Ensure the class path entry does not define packages already defined by the module. # Packages definitions cannot be split between modules. classpathEntryPackages = frozenset( _defined_packages(classpathEntry)) intersection = classpathEntryPackages.intersection( deployedModule.packages) if intersection: mx.abort( classpathEntry + ' cannot extend package(s) defined in the module ' + deployedModule.name + ': ' + ', '.join(intersection)) vmArgs.extend([ '--add-exports=' + export + '=' + ','.join(sorted(targets)) for export, targets in addedExports.iteritems() ]) if isJDK8: # Run the VM in a mode where application/test classes can # access JVMCI loaded classes. vmArgs.append('-XX:-UseJVMCIClassLoader') return (vmArgs, mainClass, mainClassArgs)
def jlink_new_jdk(jdk, dst_jdk_dir, module_dists, ignore_dists, root_module_names=None, missing_export_target_action='create', with_source=lambda x: True, vendor_info=None, dedup_legal_notices=True, use_upgrade_module_path=False): """ Uses jlink from `jdk` to create a new JDK image in `dst_jdk_dir` with `module_dists` and their dependencies added to the JDK image, replacing any existing modules of the same name. :param JDKConfig jdk: source JDK :param str dst_jdk_dir: path to use for the jlink --output option :param list module_dists: list of distributions defining modules :param list ignore_dists: list of distributions that should be ignored for missing_export_target_action :param list root_module_names: list of strings naming the module root set for the new JDK image. The named modules must either be in `module_dists` or in `jdk`. If None, then the root set will be all the modules in ``module_dists` and `jdk`. :param str missing_export_target_action: the action to perform for a qualified export target that is not present in `module_dists` and does not have a hash stored in java.base. The choices are: "create" - an empty module is created "error" - raise an error None - do nothing :param lambda with_source: returns True if the sources of a module distribution must be included in the new JDK :param dict vendor_info: values for the jlink vendor options added by JDK-8232080 :param bool use_upgrade_module_path: if True, then instead of linking `module_dists` into the image, resolve them via --upgrade-module-path at image runtime :return bool: False if use_upgrade_module_path == True and the existing image is up to date otherwise True """ assert callable(with_source) if jdk.javaCompliance < '9': mx.abort('Cannot derive a new JDK from ' + jdk.home + ' with jlink since it is not JDK 9 or later') exploded_java_base_module = join(jdk.home, 'modules', 'java.base') if exists(exploded_java_base_module): mx.abort('Cannot derive a new JDK from ' + jdk.home + ' since it appears to be a developer build with exploded modules') jimage = join(jdk.home, 'lib', 'modules') jmods_dir = join(jdk.home, 'jmods') if not isfile(jimage): mx.abort('Cannot derive a new JDK from ' + jdk.home + ' since ' + jimage + ' is missing or is not an ordinary file') if not isdir(jmods_dir): mx.abort('Cannot derive a new JDK from ' + jdk.home + ' since ' + jmods_dir + ' is missing or is not a directory') # Exclude jdk.aot due to GR-10545 and JDK-8255616 jdk_modules = {jmd.name: jmd for jmd in jdk.get_modules() if jmd.name != 'jdk.aot'} modules = [as_java_module(dist, jdk) for dist in module_dists] module_names = frozenset((m.name for m in modules)) all_module_names = frozenset(list(jdk_modules.keys())) | module_names # Read hashes stored in java.base (the only module in the JDK where hashes are stored) hashes = _read_java_base_hashes(jdk) build_dir = mx.ensure_dir_exists(join(dst_jdk_dir + ".build")) # Directory under dst_jdk_dir for artifacts related to use_upgrade_module_path upgrade_dir = join(dst_jdk_dir, 'upgrade_modules_support') # Map from JavaModuleDescriptors to post-jlink jar location. synthetic_modules = OrderedDict() try: ignore_module_names = set(mx_javamodules.get_module_name(mx.dependency(ignore_dist)) for ignore_dist in ignore_dists) # Synthesize modules for targets of qualified exports that are not present in `modules`. # Without this, runtime module resolution will fail due to missing modules. target_requires = {} for jmd in modules: for targets in jmd.exports.values(): for target in targets: if target not in all_module_names and target not in ignore_module_names and target not in hashes: target_requires.setdefault(target, set()).add(jmd.name) if target_requires and missing_export_target_action is not None: if missing_export_target_action == 'error': mx.abort('Target(s) of qualified exports cannot be resolved: ' + '.'.join(target_requires.keys())) assert missing_export_target_action == 'create', 'invalid value for missing_export_target_action: ' + str(missing_export_target_action) for name, requires in sorted(target_requires.items()): module_jar = join(build_dir, name + '.jar') jmd = JavaModuleDescriptor(name, {}, requires={module: [] for module in requires}, uses=set(), provides={}, jarpath=module_jar) module_build_dir = mx.ensure_dir_exists(join(build_dir, name)) module_info = jmd.as_module_info() module_info_java = join(module_build_dir, 'module-info.java') module_info_class = join(module_build_dir, 'module-info.class') dst_module_jar = join(upgrade_dir, name + '.jar') synthetic_modules[jmd] = dst_module_jar if use_upgrade_module_path and exists(dst_module_jar): with ZipFile(dst_module_jar, 'r') as zf: previous_module_info = zf.read('module-info.java').decode() if previous_module_info == module_info: mx.logv('[Reusing synthetic module {}]'.format(name)) os.rename(dst_module_jar, module_jar) continue mx.logv('[Rebuilding synthetic module {} as module descriptor changed]'.format(name)) with open(module_info_java, 'w') as fp: fp.write(module_info) mx.run([jdk.javac, '-d', module_build_dir, '--limit-modules=java.base,' + ','.join(jmd.requires.keys()), '--module-path=' + os.pathsep.join((m.jarpath for m in modules)), module_info_java]) with ZipFile(module_jar, 'w') as zf: zf.write(module_info_java, 'module-info.java') zf.write(module_info_class, 'module-info.class') if exists(jmd.get_jmod_path()): os.remove(jmd.get_jmod_path()) if not use_upgrade_module_path: mx.run([jdk.javac.replace('javac', 'jmod'), 'create', '--class-path=' + module_build_dir, jmd.get_jmod_path()]) modules.extend(synthetic_modules.keys()) module_names = frozenset((m.name for m in modules)) all_module_names = frozenset(list(jdk_modules.keys())) | module_names # Edit lib/security/default.policy in java.base patched_java_base = _patch_default_security_policy(build_dir, jmods_dir, dst_jdk_dir) # Now build the new JDK image with jlink jlink = [jdk.javac.replace('javac', 'jlink')] if jdk_enables_jvmci_by_default(jdk): # On JDK 9+, +EnableJVMCI forces jdk.internal.vm.ci to be in the root set jlink += ['-J-XX:-EnableJVMCI', '-J-XX:-UseJVMCICompiler'] jlink.append('--add-modules=' + ','.join(_get_image_root_modules(root_module_names, module_names, jdk_modules.keys(), use_upgrade_module_path))) module_path = patched_java_base + os.pathsep + jmods_dir if modules and not use_upgrade_module_path: module_path = os.pathsep.join((m.get_jmod_path(respect_stripping=True) for m in modules)) + os.pathsep + module_path jlink.append('--module-path=' + module_path) jlink.append('--output=' + dst_jdk_dir) # These options are derived from how OpenJDK runs jlink to produce the final runtime image. jlink.extend(['-J-XX:+UseSerialGC', '-J-Xms32M', '-J-Xmx512M', '-J-XX:TieredStopAtLevel=1']) jlink.append('-J-Dlink.debug=true') if dedup_legal_notices: jlink.append('--dedup-legal-notices=error-if-not-same-content') jlink.append('--keep-packaged-modules=' + join(dst_jdk_dir, 'jmods')) vm_options_path = join(upgrade_dir, 'vm_options') vm_options = _get_image_vm_options(jdk, use_upgrade_module_path, modules, synthetic_modules) if vm_options: jlink.append('--add-options=' + ' '.join(vm_options)) if jdk_has_new_jlink_options(jdk) and vendor_info is not None: for name, value in vendor_info.items(): jlink.append('--' + name + '=' + value) release_file = join(jdk.home, 'release') if isfile(release_file): jlink.append('--release-info=' + release_file) if exists(dst_jdk_dir): if use_upgrade_module_path and _vm_options_match(vm_options, vm_options_path): mx.logv('[Existing JDK image {} is up to date]'.format(dst_jdk_dir)) return False mx.rmtree(dst_jdk_dir) # TODO: investigate the options below used by OpenJDK to see if they should be used: # --order-resources: specifies order of resources in generated lib/modules file. # This is apparently not so important if a CDS archive is available. # --generate-jli-classes: pre-generates a set of java.lang.invoke classes. # See https://github.com/openjdk/jdk/blob/master/make/GenerateLinkOptData.gmk mx.logv('[Creating JDK image in {}]'.format(dst_jdk_dir)) mx.run(jlink) if use_upgrade_module_path: # Move synthetic upgrade modules into final location for jmd, jarpath in synthetic_modules.items(): mx.ensure_dir_exists(dirname(jarpath)) os.rename(jmd.jarpath, jarpath) # Persist VM options cooked into image to be able to skip a subsequent # jlink execution if the options do not change. with open(vm_options_path, 'w') as fp: fp.write(os.linesep.join(vm_options)) # Create src.zip in new JDK image _copy_src_zip(jdk.home, dst_jdk_dir, modules, lambda jmd: not use_upgrade_module_path and with_source(jmd.dist)) mx.logv('[Copying static libraries]') lib_directory = join(jdk.home, 'lib', 'static') if exists(lib_directory): dst_lib_directory = join(dst_jdk_dir, 'lib', 'static') try: mx.copytree(lib_directory, dst_lib_directory) except shutil.Error as e: # On AArch64, there can be a problem in the copystat part # of copytree which occurs after file and directory copying # has successfully completed. Since the metadata doesn't # matter in this case, just ensure that the content was copied. for root, _, lib_files in os.walk(lib_directory): relative_root = os.path.relpath(root, dst_lib_directory) for lib in lib_files: src_lib_path = join(root, lib) dst_lib_path = join(dst_lib_directory, relative_root, lib) if not exists(dst_lib_path): mx.abort('Error copying static libraries: {} missing in {}{}Original copytree error: {}'.format( join(relative_root, lib), dst_lib_directory, os.linesep, e)) src_lib_hash = mx.sha1OfFile(src_lib_path) dst_lib_hash = mx.sha1OfFile(dst_lib_path) if src_lib_hash != dst_lib_hash: mx.abort('Error copying static libraries: {} (hash={}) and {} (hash={}) differ{}Original copytree error: {}'.format( src_lib_path, src_lib_hash, dst_lib_path, dst_lib_hash, os.linesep, e)) # Allow older JDK versions to work else: lib_prefix = mx.add_lib_prefix('') lib_suffix = mx.add_static_lib_suffix('') lib_directory = join(jdk.home, 'lib') dst_lib_directory = join(dst_jdk_dir, 'lib') for f in os.listdir(lib_directory): if f.startswith(lib_prefix) and f.endswith(lib_suffix): lib_path = join(lib_directory, f) if isfile(lib_path): shutil.copy2(lib_path, dst_lib_directory) finally: if not mx.get_opts().verbose: # Preserve build directory so that javac command can be re-executed # by cutting and pasting verbose output. shutil.rmtree(build_dir) if not use_upgrade_module_path: # Create CDS archive (https://openjdk.java.net/jeps/341). out = mx.OutputCapture() mx.logv('[Creating CDS shared archive]') if mx.run([mx.exe_suffix(join(dst_jdk_dir, 'bin', 'java')), '-Xshare:dump', '-Xmx128M', '-Xms128M'], out=out, err=out, nonZeroIsFatal=False) != 0: mx.log(out.data) mx.abort('Error generating CDS shared archive') else: # -Xshare is incompatible with --upgrade-module-path pass return True