def _path_args(depNames=None):
"""
Gets the VM args for putting the dependencies named in `depNames` on the
class path and module path (if running on JDK9 or later).
:param names: a Dependency, str or list containing Dependency/str objects. If None,
then all registered dependencies are used.
"""
jdk = mx.get_jdk()
if jdk.javaCompliance >= '1.9':
modules = [
as_java_module(dist, jdk) for dist in _suite.dists
if get_java_module_info(dist)
]
if modules:
# Partition resources between the class path and module path
modulepath = []
classpath = []
cpEntryToModule = {m.dist.path: m for m in modules}
for e in mx.classpath(depNames).split(os.pathsep):
if cpEntryToModule.has_key(e):
modulepath.append(cpEntryToModule[e].jarpath)
else:
classpath.append(e)
# The Truffle modules must be eagerly loaded as they could be referenced from
# the main class hence the --add-modules argument
return [
'--add-modules=' + ','.join([m.name for m in modules]),
'--module-path=' + os.pathsep.join(modulepath), '-cp',
os.pathsep.join(classpath)
]
return ['-cp', mx.classpath(depNames)]
def _path_args(depNames=None):
"""
Gets the VM args for putting the dependencies named in `depNames` on the
class path and module path (if running on JDK9 or later).
:param names: a Dependency, str or list containing Dependency/str objects. If None,
then all registered dependencies are used.
"""
jdk = mx.get_jdk()
if jdk.javaCompliance >= '1.9':
modules = [as_java_module(dist, jdk) for dist in _suite.dists if get_java_module_info(dist)]
if modules:
# Partition resources between the class path and module path
modulepath = []
classpath = []
cpEntryToModule = {m.dist.path : m for m in modules}
for e in mx.classpath(depNames).split(os.pathsep):
if cpEntryToModule.has_key(e):
modulepath.append(cpEntryToModule[e].jarpath)
else:
classpath.append(e)
# The Truffle modules must be eagerly loaded as they could be referenced from
# the main class hence the --add-modules argument
return ['--add-modules=' + ','.join([m.name for m in modules]), '--module-path=' + os.pathsep.join(modulepath), '-cp', os.pathsep.join(classpath)]
return ['-cp', mx.classpath(depNames)]
def get_module_jar(self):
"""
Gets the path to the module jar file.
:rtype: str
"""
return as_java_module(self.dist(), jdk).jarpath
def jlink_new_jdk(jdk, dst_jdk_dir, module_dists, root_module_names=None, missing_export_target_action='create', with_source=lambda x: True):
"""
Uses jlink from `jdk` to create a new JDK image in `dst_jdk_dir` with `module_dists` and
their dependencies added to the JDK image, replacing any existing modules of the same name.
:param JDKConfig jdk: source JDK
:param str dst_jdk_dir: path to use for the jlink --output option
:param list module_dists: list of distributions defining modules
:param list root_module_names: list of strings naming the module root set for the new JDK image.
The named modules must either be in `module_dists` or in `jdk`. If None, then
the root set will be all the modules in ``module_dists` and `jdk`.
:param str missing_export_target_action: the action to perform for a qualifed export target that
is not present in `module_dists` and does not have a hash stored in java.base.
The choices are:
"create" - an empty module is created
"error" - raise an error
None - do nothing
:param lambda with_source: returns True if the sources of a module distribution must be included in the new JDK
"""
assert callable(with_source)
if jdk.javaCompliance < '9':
mx.abort('Cannot derive a new JDK from ' + jdk.home + ' with jlink since it is not JDK 9 or later')
exploded_java_base_module = join(jdk.home, 'modules', 'java.base')
if exists(exploded_java_base_module):
mx.abort('Cannot derive a new JDK from ' + jdk.home + ' since it appears to be a developer build with exploded modules')
jimage = join(jdk.home, 'lib', 'modules')
jmods_dir = join(jdk.home, 'jmods')
if not isfile(jimage):
mx.abort('Cannot derive a new JDK from ' + jdk.home + ' since ' + jimage + ' is missing or is not an ordinary file')
if not isdir(jmods_dir):
mx.abort('Cannot derive a new JDK from ' + jdk.home + ' since ' + jmods_dir + ' is missing or is not a directory')
jdk_modules = {jmd.name : jmd for jmd in jdk.get_modules()}
modules = [as_java_module(dist, jdk) for dist in module_dists]
all_module_names = frozenset(list(jdk_modules.keys()) + [m.name for m in modules])
# Read hashes stored in java.base (the only module in the JDK where hashes are stored)
out = mx.LinesOutputCapture()
mx.run([jdk.exe_path('jmod'), 'describe', jdk_modules['java.base'].get_jmod_path()], out=out)
lines = out.lines
hashes = {}
for line in lines:
if line.startswith('hashes'):
parts = line.split()
assert len(parts) == 4, 'expected hashes line to have 4 fields, got {} fields: {}'.format(len(parts), line)
_, module_name, algorithm, hash_value = parts
hashes[module_name] = (algorithm, hash_value)
build_dir = mx.ensure_dir_exists(join(dst_jdk_dir + ".build"))
try:
# Handle targets of qualified exports that are not present in `modules`
target_requires = {}
for jmd in modules:
for targets in jmd.exports.values():
for target in targets:
if target not in all_module_names and target not in hashes:
target_requires.setdefault(target, set()).add(jmd.name)
if target_requires and missing_export_target_action is not None:
if missing_export_target_action == 'error':
mx.abort('Target(s) of qualified exports cannot be resolved: ' + '.'.join(target_requires.keys()))
assert missing_export_target_action == 'create', 'invalid value for missing_export_target_action: ' + str(missing_export_target_action)
extra_modules = []
for name, requires in target_requires.items():
module_jar = join(build_dir, name + '.jar')
jmd = JavaModuleDescriptor(name, {}, requires={module : [] for module in requires}, uses=set(), provides={}, jarpath=module_jar)
extra_modules.append(jmd)
module_build_dir = mx.ensure_dir_exists(join(build_dir, name))
module_info_java = join(module_build_dir, 'module-info.java')
module_info_class = join(module_build_dir, 'module-info.class')
with open(module_info_java, 'w') as fp:
print(jmd.as_module_info(), file=fp)
mx.run([jdk.javac, '-d', module_build_dir, \
'--limit-modules=java.base,' + ','.join(jmd.requires.keys()), \
'--module-path=' + os.pathsep.join((m.jarpath for m in modules)), \
module_info_java])
with ZipFile(module_jar, 'w') as zf:
zf.write(module_info_class, basename(module_info_class))
if exists(jmd.get_jmod_path()):
os.remove(jmd.get_jmod_path())
mx.run([jdk.javac.replace('javac', 'jmod'), 'create', '--class-path=' + module_build_dir, jmd.get_jmod_path()])
modules.extend(extra_modules)
all_module_names = frozenset(list(jdk_modules.keys()) + [m.name for m in modules])
# Extract src.zip from source JDK
jdk_src_zip = join(jdk.home, 'lib', 'src.zip')
dst_src_zip_contents = {}
if isfile(jdk_src_zip):
mx.logv('[Extracting ' + jdk_src_zip + ']')
with ZipFile(jdk_src_zip, 'r') as zf:
for name in zf.namelist():
if not name.endswith('/'):
dst_src_zip_contents[name] = zf.read(name)
else:
mx.warn("'{}' does not exist or is not a file".format(jdk_src_zip))
for jmd in modules:
# Remove existing sources for all the modules that we include
dst_src_zip_contents = {key : dst_src_zip_contents[key] for key in dst_src_zip_contents if not key.startswith(jmd.name)}
if with_source(jmd.dist):
# Add the sources that we can share.
# Extract module sources
jmd_src_zip = jmd.jarpath[0:-len('.jar')] + '.src.zip'
if isfile(jmd_src_zip):
mx.logv('[Extracting ' + jmd_src_zip + ']')
with ZipFile(jmd_src_zip, 'r') as zf:
for name in zf.namelist():
if not name.endswith('/'):
dst_src_zip_contents[jmd.name + '/' + name] = zf.read(name)
# Add module-info.java to sources
dst_src_zip_contents[jmd.name + '/module-info.java'] = jmd.as_module_info(extras_as_comments=False)
# Now build the new JDK image with jlink
jlink = [jdk.javac.replace('javac', 'jlink')]
if jdk_enables_jvmci_by_default(jdk):
# On JDK 9+, +EnableJVMCI forces jdk.internal.vm.ci to be in the root set
jlink.append('-J-XX:-EnableJVMCI')
if root_module_names is not None:
missing = frozenset(root_module_names) - all_module_names
if missing:
mx.abort('Invalid module(s): {}.\nAvailable modules: {}'.format(','.join(missing), ','.join(sorted(all_module_names))))
jlink.append('--add-modules=' + ','.join(root_module_names))
else:
jlink.append('--add-modules=' + ','.join(sorted(all_module_names)))
module_path = jmods_dir
if modules:
module_path = os.pathsep.join((m.get_jmod_path(respect_stripping=True) for m in modules)) + os.pathsep + module_path
jlink.append('--module-path=' + module_path)
jlink.append('--output=' + dst_jdk_dir)
# These options are inspired by how OpenJDK runs jlink to produce the final runtime image.
jlink.extend(['-J-XX:+UseSerialGC', '-J-Xms32M', '-J-Xmx512M', '-J-XX:TieredStopAtLevel=1'])
jlink.append('-J-Dlink.debug=true')
jlink.append('--dedup-legal-notices=error-if-not-same-content')
jlink.append('--keep-packaged-modules=' + join(dst_jdk_dir, 'jmods'))
# TODO: investigate the options below used by OpenJDK to see if they should be used:
# --release-info: this allow extra properties to be written to the <jdk>/release file
# --order-resources: specifies order of resources in generated lib/modules file.
# This is apparently not so important if a CDS archive is available.
# --generate-jli-classes: pre-generates a set of java.lang.invoke classes.
# See https://github.com/openjdk/jdk/blob/master/make/GenerateLinkOptData.gmk
mx.logv('[Creating JDK image]')
mx.run(jlink)
dst_src_zip = join(dst_jdk_dir, 'lib', 'src.zip')
mx.logv('[Creating ' + dst_src_zip + ']')
with ZipFile(dst_src_zip, 'w', compression=ZIP_DEFLATED, allowZip64=True) as zf:
for name, contents in sorted(dst_src_zip_contents.items()):
zf.writestr(name, contents)
mx.logv('[Copying static libraries]')
lib_prefix = mx.add_lib_prefix('')
lib_suffix = '.lib' if mx.is_windows() else '.a'
lib_directory = join(jdk.home, 'lib')
dst_lib_directory = join(dst_jdk_dir, 'lib')
for f in os.listdir(lib_directory):
if f.startswith(lib_prefix) and f.endswith(lib_suffix):
lib_path = join(lib_directory, f)
if isfile(lib_path):
shutil.copy2(lib_path, dst_lib_directory)
# Build the list of modules whose classes might have annotations
# to be processed by native-image (GR-15192).
with open(join(dst_jdk_dir, 'lib', 'native-image-modules.list'), 'w') as fp:
print('# Modules whose classes might have annotations processed by native-image', file=fp)
for m in modules:
print(m.name, file=fp)
finally:
if not mx.get_opts().verbose:
# Preserve build directory so that javac command can be re-executed
# by cutting and pasting verbose output.
shutil.rmtree(build_dir)
# Create CDS archive (https://openjdk.java.net/jeps/341).
out = mx.OutputCapture()
mx.logv('[Creating CDS shared archive]')
if mx.run([mx.exe_suffix(join(dst_jdk_dir, 'bin', 'java')), '-Xshare:dump', '-Xmx128M', '-Xms128M'], out=out, err=out, nonZeroIsFatal=False) != 0:
mx.log(out.data)
mx.abort('Error generating CDS shared archive')
def _parseVmArgs(args, addDefaultArgs=True):
args = mx.expand_project_in_args(args, insitu=False)
argsPrefix = []
jacocoArgs = mx_gate.get_jacoco_agent_args()
if jacocoArgs:
argsPrefix.extend(jacocoArgs)
# Check for -G: options
def checkGOption(arg):
if arg.startswith('-G:+'):
if '=' in arg:
mx.abort('Mixing + and = in -G: option specification: ' + arg)
translation = '-Dgraal.' + arg[len('-G:+'):] + '=true'
elif arg.startswith('-G:-'):
if '=' in arg:
mx.abort('Mixing - and = in -G: option specification: ' + arg)
translation = '-Dgraal.' + arg[len('-G:+'):] + '=false'
elif arg.startswith('-G:'):
if '=' not in arg:
mx.abort('Missing "=" in non-boolean -G: option specification: ' + arg)
translation = '-Dgraal.' + arg[len('-G:'):]
else:
return arg
mx.warn('Support for -G options is deprecated and will soon be removed. Replace "' + arg + '" with "' + translation + '"')
return translation
# add default graal.options.file
options_file = join(mx.primary_suite().dir, 'graal.options')
if exists(options_file):
argsPrefix.append('-Dgraal.options.file=' + options_file)
args = [checkGOption(a) for a in args]
if '-Dgraal.PrintFlags=true' in args and '-Xcomp' not in args:
mx.warn('Using -Dgraal.PrintFlags=true may have no effect without -Xcomp as Graal initialization is lazy')
if isJDK8:
argsPrefix.append('-Djvmci.class.path.append=' + os.pathsep.join((e.get_path() for e in _jvmci_classpath)))
argsPrefix.append('-Xbootclasspath/a:' + os.pathsep.join([dep.classpath_repr() for dep in _bootclasspath_appends]))
else:
deployedDists = [entry.dist() for entry in _jvmci_classpath] + \
[e for e in _bootclasspath_appends if e.isJARDistribution()]
deployedModules = [as_java_module(dist, jdk) for dist in deployedDists]
# Set or update module path to include Graal and its dependencies as modules
graalModulepath = []
for deployedModule in deployedModules:
graalModulepath.extend([jmd.jarpath for jmd in deployedModule.modulepath if jmd.jarpath])
graalModulepath.append(deployedModule.jarpath)
graalModulepath = _uniqify(graalModulepath)
# Update added exports to include concealed JDK packages required by Graal
addedExports = {}
args = _extract_added_exports(args, addedExports)
for deployedModule in deployedModules:
for concealingModule, packages in deployedModule.concealedRequires.iteritems():
# No need to explicitly export JVMCI - it's exported via reflection
if concealingModule != 'jdk.vm.ci':
for package in packages:
addedExports.setdefault(concealingModule + '/' + package, set()).add(deployedModule.name)
for export, targets in addedExports.iteritems():
argsPrefix.append('-XaddExports:' + export + '=' + ','.join(sorted(targets)))
# Extend or set -modulepath argument
mpUpdated = False
for mpIndex in range(len(args)):
if args[mpIndex] in ['-modulepath', '-mp']:
assert mpIndex + 1 < len(args), 'VM option ' + args[mpIndex] + ' requires an argument'
args[mpIndex + 1] = os.pathsep.join(_uniqify(args[mpIndex + 1].split(os.pathsep) + graalModulepath))
mpUpdated = True
break
if not mpUpdated:
argsPrefix.append('-modulepath')
argsPrefix.append(os.pathsep.join(graalModulepath))
# Set the JVMCI compiler to Graal
argsPrefix.append('-Djvmci.Compiler=graal')
if '-version' in args:
ignoredArgs = args[args.index('-version') + 1:]
if len(ignoredArgs) > 0:
mx.log("Warning: The following options will be ignored by the VM because they come after the '-version' argument: " + ' '.join(ignoredArgs))
return jdk.processArgs(argsPrefix + args, addDefaultArgs=addDefaultArgs)
def _unittest_config_participant(config):
vmArgs, mainClass, mainClassArgs = config
cpIndex, cp = mx.find_classpath_arg(vmArgs)
if cp:
cp = _uniqify(cp.split(os.pathsep))
if isJDK8:
# Remove entries from class path that are in Graal or on the boot class path
redundantClasspathEntries = set()
for dist in [entry.dist() for entry in _jvmci_classpath]:
redundantClasspathEntries.update((d.output_dir() for d in dist.archived_deps() if d.isJavaProject()))
redundantClasspathEntries.add(dist.path)
cp = os.pathsep.join([e for e in cp if e not in redundantClasspathEntries])
vmArgs[cpIndex] = cp
else:
deployedModules = []
redundantClasspathEntries = set()
for dist in [entry.dist() for entry in _jvmci_classpath] + _bootclasspath_appends:
deployedModule = as_java_module(dist, jdk)
deployedModules.append(deployedModule)
redundantClasspathEntries.update(mx.classpath(dist, preferProjects=False, jdk=jdk).split(os.pathsep))
redundantClasspathEntries.update(mx.classpath(dist, preferProjects=True, jdk=jdk).split(os.pathsep))
# Remove entries from the class path that are in the deployed modules
cp = [classpathEntry for classpathEntry in cp if classpathEntry not in redundantClasspathEntries]
vmArgs[cpIndex] = os.pathsep.join(cp)
# Junit libraries are made into automatic modules so that they are visible to tests
# patched into modules. These automatic modules must be declared to be read by
# Graal which means they must also be made root modules (i.e., ``-addmods``)
# since ``-XaddReads`` can only be applied to root modules.
junitCp = [e.classpath_repr() for e in mx.classpath_entries(['JUNIT'])]
junitModules = [_automatic_module_name(e) for e in junitCp]
vmArgs.extend(['-modulepath', os.pathsep.join(junitCp)])
vmArgs.extend(['-addmods', ','.join(junitModules + [m.name for m in deployedModules])])
for deployedModule in deployedModules:
vmArgs.append('-XaddReads:' + deployedModule.name + '=' + ','.join(junitModules))
# Explicitly export concealed JVMCI packages required by Graal. Even though
# normally exported via jdk.vm.ci.services.Services.exportJVMCITo(), the
# Junit harness wants to access JVMCI classes (e.g., when loading classes
# to find test methods) without going through that entry point.
addedExports = {}
for deployedModule in deployedModules:
for concealingModule, packages in deployedModule.concealedRequires.iteritems():
if concealingModule == 'jdk.vm.ci':
for package in packages:
addedExports.setdefault(concealingModule + '/' + package, set()).add(deployedModule.name)
patches = {}
pathToProject = {p.output_dir() : p for p in mx.projects() if p.isJavaProject()}
for classpathEntry in cp:
# Export concealed packages used by the class path entry
_add_exports_for_concealed_packages(classpathEntry, pathToProject, addedExports, 'ALL-UNNAMED', deployedModules)
for deployedModule in deployedModules:
assert deployedModule.dist.path != classpathEntry, deployedModule.dist.path + ' should no longer be on the class path'
# Patch the class path entry into a module if it defines packages already defined by the module.
# Packages definitions cannot be split between modules.
classpathEntryPackages = frozenset(_defined_packages(classpathEntry))
if not classpathEntryPackages.isdisjoint(deployedModule.packages):
patches.setdefault(deployedModule.name, []).append(classpathEntry)
extraPackages = classpathEntryPackages - frozenset(deployedModule.exports.iterkeys())
if extraPackages:
# From http://openjdk.java.net/jeps/261:
# If a package found in a module definition on a patch path is not already exported
# by that module then it will, still, not be exported. It can be exported explicitly
# via either the reflection API or the -XaddExports option.
for package in extraPackages:
addedExports.setdefault(deployedModule.name + '/' + package, set()).update(junitModules + ['ALL-UNNAMED'])
for moduleName, cpEntries in patches.iteritems():
vmArgs.append('-Xpatch:' + moduleName + '=' + os.pathsep.join(cpEntries))
vmArgs.extend(['-XaddExports:' + export + '=' + ','.join(sorted(targets)) for export, targets in addedExports.iteritems()])
if isJDK8:
# Run the VM in a mode where application/test classes can
# access JVMCI loaded classes.
vmArgs.append('-XX:-UseJVMCIClassLoader')
return (vmArgs, mainClass, mainClassArgs)
def _unittest_config_participant(config):
vmArgs, mainClass, mainClassArgs = config
cpIndex, cp = mx.find_classpath_arg(vmArgs)
if cp:
cp = _uniqify(cp.split(os.pathsep))
if isJDK8:
# Remove entries from class path that are in Graal
excluded = set()
for entry in _jvmci_classpath:
dist = entry.dist()
excluded.update((d.output_dir() for d in dist.archived_deps() if d.isJavaProject()))
excluded.add(dist.path)
cp = os.pathsep.join([e for e in cp if e not in excluded])
vmArgs[cpIndex] = cp
else:
# Remove entries from class path that are in the Graal
assert _graal_module_descriptor is not None
module = as_java_module(_graal_module_descriptor.dist(), jdk)
junitCp = [e.classpath_repr() for e in mx.classpath_entries(['JUNIT'])]
excluded = frozenset([classpathEntry.classpath_repr() for classpathEntry in get_module_deps(module.dist)] + junitCp)
cp = [classpathEntry for classpathEntry in cp if classpathEntry not in excluded]
vmArgs[cpIndex] = os.pathsep.join(cp)
# Junit libraries are made into automatic modules so that they are visible to tests
# patched into Graal. These automatic modules must be declared to be read by
# Graal which means they must also be made root modules (i.e., ``-addmods``)
# since ``-XaddReads`` can only be applied to root modules.
junitModules = [_automatic_module_name(e) for e in junitCp]
vmArgs.extend(['-modulepath', os.pathsep.join(junitCp)])
vmArgs.extend(['-addmods', ','.join(junitModules)])
vmArgs.extend(['-XaddReads:' + module.name + '=' + ','.join(junitModules)])
# Explicitly export JVMCI to Graal
addedExports = {}
for concealingModule, packages in module.concealedRequires.iteritems():
if concealingModule == 'jdk.vm.ci':
for package in packages:
addedExports.setdefault(concealingModule + '/' + package, set()).add(module.name)
patches = []
graalConcealedPackages = list(module.conceals)
pathToProject = {p.output_dir() : p for p in mx.projects() if p.isJavaProject()}
for classpathEntry in cp:
# Export concealed JDK packages used by the class path entry
_add_exports_for_concealed_packages(classpathEntry, pathToProject, addedExports, 'ALL-UNNAMED')
# Patch the class path entry into Graal if it defines packages already defined by Graal.
# Packages definitions cannot be split between modules.
packages = frozenset(_defined_packages(classpathEntry))
if not packages.isdisjoint(module.packages):
patches.append(classpathEntry)
extraPackages = packages - module.packages
if extraPackages:
# From http://openjdk.java.net/jeps/261:
# If a package found in a module definition on a patch path is not already exported
# by that module then it will, still, not be exported. It can be exported explicitly
# via either the reflection API or the -XaddExports option.
graalConcealedPackages.extend(extraPackages)
if patches:
vmArgs.append('-Xpatch:' + module.name + '=' + os.pathsep.join(patches))
# Export all Graal packages to make them available to test classes
for package in graalConcealedPackages:
addedExports.setdefault(module.name + '/' + package, set()).update(junitModules + ['ALL-UNNAMED'])
vmArgs.extend(['-XaddExports:' + export + '=' + ','.join(sorted(targets)) for export, targets in addedExports.iteritems()])
if isJDK8:
# Run the VM in a mode where application/test classes can
# access JVMCI loaded classes.
vmArgs.append('-XX:-UseJVMCIClassLoader')
return (vmArgs, mainClass, mainClassArgs)
def jlink_new_jdk(jdk, dst_jdk_dir, module_dists,
root_module_names=None,
missing_export_target_action='create',
with_source=lambda x: True,
vendor_info=None,
dedup_legal_notices=True):
"""
Uses jlink from `jdk` to create a new JDK image in `dst_jdk_dir` with `module_dists` and
their dependencies added to the JDK image, replacing any existing modules of the same name.
:param JDKConfig jdk: source JDK
:param str dst_jdk_dir: path to use for the jlink --output option
:param list module_dists: list of distributions defining modules
:param list root_module_names: list of strings naming the module root set for the new JDK image.
The named modules must either be in `module_dists` or in `jdk`. If None, then
the root set will be all the modules in ``module_dists` and `jdk`.
:param str missing_export_target_action: the action to perform for a qualifed export target that
is not present in `module_dists` and does not have a hash stored in java.base.
The choices are:
"create" - an empty module is created
"error" - raise an error
None - do nothing
:param lambda with_source: returns True if the sources of a module distribution must be included in the new JDK
:param dict vendor_info: values for the jlink vendor options added by JDK-8232080
"""
assert callable(with_source)
if jdk.javaCompliance < '9':
mx.abort('Cannot derive a new JDK from ' + jdk.home + ' with jlink since it is not JDK 9 or later')
exploded_java_base_module = join(jdk.home, 'modules', 'java.base')
if exists(exploded_java_base_module):
mx.abort('Cannot derive a new JDK from ' + jdk.home + ' since it appears to be a developer build with exploded modules')
jimage = join(jdk.home, 'lib', 'modules')
jmods_dir = join(jdk.home, 'jmods')
if not isfile(jimage):
mx.abort('Cannot derive a new JDK from ' + jdk.home + ' since ' + jimage + ' is missing or is not an ordinary file')
if not isdir(jmods_dir):
mx.abort('Cannot derive a new JDK from ' + jdk.home + ' since ' + jmods_dir + ' is missing or is not a directory')
# Exclude jdk.aot due to GR-10545 and JDK-8255616
jdk_modules = {jmd.name: jmd for jmd in jdk.get_modules() if jmd.name != 'jdk.aot'}
modules = [as_java_module(dist, jdk) for dist in module_dists]
all_module_names = frozenset(list(jdk_modules.keys()) + [m.name for m in modules])
# Read hashes stored in java.base (the only module in the JDK where hashes are stored)
out = mx.LinesOutputCapture()
mx.run([jdk.exe_path('jmod'), 'describe', jdk_modules['java.base'].get_jmod_path()], out=out)
lines = out.lines
hashes = {}
for line in lines:
if line.startswith('hashes'):
parts = line.split()
assert len(parts) == 4, 'expected hashes line to have 4 fields, got {} fields: {}'.format(len(parts), line)
_, module_name, algorithm, hash_value = parts
hashes[module_name] = (algorithm, hash_value)
build_dir = mx.ensure_dir_exists(join(dst_jdk_dir + ".build"))
try:
# Handle targets of qualified exports that are not present in `modules`
target_requires = {}
for jmd in modules:
for targets in jmd.exports.values():
for target in targets:
if target not in all_module_names and target not in hashes:
target_requires.setdefault(target, set()).add(jmd.name)
if target_requires and missing_export_target_action is not None:
if missing_export_target_action == 'error':
mx.abort('Target(s) of qualified exports cannot be resolved: ' + '.'.join(target_requires.keys()))
assert missing_export_target_action == 'create', 'invalid value for missing_export_target_action: ' + str(missing_export_target_action)
extra_modules = []
for name, requires in target_requires.items():
module_jar = join(build_dir, name + '.jar')
jmd = JavaModuleDescriptor(name, {}, requires={module: [] for module in requires}, uses=set(), provides={}, jarpath=module_jar)
extra_modules.append(jmd)
module_build_dir = mx.ensure_dir_exists(join(build_dir, name))
module_info_java = join(module_build_dir, 'module-info.java')
module_info_class = join(module_build_dir, 'module-info.class')
with open(module_info_java, 'w') as fp:
print(jmd.as_module_info(), file=fp)
mx.run([jdk.javac, '-d', module_build_dir,
'--limit-modules=java.base,' + ','.join(jmd.requires.keys()),
'--module-path=' + os.pathsep.join((m.jarpath for m in modules)),
module_info_java])
with ZipFile(module_jar, 'w') as zf:
zf.write(module_info_class, basename(module_info_class))
if exists(jmd.get_jmod_path()):
os.remove(jmd.get_jmod_path())
mx.run([jdk.javac.replace('javac', 'jmod'), 'create', '--class-path=' + module_build_dir, jmd.get_jmod_path()])
modules.extend(extra_modules)
all_module_names = frozenset(list(jdk_modules.keys()) + [m.name for m in modules])
# Extract src.zip from source JDK
jdk_src_zip = join(jdk.home, 'lib', 'src.zip')
dst_src_zip_contents = {}
if isfile(jdk_src_zip):
mx.logv('[Extracting ' + jdk_src_zip + ']')
with ZipFile(jdk_src_zip, 'r') as zf:
for name in zf.namelist():
if not name.endswith('/'):
dst_src_zip_contents[name] = zf.read(name)
else:
mx.warn("'{}' does not exist or is not a file".format(jdk_src_zip))
# Edit lib/security/default.policy in java.base
patched_java_base = join(build_dir, 'java.base.jmod')
with open(join(jmods_dir, 'java.base.jmod'), 'rb') as src_f, open(patched_java_base, 'wb') as dst_f:
jmod_header = src_f.read(4)
if len(jmod_header) != 4 or jmod_header != b'JM\x01\x00':
raise mx.abort("Unexpected jmod header: " + b2a_hex(jmod_header).decode('ascii'))
dst_f.write(jmod_header)
policy_result = 'not found'
with ZipFile(src_f, 'r') as src_zip, ZipFile(dst_f, 'w', src_zip.compression) as dst_zip:
for i in src_zip.infolist():
if i.filename[-1] == '/':
continue
src_member = src_zip.read(i)
if i.filename == 'lib/security/default.policy':
policy_result = 'unmodified'
if 'grant codeBase "jrt:/com.oracle.graal.graal_enterprise"'.encode('utf-8') not in src_member:
policy_result = 'modified'
src_member += """
grant codeBase "jrt:/com.oracle.graal.graal_enterprise" {
permission java.security.AllPermission;
};
""".encode('utf-8')
if 'grant codeBase "jrt:/org.graalvm.truffle"'.encode('utf-8') not in src_member:
policy_result = 'modified'
src_member += """
grant codeBase "jrt:/org.graalvm.truffle" {
permission java.security.AllPermission;
};
grant codeBase "jrt:/org.graalvm.sdk" {
permission java.security.AllPermission;
};
grant codeBase "jrt:/org.graalvm.locator" {
permission java.io.FilePermission "<<ALL FILES>>", "read";
permission java.util.PropertyPermission "*", "read,write";
permission java.lang.RuntimePermission "createClassLoader";
permission java.lang.RuntimePermission "getClassLoader";
permission java.lang.RuntimePermission "getenv.*";
};
grant codeBase "file:${java.home}/languages/-" {
permission java.security.AllPermission;
};
""".encode('utf-8')
dst_zip.writestr(i, src_member)
if policy_result == 'not found':
raise mx.abort("Couldn't find `lib/security/default.policy` in " + join(jmods_dir, 'java.base.jmod'))
for jmd in modules:
# Remove existing sources for all the modules that we include
dst_src_zip_contents = {key: dst_src_zip_contents[key] for key in dst_src_zip_contents if not key.startswith(jmd.name)}
if with_source(jmd.dist):
# Add the sources that we can share.
# Extract module sources
jmd_src_zip = jmd.jarpath[0:-len('.jar')] + '.src.zip'
if isfile(jmd_src_zip):
mx.logv('[Extracting ' + jmd_src_zip + ']')
with ZipFile(jmd_src_zip, 'r') as zf:
for name in zf.namelist():
if not name.endswith('/'):
dst_src_zip_contents[jmd.name + '/' + name] = zf.read(name)
# Add module-info.java to sources
dst_src_zip_contents[jmd.name + '/module-info.java'] = jmd.as_module_info(extras_as_comments=False)
# Now build the new JDK image with jlink
jlink = [jdk.javac.replace('javac', 'jlink')]
if jdk_enables_jvmci_by_default(jdk):
# On JDK 9+, +EnableJVMCI forces jdk.internal.vm.ci to be in the root set
jlink += ['-J-XX:-EnableJVMCI', '-J-XX:-UseJVMCICompiler']
if root_module_names is not None:
missing = frozenset(root_module_names) - all_module_names
if missing:
mx.abort('Invalid module(s): {}.\nAvailable modules: {}'.format(','.join(missing), ','.join(sorted(all_module_names))))
jlink.append('--add-modules=' + ','.join(root_module_names))
else:
jlink.append('--add-modules=' + ','.join(sorted(all_module_names)))
module_path = patched_java_base + os.pathsep + jmods_dir
if modules:
module_path = os.pathsep.join((m.get_jmod_path(respect_stripping=True) for m in modules)) + os.pathsep + module_path
jlink.append('--module-path=' + module_path)
jlink.append('--output=' + dst_jdk_dir)
# These options are derived from how OpenJDK runs jlink to produce the final runtime image.
jlink.extend(['-J-XX:+UseSerialGC', '-J-Xms32M', '-J-Xmx512M', '-J-XX:TieredStopAtLevel=1'])
jlink.append('-J-Dlink.debug=true')
if dedup_legal_notices:
jlink.append('--dedup-legal-notices=error-if-not-same-content')
jlink.append('--keep-packaged-modules=' + join(dst_jdk_dir, 'jmods'))
if jdk_has_new_jlink_options(jdk):
if jdk_omits_warning_for_jlink_set_ThreadPriorityPolicy(jdk):
thread_priority_policy_option = ' -XX:ThreadPriorityPolicy=1'
else:
mx.logv('[Creating JDK without -XX:ThreadPriorityPolicy=1]')
thread_priority_policy_option = ''
if jdk_supports_enablejvmciproduct(jdk):
if any((m.name == 'jdk.internal.vm.compiler' for m in modules)):
jlink.append('--add-options=-XX:+UnlockExperimentalVMOptions -XX:+EnableJVMCIProduct -XX:-UnlockExperimentalVMOptions' + thread_priority_policy_option)
else:
# Don't default to using JVMCI as JIT unless Graal is being updated in the image.
# This avoids unexpected issues with using the out-of-date Graal compiler in
# the JDK itself.
jlink.append('--add-options=-XX:+UnlockExperimentalVMOptions -XX:+EnableJVMCIProduct -XX:-UseJVMCICompiler -XX:-UnlockExperimentalVMOptions' + thread_priority_policy_option)
else:
mx.logv('[Creating JDK without -XX:+EnableJVMCIProduct]')
if thread_priority_policy_option:
jlink.append('--add-options=' + thread_priority_policy_option.strip())
if vendor_info is not None:
for name, value in vendor_info.items():
jlink.append('--' + name + '=' + value)
release_file = join(jdk.home, 'release')
if isfile(release_file):
jlink.append('--release-info=' + release_file)
# TODO: investigate the options below used by OpenJDK to see if they should be used:
# --order-resources: specifies order of resources in generated lib/modules file.
# This is apparently not so important if a CDS archive is available.
# --generate-jli-classes: pre-generates a set of java.lang.invoke classes.
# See https://github.com/openjdk/jdk/blob/master/make/GenerateLinkOptData.gmk
mx.logv('[Creating JDK image in {}]'.format(dst_jdk_dir))
mx.run(jlink)
dst_src_zip = join(dst_jdk_dir, 'lib', 'src.zip')
mx.logv('[Creating ' + dst_src_zip + ']')
with ZipFile(dst_src_zip, 'w', compression=ZIP_DEFLATED, allowZip64=True) as zf:
for name, contents in sorted(dst_src_zip_contents.items()):
zf.writestr(name, contents)
mx.logv('[Copying static libraries]')
lib_directory = join(jdk.home, 'lib', 'static')
if exists(lib_directory):
dst_lib_directory = join(dst_jdk_dir, 'lib', 'static')
try:
mx.copytree(lib_directory, dst_lib_directory)
except shutil.Error as e:
# On AArch64, there can be a problem in the copystat part
# of copytree which occurs after file and directory copying
# has successfully completed. Since the metadata doesn't
# matter in this case, just ensure that the content was copied.
for root, _, lib_files in os.walk(lib_directory):
relative_root = os.path.relpath(root, dst_lib_directory)
for lib in lib_files:
src_lib_path = join(root, lib)
dst_lib_path = join(dst_lib_directory, relative_root, lib)
if not exists(dst_lib_path):
mx.abort('Error copying static libraries: {} missing in {}{}Original copytree error: {}'.format(
join(relative_root, lib), dst_lib_directory, os.linesep, e))
src_lib_hash = mx.sha1OfFile(src_lib_path)
dst_lib_hash = mx.sha1OfFile(dst_lib_path)
if src_lib_hash != dst_lib_hash:
mx.abort('Error copying static libraries: {} (hash={}) and {} (hash={}) differ{}Original copytree error: {}'.format(
src_lib_path, src_lib_hash,
dst_lib_path, dst_lib_hash,
os.linesep, e))
# Allow older JDK versions to work
else:
lib_prefix = mx.add_lib_prefix('')
lib_suffix = mx.add_static_lib_suffix('')
lib_directory = join(jdk.home, 'lib')
dst_lib_directory = join(dst_jdk_dir, 'lib')
for f in os.listdir(lib_directory):
if f.startswith(lib_prefix) and f.endswith(lib_suffix):
lib_path = join(lib_directory, f)
if isfile(lib_path):
shutil.copy2(lib_path, dst_lib_directory)
finally:
if not mx.get_opts().verbose:
# Preserve build directory so that javac command can be re-executed
# by cutting and pasting verbose output.
shutil.rmtree(build_dir)
# Create CDS archive (https://openjdk.java.net/jeps/341).
out = mx.OutputCapture()
mx.logv('[Creating CDS shared archive]')
if mx.run([mx.exe_suffix(join(dst_jdk_dir, 'bin', 'java')), '-Xshare:dump', '-Xmx128M', '-Xms128M'], out=out, err=out, nonZeroIsFatal=False) != 0:
mx.log(out.data)
mx.abort('Error generating CDS shared archive')
def _parseVmArgs(args, addDefaultArgs=True):
args = mx.expand_project_in_args(args, insitu=False)
argsPrefix = []
jacocoArgs = mx_gate.get_jacoco_agent_args()
if jacocoArgs:
argsPrefix.extend(jacocoArgs)
# add default graal.options.file
options_file = join(mx.primary_suite().dir, 'graal.options')
if exists(options_file):
argsPrefix.append('-Dgraal.options.file=' + options_file)
if isJDK8:
argsPrefix.append('-Djvmci.class.path.append=' + os.pathsep.join((e.get_path() for e in _jvmci_classpath)))
argsPrefix.append('-Xbootclasspath/a:' + os.pathsep.join([dep.classpath_repr() for dep in _bootclasspath_appends]))
else:
deployedDists = [entry.dist() for entry in _jvmci_classpath] + \
[e for e in _bootclasspath_appends if e.isJARDistribution()]
deployedModules = [as_java_module(dist, jdk) for dist in deployedDists]
# Set or update module path to include Graal and its dependencies as modules
graalModulepath = []
for deployedModule in deployedModules:
graalModulepath.extend([jmd.jarpath for jmd in deployedModule.modulepath if jmd.jarpath])
graalModulepath.append(deployedModule.jarpath)
graalModulepath = _uniqify(graalModulepath)
# Update added exports to include concealed JDK packages required by Graal
addedExports = {}
args = _extract_added_exports(args, addedExports)
for deployedModule in deployedModules:
for concealingModule, packages in deployedModule.concealedRequires.iteritems():
# No need to explicitly export JVMCI - it's exported via reflection
if concealingModule != 'jdk.vm.ci':
for package in packages:
addedExports.setdefault(concealingModule + '/' + package, set()).add(deployedModule.name)
for export, targets in addedExports.iteritems():
argsPrefix.append('--add-exports=' + export + '=' + ','.join(sorted(targets)))
# Extend or set --module-path argument
mpUpdated = False
for mpIndex in range(len(args)):
if args[mpIndex] == '--module-path':
assert mpIndex + 1 < len(args), 'VM option ' + args[mpIndex] + ' requires an argument'
args[mpIndex + 1] = os.pathsep.join(_uniqify(args[mpIndex + 1].split(os.pathsep) + graalModulepath))
mpUpdated = True
break
elif args[mpIndex].startswith('--module-path='):
mp = args[mpIndex][len('--module-path='):]
args[mpIndex] = '--module-path=' + os.pathsep.join(_uniqify(mp.split(os.pathsep) + graalModulepath))
mpUpdated = True
break
if not mpUpdated:
argsPrefix.append('--module-path=' + os.pathsep.join(graalModulepath))
if '-version' in args:
ignoredArgs = args[args.index('-version') + 1:]
if len(ignoredArgs) > 0:
mx.log("Warning: The following options will be ignored by the VM because they come after the '-version' argument: " + ' '.join(ignoredArgs))
return jdk.processArgs(argsPrefix + args, addDefaultArgs=addDefaultArgs)
def _unittest_config_participant(config):
vmArgs, mainClass, mainClassArgs = config
cpIndex, cp = mx.find_classpath_arg(vmArgs)
if cp:
cp = _uniqify(cp.split(os.pathsep))
if isJDK8:
# Remove entries from class path that are in Graal or on the boot class path
redundantClasspathEntries = set()
for dist in [entry.dist() for entry in _jvmci_classpath]:
redundantClasspathEntries.update((d.output_dir() for d in dist.archived_deps() if d.isJavaProject()))
redundantClasspathEntries.add(dist.path)
cp = os.pathsep.join([e for e in cp if e not in redundantClasspathEntries])
vmArgs[cpIndex] = cp
else:
deployedModules = []
redundantClasspathEntries = set()
for dist in [entry.dist() for entry in _jvmci_classpath] + _bootclasspath_appends:
deployedModule = as_java_module(dist, jdk)
deployedModules.append(deployedModule)
redundantClasspathEntries.update(mx.classpath(dist, preferProjects=False, jdk=jdk).split(os.pathsep))
redundantClasspathEntries.update(mx.classpath(dist, preferProjects=True, jdk=jdk).split(os.pathsep))
if hasattr(dist, 'overlaps'):
for o in dist.overlaps:
path = mx.distribution(o).classpath_repr()
if path:
redundantClasspathEntries.add(path)
# Remove entries from the class path that are in the deployed modules
cp = [classpathEntry for classpathEntry in cp if classpathEntry not in redundantClasspathEntries]
vmArgs[cpIndex] = os.pathsep.join(cp)
# Junit libraries are made into automatic modules so that they are visible to tests
# patched into modules. These automatic modules must be declared to be read by
# Graal which means they must also be made root modules (i.e., ``--add-modules``)
# since ``--add-reads`` can only be applied to root modules.
junitCp = [e.classpath_repr() for e in mx.classpath_entries(['JUNIT'])]
junitModules = [_automatic_module_name(e) for e in junitCp]
vmArgs.append('--module-path=' + os.pathsep.join(junitCp))
vmArgs.append('--add-modules=' + ','.join(junitModules + [m.name for m in deployedModules]))
for deployedModule in deployedModules:
vmArgs.append('--add-reads=' + deployedModule.name + '=' + ','.join(junitModules))
# Explicitly export concealed JVMCI packages required by Graal. Even though
# normally exported via jdk.vm.ci.services.Services.exportJVMCITo(), the
# Junit harness wants to access JVMCI classes (e.g., when loading classes
# to find test methods) without going through that entry point.
addedExports = {}
for deployedModule in deployedModules:
for concealingModule, packages in deployedModule.concealedRequires.iteritems():
if concealingModule == 'jdk.vm.ci':
for package in packages:
addedExports.setdefault(concealingModule + '/' + package, set()).add(deployedModule.name)
pathToDep = {p.output_dir() if p.isJavaProject() else p.path: p for p in mx.dependencies() if p.isJavaProject() or p.isJARDistribution()}
for classpathEntry in cp:
# Export concealed packages used by the class path entry
_add_exports_for_concealed_packages(classpathEntry, pathToDep, addedExports, 'ALL-UNNAMED', deployedModules)
for deployedModule in deployedModules:
assert deployedModule.dist.path != classpathEntry, deployedModule.dist.path + ' should no longer be on the class path'
# Ensure the class path entry does not define packages already defined by the module.
# Packages definitions cannot be split between modules.
classpathEntryPackages = frozenset(_defined_packages(classpathEntry))
intersection = classpathEntryPackages.intersection(deployedModule.packages)
if intersection:
mx.abort(classpathEntry + ' cannot extend package(s) defined in the module ' + deployedModule.name + ': ' + ', '.join(intersection))
vmArgs.extend(['--add-exports=' + export + '=' + ','.join(sorted(targets)) for export, targets in addedExports.iteritems()])
if isJDK8:
# Run the VM in a mode where application/test classes can
# access JVMCI loaded classes.
vmArgs.append('-XX:-UseJVMCIClassLoader')
return (vmArgs, mainClass, mainClassArgs)
def _parseVmArgs(args, addDefaultArgs=True):
args = mx.expand_project_in_args(args, insitu=False)
argsPrefix = []
jacocoArgs = mx_gate.get_jacoco_agent_args()
if jacocoArgs:
argsPrefix.extend(jacocoArgs)
# add default graal.options.file
options_file = join(mx.primary_suite().dir, 'graal.options')
if exists(options_file):
argsPrefix.append('-Dgraal.options.file=' + options_file)
if isJDK8:
argsPrefix.append('-Djvmci.class.path.append=' +
os.pathsep.join((e.get_path()
for e in _jvmci_classpath)))
argsPrefix.append('-Xbootclasspath/a:' + os.pathsep.join(
[dep.classpath_repr() for dep in _bootclasspath_appends]))
else:
deployedDists = [entry.dist() for entry in _jvmci_classpath] + \
[e for e in _bootclasspath_appends if e.isJARDistribution()]
deployedModules = [as_java_module(dist, jdk) for dist in deployedDists]
# Set or update module path to include Graal and its dependencies as modules
jdkModuleNames = frozenset([m.name for m in jdk.get_modules()])
graalModulepath = []
graalUpgrademodulepath = []
def _addToModulepath(modules):
for m in modules:
if m.jarpath:
modulepath = graalModulepath if m.name not in jdkModuleNames else graalUpgrademodulepath
if m not in modulepath:
modulepath.append(m)
for deployedModule in deployedModules:
_addToModulepath(deployedModule.modulepath)
_addToModulepath([deployedModule])
# Extend or set --module-path argument
mpUpdated = False
for mpIndex in range(len(args)):
assert not args[mpIndex].startswith('--upgrade-module-path')
if args[mpIndex] == '--module-path':
assert mpIndex + 1 < len(args), 'VM option ' + args[
mpIndex] + ' requires an argument'
args[mpIndex + 1] = os.pathsep.join(
_uniqify(args[mpIndex + 1].split(os.pathsep) +
[m.jarpath for m in graalModulepath]))
mpUpdated = True
break
elif args[mpIndex].startswith('--module-path='):
mp = args[mpIndex][len('--module-path='):]
args[mpIndex] = '--module-path=' + os.pathsep.join(
_uniqify(
mp.split(os.pathsep) +
[m.jarpath for m in graalModulepath]))
mpUpdated = True
break
if not mpUpdated:
argsPrefix.append(
'--module-path=' +
os.pathsep.join([m.jarpath for m in graalModulepath]))
if graalUpgrademodulepath:
argsPrefix.append(
'--upgrade-module-path=' +
os.pathsep.join([m.jarpath for m in graalUpgrademodulepath]))
if '-version' in args:
ignoredArgs = args[args.index('-version') + 1:]
if len(ignoredArgs) > 0:
mx.log(
"Warning: The following options will be ignored by the VM because they come after the '-version' argument: "
+ ' '.join(ignoredArgs))
return jdk.processArgs(argsPrefix + args, addDefaultArgs=addDefaultArgs)
def _parseVmArgs(args, addDefaultArgs=True):
args = mx.expand_project_in_args(args, insitu=False)
argsPrefix = []
jacocoArgs = mx_gate.get_jacoco_agent_args()
if jacocoArgs:
argsPrefix.extend(jacocoArgs)
# add default graal.options.file
options_file = join(mx.primary_suite().dir, 'graal.options')
if exists(options_file):
argsPrefix.append('-Dgraal.options.file=' + options_file)
if isJDK8:
argsPrefix.append('-Djvmci.class.path.append=' +
os.pathsep.join((e.get_path()
for e in _jvmci_classpath)))
argsPrefix.append('-Xbootclasspath/a:' + os.pathsep.join(
[dep.classpath_repr() for dep in _bootclasspath_appends]))
else:
deployedDists = [entry.dist() for entry in _jvmci_classpath] + \
[e for e in _bootclasspath_appends if e.isJARDistribution()]
deployedModules = [as_java_module(dist, jdk) for dist in deployedDists]
# Set or update module path to include Graal and its dependencies as modules
graalModulepath = []
for deployedModule in deployedModules:
graalModulepath.extend([
jmd.jarpath for jmd in deployedModule.modulepath if jmd.jarpath
])
graalModulepath.append(deployedModule.jarpath)
graalModulepath = _uniqify(graalModulepath)
# Update added exports to include concealed JDK packages required by Graal
addedExports = {}
args = _extract_added_exports(args, addedExports)
for deployedModule in deployedModules:
for concealingModule, packages in deployedModule.concealedRequires.iteritems(
):
# No need to explicitly export JVMCI - it's exported via reflection
if concealingModule != 'jdk.vm.ci':
for package in packages:
addedExports.setdefault(
concealingModule + '/' + package,
set()).add(deployedModule.name)
for export, targets in addedExports.iteritems():
argsPrefix.append('--add-exports=' + export + '=' +
','.join(sorted(targets)))
# Extend or set --module-path argument
mpUpdated = False
for mpIndex in range(len(args)):
if args[mpIndex] == '--module-path':
assert mpIndex + 1 < len(args), 'VM option ' + args[
mpIndex] + ' requires an argument'
args[mpIndex + 1] = os.pathsep.join(
_uniqify(args[mpIndex + 1].split(os.pathsep) +
graalModulepath))
mpUpdated = True
break
elif args[mpIndex].startswith('--module-path='):
mp = args[mpIndex][len('--module-path='):]
args[mpIndex] = '--module-path=' + os.pathsep.join(
_uniqify(mp.split(os.pathsep) + graalModulepath))
mpUpdated = True
break
if not mpUpdated:
argsPrefix.append('--module-path=' +
os.pathsep.join(graalModulepath))
if '-version' in args:
ignoredArgs = args[args.index('-version') + 1:]
if len(ignoredArgs) > 0:
mx.log(
"Warning: The following options will be ignored by the VM because they come after the '-version' argument: "
+ ' '.join(ignoredArgs))
return jdk.processArgs(argsPrefix + args, addDefaultArgs=addDefaultArgs)
def _unittest_config_participant(config):
vmArgs, mainClass, mainClassArgs = config
cpIndex, cp = mx.find_classpath_arg(vmArgs)
if cp:
cp = _uniqify(cp.split(os.pathsep))
if isJDK8:
# Remove entries from class path that are in Graal or on the boot class path
redundantClasspathEntries = set()
for dist in [entry.dist() for entry in _jvmci_classpath]:
redundantClasspathEntries.update(
(d.output_dir() for d in dist.archived_deps()
if d.isJavaProject()))
redundantClasspathEntries.add(dist.path)
cp = os.pathsep.join(
[e for e in cp if e not in redundantClasspathEntries])
vmArgs[cpIndex] = cp
else:
deployedModules = []
redundantClasspathEntries = set()
for dist in [entry.dist() for entry in _jvmci_classpath
] + _bootclasspath_appends:
deployedModule = as_java_module(dist, jdk)
deployedModules.append(deployedModule)
redundantClasspathEntries.update(
mx.classpath(dist, preferProjects=False,
jdk=jdk).split(os.pathsep))
redundantClasspathEntries.update(
mx.classpath(dist, preferProjects=True,
jdk=jdk).split(os.pathsep))
if hasattr(dist, 'overlaps'):
for o in dist.overlaps:
path = mx.distribution(o).classpath_repr()
if path:
redundantClasspathEntries.add(path)
# Remove entries from the class path that are in the deployed modules
cp = [
classpathEntry for classpathEntry in cp
if classpathEntry not in redundantClasspathEntries
]
vmArgs[cpIndex] = os.pathsep.join(cp)
# Junit libraries are made into automatic modules so that they are visible to tests
# patched into modules. These automatic modules must be declared to be read by
# Graal which means they must also be made root modules (i.e., ``--add-modules``)
# since ``--add-reads`` can only be applied to root modules.
junitCp = [
e.classpath_repr() for e in mx.classpath_entries(['JUNIT'])
]
junitModules = [_automatic_module_name(e) for e in junitCp]
vmArgs.append('--module-path=' + os.pathsep.join(junitCp))
vmArgs.append('--add-modules=' +
','.join(junitModules +
[m.name for m in deployedModules]))
for deployedModule in deployedModules:
vmArgs.append('--add-reads=' + deployedModule.name + '=' +
','.join(junitModules))
# Explicitly export concealed JVMCI packages required by Graal. Even though
# normally exported via jdk.vm.ci.services.Services.exportJVMCITo(), the
# Junit harness wants to access JVMCI classes (e.g., when loading classes
# to find test methods) without going through that entry point.
addedExports = {}
for deployedModule in deployedModules:
for concealingModule, packages in deployedModule.concealedRequires.iteritems(
):
if concealingModule == 'jdk.vm.ci':
for package in packages:
addedExports.setdefault(
concealingModule + '/' + package,
set()).add(deployedModule.name)
pathToDep = {
p.output_dir() if p.isJavaProject() else p.path: p
for p in mx.dependencies()
if p.isJavaProject() or p.isJARDistribution()
}
for classpathEntry in cp:
# Export concealed packages used by the class path entry
_add_exports_for_concealed_packages(classpathEntry, pathToDep,
addedExports,
'ALL-UNNAMED',
deployedModules)
for deployedModule in deployedModules:
assert deployedModule.dist.path != classpathEntry, deployedModule.dist.path + ' should no longer be on the class path'
# Ensure the class path entry does not define packages already defined by the module.
# Packages definitions cannot be split between modules.
classpathEntryPackages = frozenset(
_defined_packages(classpathEntry))
intersection = classpathEntryPackages.intersection(
deployedModule.packages)
if intersection:
mx.abort(
classpathEntry +
' cannot extend package(s) defined in the module '
+ deployedModule.name + ': ' +
', '.join(intersection))
vmArgs.extend([
'--add-exports=' + export + '=' + ','.join(sorted(targets))
for export, targets in addedExports.iteritems()
])
if isJDK8:
# Run the VM in a mode where application/test classes can
# access JVMCI loaded classes.
vmArgs.append('-XX:-UseJVMCIClassLoader')
return (vmArgs, mainClass, mainClassArgs)
def jlink_new_jdk(jdk, dst_jdk_dir, module_dists, ignore_dists,
root_module_names=None,
missing_export_target_action='create',
with_source=lambda x: True,
vendor_info=None,
dedup_legal_notices=True,
use_upgrade_module_path=False):
"""
Uses jlink from `jdk` to create a new JDK image in `dst_jdk_dir` with `module_dists` and
their dependencies added to the JDK image, replacing any existing modules of the same name.
:param JDKConfig jdk: source JDK
:param str dst_jdk_dir: path to use for the jlink --output option
:param list module_dists: list of distributions defining modules
:param list ignore_dists: list of distributions that should be ignored for missing_export_target_action
:param list root_module_names: list of strings naming the module root set for the new JDK image.
The named modules must either be in `module_dists` or in `jdk`. If None, then
the root set will be all the modules in ``module_dists` and `jdk`.
:param str missing_export_target_action: the action to perform for a qualified export target that
is not present in `module_dists` and does not have a hash stored in java.base.
The choices are:
"create" - an empty module is created
"error" - raise an error
None - do nothing
:param lambda with_source: returns True if the sources of a module distribution must be included in the new JDK
:param dict vendor_info: values for the jlink vendor options added by JDK-8232080
:param bool use_upgrade_module_path: if True, then instead of linking `module_dists` into the image, resolve
them via --upgrade-module-path at image runtime
:return bool: False if use_upgrade_module_path == True and the existing image is up to date otherwise True
"""
assert callable(with_source)
if jdk.javaCompliance < '9':
mx.abort('Cannot derive a new JDK from ' + jdk.home + ' with jlink since it is not JDK 9 or later')
exploded_java_base_module = join(jdk.home, 'modules', 'java.base')
if exists(exploded_java_base_module):
mx.abort('Cannot derive a new JDK from ' + jdk.home + ' since it appears to be a developer build with exploded modules')
jimage = join(jdk.home, 'lib', 'modules')
jmods_dir = join(jdk.home, 'jmods')
if not isfile(jimage):
mx.abort('Cannot derive a new JDK from ' + jdk.home + ' since ' + jimage + ' is missing or is not an ordinary file')
if not isdir(jmods_dir):
mx.abort('Cannot derive a new JDK from ' + jdk.home + ' since ' + jmods_dir + ' is missing or is not a directory')
# Exclude jdk.aot due to GR-10545 and JDK-8255616
jdk_modules = {jmd.name: jmd for jmd in jdk.get_modules() if jmd.name != 'jdk.aot'}
modules = [as_java_module(dist, jdk) for dist in module_dists]
module_names = frozenset((m.name for m in modules))
all_module_names = frozenset(list(jdk_modules.keys())) | module_names
# Read hashes stored in java.base (the only module in the JDK where hashes are stored)
hashes = _read_java_base_hashes(jdk)
build_dir = mx.ensure_dir_exists(join(dst_jdk_dir + ".build"))
# Directory under dst_jdk_dir for artifacts related to use_upgrade_module_path
upgrade_dir = join(dst_jdk_dir, 'upgrade_modules_support')
# Map from JavaModuleDescriptors to post-jlink jar location.
synthetic_modules = OrderedDict()
try:
ignore_module_names = set(mx_javamodules.get_module_name(mx.dependency(ignore_dist)) for ignore_dist in ignore_dists)
# Synthesize modules for targets of qualified exports that are not present in `modules`.
# Without this, runtime module resolution will fail due to missing modules.
target_requires = {}
for jmd in modules:
for targets in jmd.exports.values():
for target in targets:
if target not in all_module_names and target not in ignore_module_names and target not in hashes:
target_requires.setdefault(target, set()).add(jmd.name)
if target_requires and missing_export_target_action is not None:
if missing_export_target_action == 'error':
mx.abort('Target(s) of qualified exports cannot be resolved: ' + '.'.join(target_requires.keys()))
assert missing_export_target_action == 'create', 'invalid value for missing_export_target_action: ' + str(missing_export_target_action)
for name, requires in sorted(target_requires.items()):
module_jar = join(build_dir, name + '.jar')
jmd = JavaModuleDescriptor(name, {}, requires={module: [] for module in requires}, uses=set(), provides={}, jarpath=module_jar)
module_build_dir = mx.ensure_dir_exists(join(build_dir, name))
module_info = jmd.as_module_info()
module_info_java = join(module_build_dir, 'module-info.java')
module_info_class = join(module_build_dir, 'module-info.class')
dst_module_jar = join(upgrade_dir, name + '.jar')
synthetic_modules[jmd] = dst_module_jar
if use_upgrade_module_path and exists(dst_module_jar):
with ZipFile(dst_module_jar, 'r') as zf:
previous_module_info = zf.read('module-info.java').decode()
if previous_module_info == module_info:
mx.logv('[Reusing synthetic module {}]'.format(name))
os.rename(dst_module_jar, module_jar)
continue
mx.logv('[Rebuilding synthetic module {} as module descriptor changed]'.format(name))
with open(module_info_java, 'w') as fp:
fp.write(module_info)
mx.run([jdk.javac, '-d', module_build_dir,
'--limit-modules=java.base,' + ','.join(jmd.requires.keys()),
'--module-path=' + os.pathsep.join((m.jarpath for m in modules)),
module_info_java])
with ZipFile(module_jar, 'w') as zf:
zf.write(module_info_java, 'module-info.java')
zf.write(module_info_class, 'module-info.class')
if exists(jmd.get_jmod_path()):
os.remove(jmd.get_jmod_path())
if not use_upgrade_module_path:
mx.run([jdk.javac.replace('javac', 'jmod'), 'create', '--class-path=' + module_build_dir, jmd.get_jmod_path()])
modules.extend(synthetic_modules.keys())
module_names = frozenset((m.name for m in modules))
all_module_names = frozenset(list(jdk_modules.keys())) | module_names
# Edit lib/security/default.policy in java.base
patched_java_base = _patch_default_security_policy(build_dir, jmods_dir, dst_jdk_dir)
# Now build the new JDK image with jlink
jlink = [jdk.javac.replace('javac', 'jlink')]
if jdk_enables_jvmci_by_default(jdk):
# On JDK 9+, +EnableJVMCI forces jdk.internal.vm.ci to be in the root set
jlink += ['-J-XX:-EnableJVMCI', '-J-XX:-UseJVMCICompiler']
jlink.append('--add-modules=' + ','.join(_get_image_root_modules(root_module_names, module_names, jdk_modules.keys(), use_upgrade_module_path)))
module_path = patched_java_base + os.pathsep + jmods_dir
if modules and not use_upgrade_module_path:
module_path = os.pathsep.join((m.get_jmod_path(respect_stripping=True) for m in modules)) + os.pathsep + module_path
jlink.append('--module-path=' + module_path)
jlink.append('--output=' + dst_jdk_dir)
# These options are derived from how OpenJDK runs jlink to produce the final runtime image.
jlink.extend(['-J-XX:+UseSerialGC', '-J-Xms32M', '-J-Xmx512M', '-J-XX:TieredStopAtLevel=1'])
jlink.append('-J-Dlink.debug=true')
if dedup_legal_notices:
jlink.append('--dedup-legal-notices=error-if-not-same-content')
jlink.append('--keep-packaged-modules=' + join(dst_jdk_dir, 'jmods'))
vm_options_path = join(upgrade_dir, 'vm_options')
vm_options = _get_image_vm_options(jdk, use_upgrade_module_path, modules, synthetic_modules)
if vm_options:
jlink.append('--add-options=' + ' '.join(vm_options))
if jdk_has_new_jlink_options(jdk) and vendor_info is not None:
for name, value in vendor_info.items():
jlink.append('--' + name + '=' + value)
release_file = join(jdk.home, 'release')
if isfile(release_file):
jlink.append('--release-info=' + release_file)
if exists(dst_jdk_dir):
if use_upgrade_module_path and _vm_options_match(vm_options, vm_options_path):
mx.logv('[Existing JDK image {} is up to date]'.format(dst_jdk_dir))
return False
mx.rmtree(dst_jdk_dir)
# TODO: investigate the options below used by OpenJDK to see if they should be used:
# --order-resources: specifies order of resources in generated lib/modules file.
# This is apparently not so important if a CDS archive is available.
# --generate-jli-classes: pre-generates a set of java.lang.invoke classes.
# See https://github.com/openjdk/jdk/blob/master/make/GenerateLinkOptData.gmk
mx.logv('[Creating JDK image in {}]'.format(dst_jdk_dir))
mx.run(jlink)
if use_upgrade_module_path:
# Move synthetic upgrade modules into final location
for jmd, jarpath in synthetic_modules.items():
mx.ensure_dir_exists(dirname(jarpath))
os.rename(jmd.jarpath, jarpath)
# Persist VM options cooked into image to be able to skip a subsequent
# jlink execution if the options do not change.
with open(vm_options_path, 'w') as fp:
fp.write(os.linesep.join(vm_options))
# Create src.zip in new JDK image
_copy_src_zip(jdk.home, dst_jdk_dir, modules, lambda jmd: not use_upgrade_module_path and with_source(jmd.dist))
mx.logv('[Copying static libraries]')
lib_directory = join(jdk.home, 'lib', 'static')
if exists(lib_directory):
dst_lib_directory = join(dst_jdk_dir, 'lib', 'static')
try:
mx.copytree(lib_directory, dst_lib_directory)
except shutil.Error as e:
# On AArch64, there can be a problem in the copystat part
# of copytree which occurs after file and directory copying
# has successfully completed. Since the metadata doesn't
# matter in this case, just ensure that the content was copied.
for root, _, lib_files in os.walk(lib_directory):
relative_root = os.path.relpath(root, dst_lib_directory)
for lib in lib_files:
src_lib_path = join(root, lib)
dst_lib_path = join(dst_lib_directory, relative_root, lib)
if not exists(dst_lib_path):
mx.abort('Error copying static libraries: {} missing in {}{}Original copytree error: {}'.format(
join(relative_root, lib), dst_lib_directory, os.linesep, e))
src_lib_hash = mx.sha1OfFile(src_lib_path)
dst_lib_hash = mx.sha1OfFile(dst_lib_path)
if src_lib_hash != dst_lib_hash:
mx.abort('Error copying static libraries: {} (hash={}) and {} (hash={}) differ{}Original copytree error: {}'.format(
src_lib_path, src_lib_hash,
dst_lib_path, dst_lib_hash,
os.linesep, e))
# Allow older JDK versions to work
else:
lib_prefix = mx.add_lib_prefix('')
lib_suffix = mx.add_static_lib_suffix('')
lib_directory = join(jdk.home, 'lib')
dst_lib_directory = join(dst_jdk_dir, 'lib')
for f in os.listdir(lib_directory):
if f.startswith(lib_prefix) and f.endswith(lib_suffix):
lib_path = join(lib_directory, f)
if isfile(lib_path):
shutil.copy2(lib_path, dst_lib_directory)
finally:
if not mx.get_opts().verbose:
# Preserve build directory so that javac command can be re-executed
# by cutting and pasting verbose output.
shutil.rmtree(build_dir)
if not use_upgrade_module_path:
# Create CDS archive (https://openjdk.java.net/jeps/341).
out = mx.OutputCapture()
mx.logv('[Creating CDS shared archive]')
if mx.run([mx.exe_suffix(join(dst_jdk_dir, 'bin', 'java')), '-Xshare:dump', '-Xmx128M', '-Xms128M'], out=out, err=out, nonZeroIsFatal=False) != 0:
mx.log(out.data)
mx.abort('Error generating CDS shared archive')
else:
# -Xshare is incompatible with --upgrade-module-path
pass
return True