def query_native_image(all_args, option):
out = mx.LinesOutputCapture()
_native_image(['--dry-run'] + all_args, out=out)
for line in out.lines:
_, sep, after = line.partition(option)
if sep:
return after.split(' ')[0].rstrip()
return None
def jdk_enables_jvmci_by_default(jdk):
"""
Gets the default value for the EnableJVMCI VM option in `jdk`.
"""
if not hasattr(jdk, '.enables_jvmci_by_default'):
out = mx.LinesOutputCapture()
sink = lambda x: x
mx.run([jdk.java, '-XX:+UnlockExperimentalVMOptions', '-XX:+PrintFlagsFinal', '-version'], out=out, err=sink)
setattr(jdk, '.enables_jvmci_by_default', any('EnableJVMCI' in line and 'true' in line for line in out.lines))
return getattr(jdk, '.enables_jvmci_by_default')
def verify_graalvm_configs(suites=None):
"""
Check the consistency of registered GraalVM configs.
:param suites: optionally restrict the check to the configs registered by this list of suites.
:type suites: list[str] or None
"""
import mx_sdk_vm_impl
child_env = os.environ.copy()
for env_var in ['DYNAMIC_IMPORTS', 'DEFAULT_DYNAMIC_IMPORTS', 'COMPONENTS', 'EXCLUDE_COMPONENTS', 'SKIP_LIBRARIES', 'NATIVE_IMAGES', 'FORCE_BASH_LAUNCHERS', 'DISABLE_POLYGLOT', 'DISABLE_LIBPOLYGLOT']:
if env_var in child_env:
del child_env[env_var]
for dist_name, _, components, suite, env_file in _vm_configs:
if env_file is not False and (suites is None or suite.name in suites):
_env_file = env_file or dist_name
graalvm_dist_name = '{base_name}_{dist_name}_JAVA{jdk_version}'.format(base_name=mx_sdk_vm_impl._graalvm_base_name, dist_name=dist_name, jdk_version=mx_sdk_vm_impl._src_jdk_version).upper().replace('-', '_')
mx.log("Checking that the env file '{}' in suite '{}' produces a GraalVM distribution named '{}'".format(_env_file, suite.name, graalvm_dist_name))
out = mx.LinesOutputCapture()
err = mx.LinesOutputCapture()
retcode = mx.run_mx(['--quiet', '--no-warning', '--env', _env_file, 'graalvm-dist-name'], suite, out=out, err=err, env=child_env, nonZeroIsFatal=False)
if retcode != 0:
mx.abort("Unexpected return code '{}' for 'graalvm-dist-name' for env file '{}' in suite '{}'. Output:\n{}\nError:\n{}".format(retcode, _env_file, suite.name, '\n'.join(out.lines), '\n'.join(err.lines)))
if len(out.lines) != 1 or out.lines[0] != graalvm_dist_name:
out2 = mx.LinesOutputCapture()
retcode2 = mx.run_mx(['--no-warning', '--env', _env_file, 'graalvm-components'], suite, out=out2, err=out2, env=child_env, nonZeroIsFatal=False)
if retcode2 or len(out2.lines) != 1:
got_components = '<error>'
diff = ''
else:
got_components = out2.lines[0] # example string: "['bpolyglot', 'cmp']"
got_components_set = set(got_components[1:-1].replace('\'', '').split(', '))
components_set = set(components)
added = list(got_components_set - components_set)
removed = list(components_set - got_components_set)
diff = ('Added:\n{}\n'.format(added) if added else '') + ('Removed:\n{}\n'.format(removed) if removed else '')
mx.abort("""\
Unexpected GraalVM dist name for env file '{}' in suite '{}'.
Expected dist name: '{}'
Actual dist name: '{}'.
Expected component list:
{}
Actual component list:
{}
{}Did you forget to update the registration of the GraalVM config?""".format(_env_file, suite.name, graalvm_dist_name, '\n'.join(out.lines + err.lines), sorted(components), got_components, diff))
def jdk_supports_enablejvmciproduct(jdk):
"""
Determines if the jdk supports flag -XX:+EnableJVMCIProduct which isn't the case
for some OpenJDK 11u distros.
"""
if not hasattr(jdk, '.supports_enablejvmciproduct'):
out = mx.LinesOutputCapture()
sink = lambda x: x
mx.run([jdk.java, '-XX:+UnlockExperimentalVMOptions', '-XX:+PrintFlagsFinal', '-version'], out=out, err=sink)
setattr(jdk, '.supports_enablejvmciproduct', any('EnableJVMCIProduct' in line for line in out.lines))
return getattr(jdk, '.supports_enablejvmciproduct')
def _find_classes_by_annotated_methods(annotations, dists, jdk=None):
if len(dists) == 0:
return {}
candidates = {}
# Create map from jar file to the binary suite distribution defining it
jarsToDists = {d.classpath_repr(): d for d in dists}
primarySuite = mx.primary_suite()
cachesDir = None
jarsToParse = []
if primarySuite and primarySuite != mx._mx_suite:
cachesDir = mx.ensure_dir_exists(
join(primarySuite.get_output_root(), 'unittest'))
for d in dists:
jar = d.classpath_repr()
testclasses = _read_cached_testclasses(
cachesDir, jar, jdk if jdk else mx.get_jdk())
if testclasses is not None:
for classname in testclasses:
candidates[classname] = jarsToDists[jar]
else:
jarsToParse.append(jar)
if jarsToParse:
# Ensure Java support class is built
mx.build(['--no-daemon', '--dependencies', 'com.oracle.mxtool.junit'])
cp = mx.classpath(['com.oracle.mxtool.junit'] +
list(jarsToDists.values()),
jdk=jdk)
out = mx.LinesOutputCapture()
mx.run_java([
'-cp', cp, 'com.oracle.mxtool.junit.FindClassesByAnnotatedMethods'
] + annotations + jarsToParse,
out=out,
addDefaultArgs=False)
for line in out.lines:
parts = line.split(' ')
jar = parts[0]
reportedclasses = parts[1:] if len(parts) > 1 else []
testclasses = [c for c in reportedclasses if not c.startswith("!")]
excludedclasses = [c for c in reportedclasses if c.startswith("!")]
if cachesDir:
_write_cached_testclasses(cachesDir, jar,
jdk if jdk else mx.get_jdk(),
testclasses, excludedclasses)
for classname in testclasses:
candidates[classname] = jarsToDists[jar]
return candidates
def _read_java_base_hashes(jdk):
"""
Read the hashes stored in the ``java.base`` module of `jdk`.
"""
hashes = {}
out = mx.LinesOutputCapture()
mx.run([jdk.exe_path('jmod'), 'describe', join(jdk.home, 'jmods', 'java.base.jmod')], out=out)
lines = out.lines
for line in lines:
if line.startswith('hashes'):
parts = line.split()
assert len(parts) == 4, 'expected hashes line to have 4 fields, got {} fields: {}'.format(len(parts), line)
_, module_name, algorithm, hash_value = parts
hashes[module_name] = (algorithm, hash_value)
return hashes
def _probe_jvmci_info(jdk, attribute_name):
if not hasattr(jdk, '.enables_jvmci_by_default'):
out = mx.LinesOutputCapture()
sink = lambda x: x
mx.run([jdk.java, '-XX:+UnlockExperimentalVMOptions', '-XX:+PrintFlagsFinal', '-version'], out=out, err=sink)
enableJVMCI = False
enableJVMCIProduct = False
for line in out.lines:
if 'EnableJVMCI' in line and 'true' in line:
enableJVMCI = True
if 'EnableJVMCIProduct' in line:
enableJVMCIProduct = True
setattr(jdk, '.enables_jvmci_by_default', enableJVMCI)
setattr(jdk, '.supports_enablejvmciproduct', enableJVMCIProduct)
return getattr(jdk, attribute_name)
def config_name(self):
suffix = ''
output_capture = mx.LinesOutputCapture()
image_path = os.path.join(mx_vm.graalvm_home(fatalIfMissing=False),
'bin', 'native-image')
if os.path.exists(image_path):
mx.run([image_path, '--version'],
out=output_capture,
err=output_capture,
cwd=None,
nonZeroIsFatal=True)
assert len(output_capture.lines
) == 1, 'Should be one line in a version got: ' + str(
output_capture.lines)
suffix = 'ee' if mx_vm.has_component('svmee',
stage1=True) else 'ce'
return super(NativeImageVM, self).config_name() + '-' + suffix
def gate_body(args, tasks):
with Task('Vm: Basic GraalVM Tests', tasks,
tags=[VmGateTasks.compiler]) as t:
if t and mx_vm.has_component('GraalVM compiler'):
# 1. the build must be a GraalVM
# 2. the build must be JVMCI-enabled since the 'GraalVM compiler' component is registered
mx_vm.check_versions(
mx_vm.graalvm_output(),
graalvm_version_regex=mx_vm.graalvm_version_regex,
expect_graalvm=True,
check_jvmci=True)
with Task('Vm: GraalVM dist names', tasks,
tags=[VmGateTasks.integration]) as t:
if t:
for suite, env_file_name, graalvm_dist_name in env_tests:
out = mx.LinesOutputCapture()
mx.run_mx([
'--no-warning', '--env', env_file_name, 'graalvm-dist-name'
],
suite,
out=out,
err=out,
env={})
mx.log(
"Checking that the env file '{}' in suite '{}' produces a GraalVM distribution named '{}'"
.format(env_file_name, suite.name, graalvm_dist_name))
if len(out.lines) != 1 or out.lines[0] != graalvm_dist_name:
mx.abort(
"Unexpected GraalVM dist name for env file '{}' in suite '{}'.\nExpected: '{}', actual: '{}'.\nDid you forget to update the registration of the GraalVM config?"
.format(env_file_name, suite.name, graalvm_dist_name,
'\n'.join(out.lines)))
if mx_vm.has_component('LibGraal'):
libgraal_location = mx_vm.get_native_image_locations(
'LibGraal', 'jvmcicompiler')
if libgraal_location is None:
mx.warn(
"Skipping libgraal tests: no library enabled in the LibGraal component"
)
else:
extra_vm_arguments = [
'-XX:+UseJVMCICompiler', '-XX:+UseJVMCINativeLibrary',
'-XX:JVMCILibPath=' + dirname(libgraal_location)
]
if args.extra_vm_argument:
extra_vm_arguments += args.extra_vm_argument
import mx_compiler
# run avrora on the GraalVM binary itself
with Task('LibGraal Compiler:GraalVM DaCapo-avrora',
tasks,
tags=[VmGateTasks.libgraal]) as t:
if t:
mx.run([
join(mx_vm.graalvm_home(), 'bin',
'java'), '-XX:+UseJVMCICompiler',
'-XX:+UseJVMCINativeLibrary', '-jar',
mx.library('DACAPO').get_path(True), 'avrora'
])
with Task('LibGraal Compiler:CTW',
tasks,
tags=[VmGateTasks.libgraal]) as t:
if t:
mx_compiler.ctw([
'-DCompileTheWorld.Config=Inline=false CompilationFailureAction=ExitVM',
'-esa',
'-XX:+EnableJVMCI',
'-DCompileTheWorld.MultiThreaded=true',
'-Dgraal.InlineDuringParsing=false',
'-Dgraal.TrackNodeSourcePosition=true',
'-DCompileTheWorld.Verbose=false',
'-XX:ReservedCodeCacheSize=300m',
], extra_vm_arguments)
mx_compiler.compiler_gate_benchmark_runner(
tasks, extra_vm_arguments, prefix='LibGraal Compiler:')
with Task('LibGraal Truffle:unittest',
tasks,
tags=[VmGateTasks.libgraal]) as t:
if t:
def _unittest_config_participant(config):
vmArgs, mainClass, mainClassArgs = config
def is_truffle_fallback(arg):
fallback_args = [
"-Dtruffle.TruffleRuntime=com.oracle.truffle.api.impl.DefaultTruffleRuntime",
"-Dgraalvm.ForcePolyglotInvalid=true"
]
return arg in fallback_args
newVmArgs = [
arg for arg in vmArgs
if not is_truffle_fallback(arg)
]
return (newVmArgs, mainClass, mainClassArgs)
mx_unittest.add_config_participant(
_unittest_config_participant)
excluded_tests = environ.get("TEST_LIBGRAAL_EXCLUDE")
if excluded_tests:
with NamedTemporaryFile(prefix='blacklist.',
mode='w',
delete=False) as fp:
fp.file.writelines(
[l + '\n' for l in excluded_tests.split()])
unittest_args = ["--blacklist", fp.name]
else:
unittest_args = []
unittest_args = unittest_args + [
"--enable-timing", "--verbose"
]
mx_unittest.unittest(unittest_args + extra_vm_arguments + [
"-Dgraal.TruffleCompileImmediately=true",
"-Dgraal.TruffleBackgroundCompilation=false", "truffle"
])
else:
mx.warn("Skipping libgraal tests: component not enabled")
gate_substratevm(tasks)
gate_sulong(tasks)
gate_ruby(tasks)
gate_python(tasks)
gate_svm_truffle_tck_js(tasks)
def jlink_new_jdk(jdk, dst_jdk_dir, module_dists, root_module_names=None, missing_export_target_action='create', with_source=lambda x: True):
"""
Uses jlink from `jdk` to create a new JDK image in `dst_jdk_dir` with `module_dists` and
their dependencies added to the JDK image, replacing any existing modules of the same name.
:param JDKConfig jdk: source JDK
:param str dst_jdk_dir: path to use for the jlink --output option
:param list module_dists: list of distributions defining modules
:param list root_module_names: list of strings naming the module root set for the new JDK image.
The named modules must either be in `module_dists` or in `jdk`. If None, then
the root set will be all the modules in ``module_dists` and `jdk`.
:param str missing_export_target_action: the action to perform for a qualifed export target that
is not present in `module_dists` and does not have a hash stored in java.base.
The choices are:
"create" - an empty module is created
"error" - raise an error
None - do nothing
:param lambda with_source: returns True if the sources of a module distribution must be included in the new JDK
"""
assert callable(with_source)
if jdk.javaCompliance < '9':
mx.abort('Cannot derive a new JDK from ' + jdk.home + ' with jlink since it is not JDK 9 or later')
exploded_java_base_module = join(jdk.home, 'modules', 'java.base')
if exists(exploded_java_base_module):
mx.abort('Cannot derive a new JDK from ' + jdk.home + ' since it appears to be a developer build with exploded modules')
jimage = join(jdk.home, 'lib', 'modules')
jmods_dir = join(jdk.home, 'jmods')
if not isfile(jimage):
mx.abort('Cannot derive a new JDK from ' + jdk.home + ' since ' + jimage + ' is missing or is not an ordinary file')
if not isdir(jmods_dir):
mx.abort('Cannot derive a new JDK from ' + jdk.home + ' since ' + jmods_dir + ' is missing or is not a directory')
jdk_modules = {jmd.name : jmd for jmd in jdk.get_modules()}
modules = [as_java_module(dist, jdk) for dist in module_dists]
all_module_names = frozenset(list(jdk_modules.keys()) + [m.name for m in modules])
# Read hashes stored in java.base (the only module in the JDK where hashes are stored)
out = mx.LinesOutputCapture()
mx.run([jdk.exe_path('jmod'), 'describe', jdk_modules['java.base'].get_jmod_path()], out=out)
lines = out.lines
hashes = {}
for line in lines:
if line.startswith('hashes'):
parts = line.split()
assert len(parts) == 4, 'expected hashes line to have 4 fields, got {} fields: {}'.format(len(parts), line)
_, module_name, algorithm, hash_value = parts
hashes[module_name] = (algorithm, hash_value)
build_dir = mx.ensure_dir_exists(join(dst_jdk_dir + ".build"))
try:
# Handle targets of qualified exports that are not present in `modules`
target_requires = {}
for jmd in modules:
for targets in jmd.exports.values():
for target in targets:
if target not in all_module_names and target not in hashes:
target_requires.setdefault(target, set()).add(jmd.name)
if target_requires and missing_export_target_action is not None:
if missing_export_target_action == 'error':
mx.abort('Target(s) of qualified exports cannot be resolved: ' + '.'.join(target_requires.keys()))
assert missing_export_target_action == 'create', 'invalid value for missing_export_target_action: ' + str(missing_export_target_action)
extra_modules = []
for name, requires in target_requires.items():
module_jar = join(build_dir, name + '.jar')
jmd = JavaModuleDescriptor(name, {}, requires={module : [] for module in requires}, uses=set(), provides={}, jarpath=module_jar)
extra_modules.append(jmd)
module_build_dir = mx.ensure_dir_exists(join(build_dir, name))
module_info_java = join(module_build_dir, 'module-info.java')
module_info_class = join(module_build_dir, 'module-info.class')
with open(module_info_java, 'w') as fp:
print(jmd.as_module_info(), file=fp)
mx.run([jdk.javac, '-d', module_build_dir, \
'--limit-modules=java.base,' + ','.join(jmd.requires.keys()), \
'--module-path=' + os.pathsep.join((m.jarpath for m in modules)), \
module_info_java])
with ZipFile(module_jar, 'w') as zf:
zf.write(module_info_class, basename(module_info_class))
if exists(jmd.get_jmod_path()):
os.remove(jmd.get_jmod_path())
mx.run([jdk.javac.replace('javac', 'jmod'), 'create', '--class-path=' + module_build_dir, jmd.get_jmod_path()])
modules.extend(extra_modules)
all_module_names = frozenset(list(jdk_modules.keys()) + [m.name for m in modules])
# Extract src.zip from source JDK
jdk_src_zip = join(jdk.home, 'lib', 'src.zip')
dst_src_zip_contents = {}
if isfile(jdk_src_zip):
mx.logv('[Extracting ' + jdk_src_zip + ']')
with ZipFile(jdk_src_zip, 'r') as zf:
for name in zf.namelist():
if not name.endswith('/'):
dst_src_zip_contents[name] = zf.read(name)
else:
mx.warn("'{}' does not exist or is not a file".format(jdk_src_zip))
for jmd in modules:
# Remove existing sources for all the modules that we include
dst_src_zip_contents = {key : dst_src_zip_contents[key] for key in dst_src_zip_contents if not key.startswith(jmd.name)}
if with_source(jmd.dist):
# Add the sources that we can share.
# Extract module sources
jmd_src_zip = jmd.jarpath[0:-len('.jar')] + '.src.zip'
if isfile(jmd_src_zip):
mx.logv('[Extracting ' + jmd_src_zip + ']')
with ZipFile(jmd_src_zip, 'r') as zf:
for name in zf.namelist():
if not name.endswith('/'):
dst_src_zip_contents[jmd.name + '/' + name] = zf.read(name)
# Add module-info.java to sources
dst_src_zip_contents[jmd.name + '/module-info.java'] = jmd.as_module_info(extras_as_comments=False)
# Now build the new JDK image with jlink
jlink = [jdk.javac.replace('javac', 'jlink')]
if jdk_enables_jvmci_by_default(jdk):
# On JDK 9+, +EnableJVMCI forces jdk.internal.vm.ci to be in the root set
jlink.append('-J-XX:-EnableJVMCI')
if root_module_names is not None:
missing = frozenset(root_module_names) - all_module_names
if missing:
mx.abort('Invalid module(s): {}.\nAvailable modules: {}'.format(','.join(missing), ','.join(sorted(all_module_names))))
jlink.append('--add-modules=' + ','.join(root_module_names))
else:
jlink.append('--add-modules=' + ','.join(sorted(all_module_names)))
module_path = jmods_dir
if modules:
module_path = os.pathsep.join((m.get_jmod_path(respect_stripping=True) for m in modules)) + os.pathsep + module_path
jlink.append('--module-path=' + module_path)
jlink.append('--output=' + dst_jdk_dir)
# These options are inspired by how OpenJDK runs jlink to produce the final runtime image.
jlink.extend(['-J-XX:+UseSerialGC', '-J-Xms32M', '-J-Xmx512M', '-J-XX:TieredStopAtLevel=1'])
jlink.append('-J-Dlink.debug=true')
jlink.append('--dedup-legal-notices=error-if-not-same-content')
jlink.append('--keep-packaged-modules=' + join(dst_jdk_dir, 'jmods'))
# TODO: investigate the options below used by OpenJDK to see if they should be used:
# --release-info: this allow extra properties to be written to the <jdk>/release file
# --order-resources: specifies order of resources in generated lib/modules file.
# This is apparently not so important if a CDS archive is available.
# --generate-jli-classes: pre-generates a set of java.lang.invoke classes.
# See https://github.com/openjdk/jdk/blob/master/make/GenerateLinkOptData.gmk
mx.logv('[Creating JDK image]')
mx.run(jlink)
dst_src_zip = join(dst_jdk_dir, 'lib', 'src.zip')
mx.logv('[Creating ' + dst_src_zip + ']')
with ZipFile(dst_src_zip, 'w', compression=ZIP_DEFLATED, allowZip64=True) as zf:
for name, contents in sorted(dst_src_zip_contents.items()):
zf.writestr(name, contents)
mx.logv('[Copying static libraries]')
lib_prefix = mx.add_lib_prefix('')
lib_suffix = '.lib' if mx.is_windows() else '.a'
lib_directory = join(jdk.home, 'lib')
dst_lib_directory = join(dst_jdk_dir, 'lib')
for f in os.listdir(lib_directory):
if f.startswith(lib_prefix) and f.endswith(lib_suffix):
lib_path = join(lib_directory, f)
if isfile(lib_path):
shutil.copy2(lib_path, dst_lib_directory)
# Build the list of modules whose classes might have annotations
# to be processed by native-image (GR-15192).
with open(join(dst_jdk_dir, 'lib', 'native-image-modules.list'), 'w') as fp:
print('# Modules whose classes might have annotations processed by native-image', file=fp)
for m in modules:
print(m.name, file=fp)
finally:
if not mx.get_opts().verbose:
# Preserve build directory so that javac command can be re-executed
# by cutting and pasting verbose output.
shutil.rmtree(build_dir)
# Create CDS archive (https://openjdk.java.net/jeps/341).
out = mx.OutputCapture()
mx.logv('[Creating CDS shared archive]')
if mx.run([mx.exe_suffix(join(dst_jdk_dir, 'bin', 'java')), '-Xshare:dump', '-Xmx128M', '-Xms128M'], out=out, err=out, nonZeroIsFatal=False) != 0:
mx.log(out.data)
mx.abort('Error generating CDS shared archive')
def jlink_new_jdk(jdk, dst_jdk_dir, module_dists,
root_module_names=None,
missing_export_target_action='create',
with_source=lambda x: True,
vendor_info=None,
dedup_legal_notices=True):
"""
Uses jlink from `jdk` to create a new JDK image in `dst_jdk_dir` with `module_dists` and
their dependencies added to the JDK image, replacing any existing modules of the same name.
:param JDKConfig jdk: source JDK
:param str dst_jdk_dir: path to use for the jlink --output option
:param list module_dists: list of distributions defining modules
:param list root_module_names: list of strings naming the module root set for the new JDK image.
The named modules must either be in `module_dists` or in `jdk`. If None, then
the root set will be all the modules in ``module_dists` and `jdk`.
:param str missing_export_target_action: the action to perform for a qualifed export target that
is not present in `module_dists` and does not have a hash stored in java.base.
The choices are:
"create" - an empty module is created
"error" - raise an error
None - do nothing
:param lambda with_source: returns True if the sources of a module distribution must be included in the new JDK
:param dict vendor_info: values for the jlink vendor options added by JDK-8232080
"""
assert callable(with_source)
if jdk.javaCompliance < '9':
mx.abort('Cannot derive a new JDK from ' + jdk.home + ' with jlink since it is not JDK 9 or later')
exploded_java_base_module = join(jdk.home, 'modules', 'java.base')
if exists(exploded_java_base_module):
mx.abort('Cannot derive a new JDK from ' + jdk.home + ' since it appears to be a developer build with exploded modules')
jimage = join(jdk.home, 'lib', 'modules')
jmods_dir = join(jdk.home, 'jmods')
if not isfile(jimage):
mx.abort('Cannot derive a new JDK from ' + jdk.home + ' since ' + jimage + ' is missing or is not an ordinary file')
if not isdir(jmods_dir):
mx.abort('Cannot derive a new JDK from ' + jdk.home + ' since ' + jmods_dir + ' is missing or is not a directory')
# Exclude jdk.aot due to GR-10545 and JDK-8255616
jdk_modules = {jmd.name: jmd for jmd in jdk.get_modules() if jmd.name != 'jdk.aot'}
modules = [as_java_module(dist, jdk) for dist in module_dists]
all_module_names = frozenset(list(jdk_modules.keys()) + [m.name for m in modules])
# Read hashes stored in java.base (the only module in the JDK where hashes are stored)
out = mx.LinesOutputCapture()
mx.run([jdk.exe_path('jmod'), 'describe', jdk_modules['java.base'].get_jmod_path()], out=out)
lines = out.lines
hashes = {}
for line in lines:
if line.startswith('hashes'):
parts = line.split()
assert len(parts) == 4, 'expected hashes line to have 4 fields, got {} fields: {}'.format(len(parts), line)
_, module_name, algorithm, hash_value = parts
hashes[module_name] = (algorithm, hash_value)
build_dir = mx.ensure_dir_exists(join(dst_jdk_dir + ".build"))
try:
# Handle targets of qualified exports that are not present in `modules`
target_requires = {}
for jmd in modules:
for targets in jmd.exports.values():
for target in targets:
if target not in all_module_names and target not in hashes:
target_requires.setdefault(target, set()).add(jmd.name)
if target_requires and missing_export_target_action is not None:
if missing_export_target_action == 'error':
mx.abort('Target(s) of qualified exports cannot be resolved: ' + '.'.join(target_requires.keys()))
assert missing_export_target_action == 'create', 'invalid value for missing_export_target_action: ' + str(missing_export_target_action)
extra_modules = []
for name, requires in target_requires.items():
module_jar = join(build_dir, name + '.jar')
jmd = JavaModuleDescriptor(name, {}, requires={module: [] for module in requires}, uses=set(), provides={}, jarpath=module_jar)
extra_modules.append(jmd)
module_build_dir = mx.ensure_dir_exists(join(build_dir, name))
module_info_java = join(module_build_dir, 'module-info.java')
module_info_class = join(module_build_dir, 'module-info.class')
with open(module_info_java, 'w') as fp:
print(jmd.as_module_info(), file=fp)
mx.run([jdk.javac, '-d', module_build_dir,
'--limit-modules=java.base,' + ','.join(jmd.requires.keys()),
'--module-path=' + os.pathsep.join((m.jarpath for m in modules)),
module_info_java])
with ZipFile(module_jar, 'w') as zf:
zf.write(module_info_class, basename(module_info_class))
if exists(jmd.get_jmod_path()):
os.remove(jmd.get_jmod_path())
mx.run([jdk.javac.replace('javac', 'jmod'), 'create', '--class-path=' + module_build_dir, jmd.get_jmod_path()])
modules.extend(extra_modules)
all_module_names = frozenset(list(jdk_modules.keys()) + [m.name for m in modules])
# Extract src.zip from source JDK
jdk_src_zip = join(jdk.home, 'lib', 'src.zip')
dst_src_zip_contents = {}
if isfile(jdk_src_zip):
mx.logv('[Extracting ' + jdk_src_zip + ']')
with ZipFile(jdk_src_zip, 'r') as zf:
for name in zf.namelist():
if not name.endswith('/'):
dst_src_zip_contents[name] = zf.read(name)
else:
mx.warn("'{}' does not exist or is not a file".format(jdk_src_zip))
# Edit lib/security/default.policy in java.base
patched_java_base = join(build_dir, 'java.base.jmod')
with open(join(jmods_dir, 'java.base.jmod'), 'rb') as src_f, open(patched_java_base, 'wb') as dst_f:
jmod_header = src_f.read(4)
if len(jmod_header) != 4 or jmod_header != b'JM\x01\x00':
raise mx.abort("Unexpected jmod header: " + b2a_hex(jmod_header).decode('ascii'))
dst_f.write(jmod_header)
policy_result = 'not found'
with ZipFile(src_f, 'r') as src_zip, ZipFile(dst_f, 'w', src_zip.compression) as dst_zip:
for i in src_zip.infolist():
if i.filename[-1] == '/':
continue
src_member = src_zip.read(i)
if i.filename == 'lib/security/default.policy':
policy_result = 'unmodified'
if 'grant codeBase "jrt:/com.oracle.graal.graal_enterprise"'.encode('utf-8') not in src_member:
policy_result = 'modified'
src_member += """
grant codeBase "jrt:/com.oracle.graal.graal_enterprise" {
permission java.security.AllPermission;
};
""".encode('utf-8')
if 'grant codeBase "jrt:/org.graalvm.truffle"'.encode('utf-8') not in src_member:
policy_result = 'modified'
src_member += """
grant codeBase "jrt:/org.graalvm.truffle" {
permission java.security.AllPermission;
};
grant codeBase "jrt:/org.graalvm.sdk" {
permission java.security.AllPermission;
};
grant codeBase "jrt:/org.graalvm.locator" {
permission java.io.FilePermission "<<ALL FILES>>", "read";
permission java.util.PropertyPermission "*", "read,write";
permission java.lang.RuntimePermission "createClassLoader";
permission java.lang.RuntimePermission "getClassLoader";
permission java.lang.RuntimePermission "getenv.*";
};
grant codeBase "file:${java.home}/languages/-" {
permission java.security.AllPermission;
};
""".encode('utf-8')
dst_zip.writestr(i, src_member)
if policy_result == 'not found':
raise mx.abort("Couldn't find `lib/security/default.policy` in " + join(jmods_dir, 'java.base.jmod'))
for jmd in modules:
# Remove existing sources for all the modules that we include
dst_src_zip_contents = {key: dst_src_zip_contents[key] for key in dst_src_zip_contents if not key.startswith(jmd.name)}
if with_source(jmd.dist):
# Add the sources that we can share.
# Extract module sources
jmd_src_zip = jmd.jarpath[0:-len('.jar')] + '.src.zip'
if isfile(jmd_src_zip):
mx.logv('[Extracting ' + jmd_src_zip + ']')
with ZipFile(jmd_src_zip, 'r') as zf:
for name in zf.namelist():
if not name.endswith('/'):
dst_src_zip_contents[jmd.name + '/' + name] = zf.read(name)
# Add module-info.java to sources
dst_src_zip_contents[jmd.name + '/module-info.java'] = jmd.as_module_info(extras_as_comments=False)
# Now build the new JDK image with jlink
jlink = [jdk.javac.replace('javac', 'jlink')]
if jdk_enables_jvmci_by_default(jdk):
# On JDK 9+, +EnableJVMCI forces jdk.internal.vm.ci to be in the root set
jlink += ['-J-XX:-EnableJVMCI', '-J-XX:-UseJVMCICompiler']
if root_module_names is not None:
missing = frozenset(root_module_names) - all_module_names
if missing:
mx.abort('Invalid module(s): {}.\nAvailable modules: {}'.format(','.join(missing), ','.join(sorted(all_module_names))))
jlink.append('--add-modules=' + ','.join(root_module_names))
else:
jlink.append('--add-modules=' + ','.join(sorted(all_module_names)))
module_path = patched_java_base + os.pathsep + jmods_dir
if modules:
module_path = os.pathsep.join((m.get_jmod_path(respect_stripping=True) for m in modules)) + os.pathsep + module_path
jlink.append('--module-path=' + module_path)
jlink.append('--output=' + dst_jdk_dir)
# These options are derived from how OpenJDK runs jlink to produce the final runtime image.
jlink.extend(['-J-XX:+UseSerialGC', '-J-Xms32M', '-J-Xmx512M', '-J-XX:TieredStopAtLevel=1'])
jlink.append('-J-Dlink.debug=true')
if dedup_legal_notices:
jlink.append('--dedup-legal-notices=error-if-not-same-content')
jlink.append('--keep-packaged-modules=' + join(dst_jdk_dir, 'jmods'))
if jdk_has_new_jlink_options(jdk):
if jdk_omits_warning_for_jlink_set_ThreadPriorityPolicy(jdk):
thread_priority_policy_option = ' -XX:ThreadPriorityPolicy=1'
else:
mx.logv('[Creating JDK without -XX:ThreadPriorityPolicy=1]')
thread_priority_policy_option = ''
if jdk_supports_enablejvmciproduct(jdk):
if any((m.name == 'jdk.internal.vm.compiler' for m in modules)):
jlink.append('--add-options=-XX:+UnlockExperimentalVMOptions -XX:+EnableJVMCIProduct -XX:-UnlockExperimentalVMOptions' + thread_priority_policy_option)
else:
# Don't default to using JVMCI as JIT unless Graal is being updated in the image.
# This avoids unexpected issues with using the out-of-date Graal compiler in
# the JDK itself.
jlink.append('--add-options=-XX:+UnlockExperimentalVMOptions -XX:+EnableJVMCIProduct -XX:-UseJVMCICompiler -XX:-UnlockExperimentalVMOptions' + thread_priority_policy_option)
else:
mx.logv('[Creating JDK without -XX:+EnableJVMCIProduct]')
if thread_priority_policy_option:
jlink.append('--add-options=' + thread_priority_policy_option.strip())
if vendor_info is not None:
for name, value in vendor_info.items():
jlink.append('--' + name + '=' + value)
release_file = join(jdk.home, 'release')
if isfile(release_file):
jlink.append('--release-info=' + release_file)
# TODO: investigate the options below used by OpenJDK to see if they should be used:
# --order-resources: specifies order of resources in generated lib/modules file.
# This is apparently not so important if a CDS archive is available.
# --generate-jli-classes: pre-generates a set of java.lang.invoke classes.
# See https://github.com/openjdk/jdk/blob/master/make/GenerateLinkOptData.gmk
mx.logv('[Creating JDK image in {}]'.format(dst_jdk_dir))
mx.run(jlink)
dst_src_zip = join(dst_jdk_dir, 'lib', 'src.zip')
mx.logv('[Creating ' + dst_src_zip + ']')
with ZipFile(dst_src_zip, 'w', compression=ZIP_DEFLATED, allowZip64=True) as zf:
for name, contents in sorted(dst_src_zip_contents.items()):
zf.writestr(name, contents)
mx.logv('[Copying static libraries]')
lib_directory = join(jdk.home, 'lib', 'static')
if exists(lib_directory):
dst_lib_directory = join(dst_jdk_dir, 'lib', 'static')
try:
mx.copytree(lib_directory, dst_lib_directory)
except shutil.Error as e:
# On AArch64, there can be a problem in the copystat part
# of copytree which occurs after file and directory copying
# has successfully completed. Since the metadata doesn't
# matter in this case, just ensure that the content was copied.
for root, _, lib_files in os.walk(lib_directory):
relative_root = os.path.relpath(root, dst_lib_directory)
for lib in lib_files:
src_lib_path = join(root, lib)
dst_lib_path = join(dst_lib_directory, relative_root, lib)
if not exists(dst_lib_path):
mx.abort('Error copying static libraries: {} missing in {}{}Original copytree error: {}'.format(
join(relative_root, lib), dst_lib_directory, os.linesep, e))
src_lib_hash = mx.sha1OfFile(src_lib_path)
dst_lib_hash = mx.sha1OfFile(dst_lib_path)
if src_lib_hash != dst_lib_hash:
mx.abort('Error copying static libraries: {} (hash={}) and {} (hash={}) differ{}Original copytree error: {}'.format(
src_lib_path, src_lib_hash,
dst_lib_path, dst_lib_hash,
os.linesep, e))
# Allow older JDK versions to work
else:
lib_prefix = mx.add_lib_prefix('')
lib_suffix = mx.add_static_lib_suffix('')
lib_directory = join(jdk.home, 'lib')
dst_lib_directory = join(dst_jdk_dir, 'lib')
for f in os.listdir(lib_directory):
if f.startswith(lib_prefix) and f.endswith(lib_suffix):
lib_path = join(lib_directory, f)
if isfile(lib_path):
shutil.copy2(lib_path, dst_lib_directory)
finally:
if not mx.get_opts().verbose:
# Preserve build directory so that javac command can be re-executed
# by cutting and pasting verbose output.
shutil.rmtree(build_dir)
# Create CDS archive (https://openjdk.java.net/jeps/341).
out = mx.OutputCapture()
mx.logv('[Creating CDS shared archive]')
if mx.run([mx.exe_suffix(join(dst_jdk_dir, 'bin', 'java')), '-Xshare:dump', '-Xmx128M', '-Xms128M'], out=out, err=out, nonZeroIsFatal=False) != 0:
mx.log(out.data)
mx.abort('Error generating CDS shared archive')
def makegraaljdk(args):
"""make a JDK with Graal as the default top level JIT"""
parser = ArgumentParser(prog='mx makegraaljdk')
parser.add_argument('-f', '--force', action='store_true', help='overwrite existing GraalJDK')
parser.add_argument('-a', '--archive', action='store', help='name of archive to create', metavar='<path>')
parser.add_argument('dest', help='destination directory for GraalJDK', metavar='<path>')
args = parser.parse_args(args)
if isJDK8:
dstJdk = os.path.abspath(args.dest)
srcJdk = jdk.home
if exists(dstJdk):
if args.force:
shutil.rmtree(dstJdk)
else:
mx.abort('Use --force to overwrite existing directory ' + dstJdk)
mx.log('Creating {} from {}'.format(dstJdk, srcJdk))
shutil.copytree(srcJdk, dstJdk)
bootDir = mx.ensure_dir_exists(join(dstJdk, 'jre', 'lib', 'boot'))
jvmciDir = join(dstJdk, 'jre', 'lib', 'jvmci')
assert exists(jvmciDir), jvmciDir + ' does not exist'
if mx.get_os() == 'darwin' or mx.get_os() == 'windows':
jvmlibDir = join(dstJdk, 'jre', 'lib', 'server')
else:
jvmlibDir = join(dstJdk, 'jre', 'lib', mx.get_arch(), 'server')
jvmlib = join(jvmlibDir, mx.add_lib_prefix(mx.add_lib_suffix('jvm')))
assert exists(jvmlib), jvmlib + ' does not exist'
with open(join(jvmciDir, 'compiler-name'), 'w') as fp:
print >> fp, 'graal'
vmName = 'Graal'
mapFiles = set()
for e in _jvmci_classpath:
src = basename(e.get_path())
mx.log('Copying {} to {}'.format(e.get_path(), jvmciDir))
candidate = e.get_path() + '.map'
if exists(candidate):
mapFiles.add(candidate)
with open(join(dstJdk, 'release'), 'a') as fp:
d = e.dist()
s = d.suite
print >> fp, '{}={}'.format(d.name, s.vc.parent(s.dir))
vmName = vmName + ':' + s.name + '_' + s.version()
shutil.copyfile(e.get_path(), join(jvmciDir, src))
for e in _bootclasspath_appends:
src = basename(e.classpath_repr())
mx.log('Copying {} to {}'.format(e.classpath_repr(), bootDir))
candidate = e.classpath_repr() + '.map'
if exists(candidate):
mapFiles.add(candidate)
with open(join(dstJdk, 'release'), 'a') as fp:
s = e.suite
print >> fp, '{}={}'.format(e.name, s.vc.parent(s.dir))
shutil.copyfile(e.classpath_repr(), join(bootDir, src))
out = mx.LinesOutputCapture()
mx.run([jdk.java, '-version'], err=out)
line = None
pattern = re.compile(r'(.* )(?:Server|Graal) VM \(build.*')
for line in out.lines:
m = pattern.match(line)
if m:
with open(join(jvmlibDir, 'vm.properties'), 'w') as fp:
# Modify VM name in `java -version` to be Graal along
# with a suffix denoting the commit of each Graal jar.
# For example:
# Java HotSpot(TM) 64-Bit Graal:compiler_88847fb25d1a62977a178331a5e78fa5f8fcbb1a (build 25.71-b01-internal-jvmci-0.34, mixed mode)
print >> fp, 'name=' + m.group(1) + vmName
line = True
break
if line is not True:
mx.abort('Could not find "{}" in output of `java -version`:\n{}'.format(pattern.pattern, os.linesep.join(out.lines)))
exe = join(dstJdk, 'bin', mx.exe_suffix('java'))
with StdoutUnstripping(args=[], out=None, err=None, mapFiles=mapFiles) as u:
mx.run([exe, '-XX:+BootstrapJVMCI', '-version'], out=u.out, err=u.err)
if args.archive:
mx.log('Archiving {}'.format(args.archive))
create_archive(dstJdk, args.archive, basename(args.dest) + '/')
else:
mx.abort('Can only make GraalJDK for JDK 8 currently')
def get_library_as_module(dep, jdk):
"""
Converts a (modular or non-modular) jar library to a module descriptor.
:param Library dep: a library dependency
:param JDKConfig jdk: a JDK with a version >= 9 that can be used to describe the module
:return: a module descriptor
"""
assert dep.isLibrary()
def is_valid_module_name(name):
identRE = re.compile(r"^[A-Za-z][A-Za-z0-9]*$")
return all(identRE.match(ident) for ident in name.split('.'))
if hasattr(dep, 'moduleName'):
moduleName = dep.moduleName
else:
moduleName = jdk.get_automatic_module_name(dep.path)
if not is_valid_module_name(moduleName):
mx.abort(
"Invalid identifier in automatic module name derived for library {}: {} (path: {})"
.format(dep.name, moduleName, dep.path))
dep.moduleName = moduleName
modulesDir = mx.ensure_dir_exists(
join(mx.primary_suite().get_output_root(), 'modules'))
cache = join(modulesDir, moduleName + '.desc')
fullpath = dep.get_path(resolve=True)
save = False
if not exists(cache) or mx.TimeStampFile(fullpath).isNewerThan(
cache) or mx.TimeStampFile(__file__).isNewerThan(cache):
out = mx.LinesOutputCapture()
rc = mx.run([
jdk.java, '--module-path', fullpath, '--describe-module',
moduleName
],
out=out,
err=out,
nonZeroIsFatal=False)
lines = out.lines
if rc != 0:
mx.abort(
"java --describe-module {} failed. Please verify the moduleName attribute of {}.\n{}"
.format(moduleName, dep.name, "\n".join(lines)))
save = True
else:
with open(cache) as fp:
lines = fp.read().splitlines()
assert lines and lines[0].startswith(moduleName), (dep.name, moduleName,
lines)
accepted_modifiers = set(['transitive'])
requires = {}
exports = {}
provides = {}
uses = set()
packages = set()
for line in lines[1:]:
parts = line.strip().split()
assert len(parts) >= 2, '>>>' + line + '<<<'
if parts[0:2] == ['qualified', 'exports']:
parts = parts[1:]
a = parts[0]
if a == 'requires':
module = parts[1]
modifiers = parts[2:]
requires[module] = set(m for m in modifiers
if m in accepted_modifiers)
elif a == 'exports':
source = parts[1]
if len(parts) > 2:
assert parts[2] == 'to'
targets = parts[3:]
else:
targets = []
exports[source] = targets
elif a == 'uses':
uses.update(parts[1:])
elif a == 'contains':
packages.update(parts[1:])
elif a == 'provides':
assert len(parts) >= 4 and parts[2] == 'with'
service = parts[1]
providers = parts[3:]
provides.setdefault(service, []).extend(providers)
else:
mx.abort('Cannot parse module descriptor line: ' + str(parts))
packages.update(exports.keys())
if save:
try:
with open(cache, 'w') as fp:
fp.write('\n'.join(lines) + '\n')
except IOError as e:
mx.warn('Error writing to ' + cache + ': ' + str(e))
os.remove(cache)
return JavaModuleDescriptor(moduleName,
exports,
requires,
uses,
provides,
packages,
jarpath=fullpath)
