def _espresso_command(launcher, args):
import mx_sdk_vm_impl
bin_dir = join(mx_sdk_vm_impl.graalvm_home(fatalIfMissing=True), 'bin')
exe = join(bin_dir, mx.exe_suffix(launcher))
if not os.path.exists(exe):
exe = join(bin_dir, mx.cmd_suffix(launcher))
return [exe] + args
def java_base_unittest(args):
"""tests whether graal compiler runs on JDK9 with limited set of modules"""
jlink = mx.exe_suffix(join(jdk.home, 'bin', 'jlink'))
if not exists(jlink):
raise mx.JDKConfigException('jlink tool does not exist: ' + jlink)
basejdk_dir = join(_suite.get_output_root(), 'jdkbase')
basemodules = 'java.base,jdk.internal.vm.ci,jdk.unsupported'
if exists(basejdk_dir):
shutil.rmtree(basejdk_dir)
mx.run([jlink, '--output', basejdk_dir, '--add-modules', basemodules, '--module-path', join(jdk.home, 'jmods')])
jdwp = mx.add_lib_suffix(mx.add_lib_prefix('jdwp'))
shutil.copy(join(jdk.home, 'lib', jdwp), join(basejdk_dir, 'lib', jdwp))
dt_socket = mx.add_lib_suffix(mx.add_lib_prefix('dt_socket'))
shutil.copy(join(jdk.home, 'lib', dt_socket), join(basejdk_dir, 'lib', dt_socket))
if not args:
args = []
fakeJavac = join(basejdk_dir, 'bin', 'javac')
open(fakeJavac, 'a').close()
basejdk = mx.JDKConfig(basejdk_dir)
savedJava = jdk.java
try:
jdk.java = basejdk.java
if mx_gate.Task.verbose:
extra_args = ['--verbose', '--enable-timing']
else:
extra_args = []
mx_unittest.unittest(['--suite', 'compiler', '--fail-fast'] + extra_args + args)
finally:
jdk.java = savedJava
def jdk_omits_warning_for_jlink_set_ThreadPriorityPolicy(jdk): # pylint: disable=invalid-name
"""
Determines if the `jdk` suppresses a warning about ThreadPriorityPolicy when it
is non-zero if the value is set from the jimage.
https://bugs.openjdk.java.net/browse/JDK-8235908.
"""
if not hasattr(jdk, '.omits_ThreadPriorityPolicy_warning'):
out = mx.OutputCapture()
sink = lambda x: x
tmpdir = tempfile.mkdtemp(
prefix='jdk_omits_warning_for_jlink_set_ThreadPriorityPolicy')
jlink_exe = jdk.javac.replace('javac', 'jlink')
mx.run([
jlink_exe, '--add-options=-XX:ThreadPriorityPolicy=1',
'--output=' + join(tmpdir, 'jdk'), '--add-modules=java.base'
])
mx.run([mx.exe_suffix(join(tmpdir, 'jdk', 'bin', 'java')), '-version'],
out=sink,
err=out)
shutil.rmtree(tmpdir)
setattr(
jdk, '.omits_ThreadPriorityPolicy_warning',
'-XX:ThreadPriorityPolicy=1 may require system level permission'
not in out.data)
return getattr(jdk, '.omits_ThreadPriorityPolicy_warning')
def getResults(self):
output_dir = self.getOutputDir()
subdirs = set()
for root, filename in self.getProgramSources():
subdir = os.path.relpath(root, self.getSourceDir())
subdirs.add(subdir)
build_output_name = lambda ext: os.path.join(output_dir, subdir, remove_extension(filename) + ext)
node_build_output_name = lambda ext: os.path.join(output_dir, subdir, NODE_BENCH_DIR, remove_extension(filename) + ext)
native_build_output_name = lambda ext: os.path.join(output_dir, subdir, NATIVE_BENCH_DIR, remove_extension(filename) + ext)
result_path = os.path.join(root, remove_extension(filename) + ".result")
# The result files may be optional in some cases.
if os.path.isfile(result_path):
yield build_output_name(".result")
opts_path = os.path.join(root, remove_extension(filename) + ".opts")
# Some benchmarks may specify custom options.
if os.path.isfile(opts_path):
yield build_output_name(".opts")
# Textual WebAssembly file is included for convenience.
yield build_output_name(".wat")
# A binary WebAssembly file contains the program.
yield build_output_name(".wasm")
if filename.endswith(".c"):
# C-compiled sources generate an initialization file.
yield build_output_name(".init")
if self.isBenchmarkProject():
# The JS file and the WebAssembly binary are used by Node.
yield node_build_output_name(".js")
yield node_build_output_name(".wasm")
# The raw binary is used to run the program directly.
yield native_build_output_name(mx.exe_suffix(""))
for subdir in subdirs:
yield os.path.join(output_dir, subdir, "wasm_test_index")
def get_output_binary_name(self):
if self.type == SVMImageProject.Type.SHARED_LIBRARY:
return mx.add_lib_suffix(self.get_image_builder_binary_name())
elif self.type == SVMImageProject.Type.EXECUTABLE:
return mx.exe_suffix(self.get_image_builder_binary_name())
else:
mx.abort("Unsupported type: " + str(self.type))
def _espresso_java_command(args):
"""Java launcher using libespresso in GraalVM + arguments"""
import mx_sdk_vm_impl
bin_dir = os.path.join(mx_sdk_vm_impl.graalvm_home(fatalIfMissing=True),
'bin')
exe = os.path.join(bin_dir, mx.exe_suffix('java'))
if not os.path.exists(exe):
exe = os.path.join(bin_dir, mx.cmd_suffix('java'))
return [exe, '-truffle'] + args
def _espresso_launcher_command(args):
"""Espresso launcher embedded in GraalVM + arguments"""
import mx_sdk_vm_impl
bin_dir = os.path.join(mx_sdk_vm_impl.graalvm_home(fatalIfMissing=True),
'bin')
exe = os.path.join(bin_dir, mx.exe_suffix('espresso'))
if not os.path.exists(exe):
exe = os.path.join(bin_dir, mx.cmd_suffix('espresso'))
return [exe] + args
def makegraaljdk(args):
"""make a JDK with Graal as the default top level JIT"""
parser = ArgumentParser(prog='mx makegraaljdk')
parser.add_argument('-f',
'--force',
action='store_true',
help='overwrite existing GraalJDK')
parser.add_argument('-a',
'--archive',
action='store',
help='name of archive to create',
metavar='<path>')
parser.add_argument('dest',
help='destination directory for GraalJDK',
metavar='<path>')
args = parser.parse_args(args)
if isJDK8:
dstJdk = os.path.abspath(args.dest)
srcJdk = jdk.home
if exists(dstJdk):
if args.force:
shutil.rmtree(dstJdk)
else:
mx.abort('Use --force to overwrite existing directory ' +
dstJdk)
mx.log('Creating {} from {}'.format(dstJdk, srcJdk))
shutil.copytree(srcJdk, dstJdk)
bootDir = mx.ensure_dir_exists(join(dstJdk, 'jre', 'lib', 'boot'))
jvmciDir = join(dstJdk, 'jre', 'lib', 'jvmci')
assert exists(jvmciDir), jvmciDir + ' does not exist'
with open(join(jvmciDir, 'compiler-name'), 'w') as fp:
print >> fp, 'graal'
for e in _jvmci_classpath:
src = basename(e.get_path())
mx.log('Copying {} to {}'.format(src, jvmciDir))
with open(join(dstJdk, 'release'), 'a') as fp:
d = e.dist()
s = d.suite
print >> fp, '{}={}'.format(d.name, s.vc.parent(s.dir))
shutil.copyfile(e.get_path(), join(jvmciDir, src))
for e in _bootclasspath_appends:
src = basename(e.classpath_repr())
mx.log('Copying {} to {}'.format(src, bootDir))
with open(join(dstJdk, 'release'), 'a') as fp:
s = e.suite
print >> fp, '{}={}'.format(e.name, s.vc.parent(s.dir))
shutil.copyfile(e.classpath_repr(), join(bootDir, src))
exe = join(dstJdk, 'bin', mx.exe_suffix('java'))
mx.run([exe, '-XX:+BootstrapJVMCI', '-version'])
if args.archive:
mx.log('Archiving {}'.format(args.archive))
create_archive(dstJdk, args.archive, basename(args.dest) + '/')
else:
mx.abort('Can only make GraalJDK for JDK 8 currently')
def getResults(self, replaceVar=mx_subst.results_substitutions):
if not self.results:
self.results = []
for t in self.getTests():
t = t + self.getTestDirExt()
if self.buildRef:
self.results.append(
os.path.join(t, mx.exe_suffix('ref.out')))
for v in self.getVariants():
result_file = v + '.so' if self.buildSharedObject else v + '.bc'
self.results.append(os.path.join(t, result_file))
return super(SulongTestSuiteMixin,
self).getResults(replaceVar=replaceVar)
def run_vm(self, args, out=None, err=None, cwd=None, nonZeroIsFatal=False):
jar, suite, benchmark = self.parse_jar_suite_benchmark(args)
tmp_dir = None
try:
mode = self.config_name()
tmp_dir = self.extract_jar_to_tempdir(jar, mode, suite, benchmark)
binary_path = os.path.join(tmp_dir, "bench", suite, mode, mx.exe_suffix(benchmark))
os.chmod(binary_path, stat.S_IRUSR | stat.S_IXUSR)
cmd_line = [binary_path]
mx.log("Running benchmark " + benchmark + " natively.")
mx.run(cmd_line, cwd=tmp_dir, out=out, err=err, nonZeroIsFatal=nonZeroIsFatal)
finally:
if tmp_dir:
shutil.rmtree(tmp_dir)
return 0
def findBundledLLVMProgram(llvm_program):
llvm_dist = 'LLVM_TOOLCHAIN'
dep = mx.dependency(llvm_dist, fatalIfMissing=True)
return os.path.join(dep.get_output(), 'bin', mx.exe_suffix(llvm_program))
import mx
_suite = mx.suite('aot-bench')
jdk = mx.get_jdk()
jdk.aot_image = mx.exe_suffix(mx.join(jdk.home, 'bin', 'aot-image'))
_bench_suite = {
'fannkuchredux': [('15000', '10', '9'), ['-H:+MultiThreaded']],
'mandelbrot': [('15000', '10', '500'), ['-H:+MultiThreaded']],
'binarytrees': [('15000', '10', '14'), ['-H:+MultiThreaded']],
'nbody': [('10000', '10', '100000'), []],
# Throws NPE at the moment
# 'javac': [('1000', '10'), [
# '-H:+ReportUnsupportedElementsAtRuntime',
# '-H:IncludeResourceBundles=com.sun.tools.javac.resources.compiler,com.sun.tools.javac.resources.javac,com.sun.tools.javac.resources.version',
# ]],
'spectralnorm': [('5000', '10', '500'), ['-H:+MultiThreaded']],
'pidigits': [('7500', '50', '500'), []],
'scalariform': [('10000', '10'), []],
'chameneosredux': [('10000', '10', '12000'), ['-H:+MultiThreaded']],
'meteor': [('1000', '10'), []],
}
class Bench(object):
def __init__(self, src_file, class_file, classpath, runtime_args,
build_args):
self.src_file = src_file
self.class_file = class_file
self.classpath = classpath
def makegraaljdk(args):
"""make a JDK with Graal as the default top level JIT"""
parser = ArgumentParser(prog='mx makegraaljdk')
parser.add_argument('-f', '--force', action='store_true', help='overwrite existing GraalJDK')
parser.add_argument('-a', '--archive', action='store', help='name of archive to create', metavar='<path>')
parser.add_argument('dest', help='destination directory for GraalJDK', metavar='<path>')
args = parser.parse_args(args)
if isJDK8:
dstJdk = os.path.abspath(args.dest)
srcJdk = jdk.home
if exists(dstJdk):
if args.force:
shutil.rmtree(dstJdk)
else:
mx.abort('Use --force to overwrite existing directory ' + dstJdk)
mx.log('Creating {} from {}'.format(dstJdk, srcJdk))
shutil.copytree(srcJdk, dstJdk)
bootDir = mx.ensure_dir_exists(join(dstJdk, 'jre', 'lib', 'boot'))
jvmciDir = join(dstJdk, 'jre', 'lib', 'jvmci')
assert exists(jvmciDir), jvmciDir + ' does not exist'
if mx.get_os() == 'darwin' or mx.get_os() == 'windows':
jvmlibDir = join(dstJdk, 'jre', 'lib', 'server')
else:
jvmlibDir = join(dstJdk, 'jre', 'lib', mx.get_arch(), 'server')
jvmlib = join(jvmlibDir, mx.add_lib_prefix(mx.add_lib_suffix('jvm')))
assert exists(jvmlib), jvmlib + ' does not exist'
with open(join(jvmciDir, 'compiler-name'), 'w') as fp:
print >> fp, 'graal'
vmName = 'Graal'
mapFiles = set()
for e in _jvmci_classpath:
src = basename(e.get_path())
mx.log('Copying {} to {}'.format(e.get_path(), jvmciDir))
candidate = e.get_path() + '.map'
if exists(candidate):
mapFiles.add(candidate)
with open(join(dstJdk, 'release'), 'a') as fp:
d = e.dist()
s = d.suite
print >> fp, '{}={}'.format(d.name, s.vc.parent(s.dir))
vmName = vmName + ':' + s.name + '_' + s.version()
shutil.copyfile(e.get_path(), join(jvmciDir, src))
for e in _bootclasspath_appends:
src = basename(e.classpath_repr())
mx.log('Copying {} to {}'.format(e.classpath_repr(), bootDir))
candidate = e.classpath_repr() + '.map'
if exists(candidate):
mapFiles.add(candidate)
with open(join(dstJdk, 'release'), 'a') as fp:
s = e.suite
print >> fp, '{}={}'.format(e.name, s.vc.parent(s.dir))
shutil.copyfile(e.classpath_repr(), join(bootDir, src))
out = mx.LinesOutputCapture()
mx.run([jdk.java, '-version'], err=out)
line = None
pattern = re.compile(r'(.* )(?:Server|Graal) VM \(build.*')
for line in out.lines:
m = pattern.match(line)
if m:
with open(join(jvmlibDir, 'vm.properties'), 'w') as fp:
# Modify VM name in `java -version` to be Graal along
# with a suffix denoting the commit of each Graal jar.
# For example:
# Java HotSpot(TM) 64-Bit Graal:compiler_88847fb25d1a62977a178331a5e78fa5f8fcbb1a (build 25.71-b01-internal-jvmci-0.34, mixed mode)
print >> fp, 'name=' + m.group(1) + vmName
line = True
break
if line is not True:
mx.abort('Could not find "{}" in output of `java -version`:\n{}'.format(pattern.pattern, os.linesep.join(out.lines)))
exe = join(dstJdk, 'bin', mx.exe_suffix('java'))
with StdoutUnstripping(args=[], out=None, err=None, mapFiles=mapFiles) as u:
mx.run([exe, '-XX:+BootstrapJVMCI', '-version'], out=u.out, err=u.err)
if args.archive:
mx.log('Archiving {}'.format(args.archive))
create_archive(dstJdk, args.archive, basename(args.dest) + '/')
else:
mx.abort('Can only make GraalJDK for JDK 8 currently')
def jlink_new_jdk(jdk, dst_jdk_dir, module_dists, ignore_dists,
root_module_names=None,
missing_export_target_action='create',
with_source=lambda x: True,
vendor_info=None,
dedup_legal_notices=True,
use_upgrade_module_path=False):
"""
Uses jlink from `jdk` to create a new JDK image in `dst_jdk_dir` with `module_dists` and
their dependencies added to the JDK image, replacing any existing modules of the same name.
:param JDKConfig jdk: source JDK
:param str dst_jdk_dir: path to use for the jlink --output option
:param list module_dists: list of distributions defining modules
:param list ignore_dists: list of distributions that should be ignored for missing_export_target_action
:param list root_module_names: list of strings naming the module root set for the new JDK image.
The named modules must either be in `module_dists` or in `jdk`. If None, then
the root set will be all the modules in ``module_dists` and `jdk`.
:param str missing_export_target_action: the action to perform for a qualified export target that
is not present in `module_dists` and does not have a hash stored in java.base.
The choices are:
"create" - an empty module is created
"error" - raise an error
None - do nothing
:param lambda with_source: returns True if the sources of a module distribution must be included in the new JDK
:param dict vendor_info: values for the jlink vendor options added by JDK-8232080
:param bool use_upgrade_module_path: if True, then instead of linking `module_dists` into the image, resolve
them via --upgrade-module-path at image runtime
:return bool: False if use_upgrade_module_path == True and the existing image is up to date otherwise True
"""
assert callable(with_source)
if jdk.javaCompliance < '9':
mx.abort('Cannot derive a new JDK from ' + jdk.home + ' with jlink since it is not JDK 9 or later')
exploded_java_base_module = join(jdk.home, 'modules', 'java.base')
if exists(exploded_java_base_module):
mx.abort('Cannot derive a new JDK from ' + jdk.home + ' since it appears to be a developer build with exploded modules')
jimage = join(jdk.home, 'lib', 'modules')
jmods_dir = join(jdk.home, 'jmods')
if not isfile(jimage):
mx.abort('Cannot derive a new JDK from ' + jdk.home + ' since ' + jimage + ' is missing or is not an ordinary file')
if not isdir(jmods_dir):
mx.abort('Cannot derive a new JDK from ' + jdk.home + ' since ' + jmods_dir + ' is missing or is not a directory')
# Exclude jdk.aot due to GR-10545 and JDK-8255616
jdk_modules = {jmd.name: jmd for jmd in jdk.get_modules() if jmd.name != 'jdk.aot'}
modules = [as_java_module(dist, jdk) for dist in module_dists]
module_names = frozenset((m.name for m in modules))
all_module_names = frozenset(list(jdk_modules.keys())) | module_names
# Read hashes stored in java.base (the only module in the JDK where hashes are stored)
hashes = _read_java_base_hashes(jdk)
build_dir = mx.ensure_dir_exists(join(dst_jdk_dir + ".build"))
# Directory under dst_jdk_dir for artifacts related to use_upgrade_module_path
upgrade_dir = join(dst_jdk_dir, 'upgrade_modules_support')
# Map from JavaModuleDescriptors to post-jlink jar location.
synthetic_modules = OrderedDict()
try:
ignore_module_names = set(mx_javamodules.get_module_name(mx.dependency(ignore_dist)) for ignore_dist in ignore_dists)
# Synthesize modules for targets of qualified exports that are not present in `modules`.
# Without this, runtime module resolution will fail due to missing modules.
target_requires = {}
for jmd in modules:
for targets in jmd.exports.values():
for target in targets:
if target not in all_module_names and target not in ignore_module_names and target not in hashes:
target_requires.setdefault(target, set()).add(jmd.name)
if target_requires and missing_export_target_action is not None:
if missing_export_target_action == 'error':
mx.abort('Target(s) of qualified exports cannot be resolved: ' + '.'.join(target_requires.keys()))
assert missing_export_target_action == 'create', 'invalid value for missing_export_target_action: ' + str(missing_export_target_action)
for name, requires in sorted(target_requires.items()):
module_jar = join(build_dir, name + '.jar')
jmd = JavaModuleDescriptor(name, {}, requires={module: [] for module in requires}, uses=set(), provides={}, jarpath=module_jar)
module_build_dir = mx.ensure_dir_exists(join(build_dir, name))
module_info = jmd.as_module_info()
module_info_java = join(module_build_dir, 'module-info.java')
module_info_class = join(module_build_dir, 'module-info.class')
dst_module_jar = join(upgrade_dir, name + '.jar')
synthetic_modules[jmd] = dst_module_jar
if use_upgrade_module_path and exists(dst_module_jar):
with ZipFile(dst_module_jar, 'r') as zf:
previous_module_info = zf.read('module-info.java').decode()
if previous_module_info == module_info:
mx.logv('[Reusing synthetic module {}]'.format(name))
os.rename(dst_module_jar, module_jar)
continue
mx.logv('[Rebuilding synthetic module {} as module descriptor changed]'.format(name))
with open(module_info_java, 'w') as fp:
fp.write(module_info)
mx.run([jdk.javac, '-d', module_build_dir,
'--limit-modules=java.base,' + ','.join(jmd.requires.keys()),
'--module-path=' + os.pathsep.join((m.jarpath for m in modules)),
module_info_java])
with ZipFile(module_jar, 'w') as zf:
zf.write(module_info_java, 'module-info.java')
zf.write(module_info_class, 'module-info.class')
if exists(jmd.get_jmod_path()):
os.remove(jmd.get_jmod_path())
if not use_upgrade_module_path:
mx.run([jdk.javac.replace('javac', 'jmod'), 'create', '--class-path=' + module_build_dir, jmd.get_jmod_path()])
modules.extend(synthetic_modules.keys())
module_names = frozenset((m.name for m in modules))
all_module_names = frozenset(list(jdk_modules.keys())) | module_names
# Edit lib/security/default.policy in java.base
patched_java_base = _patch_default_security_policy(build_dir, jmods_dir, dst_jdk_dir)
# Now build the new JDK image with jlink
jlink = [jdk.javac.replace('javac', 'jlink')]
if jdk_enables_jvmci_by_default(jdk):
# On JDK 9+, +EnableJVMCI forces jdk.internal.vm.ci to be in the root set
jlink += ['-J-XX:-EnableJVMCI', '-J-XX:-UseJVMCICompiler']
jlink.append('--add-modules=' + ','.join(_get_image_root_modules(root_module_names, module_names, jdk_modules.keys(), use_upgrade_module_path)))
module_path = patched_java_base + os.pathsep + jmods_dir
if modules and not use_upgrade_module_path:
module_path = os.pathsep.join((m.get_jmod_path(respect_stripping=True) for m in modules)) + os.pathsep + module_path
jlink.append('--module-path=' + module_path)
jlink.append('--output=' + dst_jdk_dir)
# These options are derived from how OpenJDK runs jlink to produce the final runtime image.
jlink.extend(['-J-XX:+UseSerialGC', '-J-Xms32M', '-J-Xmx512M', '-J-XX:TieredStopAtLevel=1'])
jlink.append('-J-Dlink.debug=true')
if dedup_legal_notices:
jlink.append('--dedup-legal-notices=error-if-not-same-content')
jlink.append('--keep-packaged-modules=' + join(dst_jdk_dir, 'jmods'))
vm_options_path = join(upgrade_dir, 'vm_options')
vm_options = _get_image_vm_options(jdk, use_upgrade_module_path, modules, synthetic_modules)
if vm_options:
jlink.append('--add-options=' + ' '.join(vm_options))
if jdk_has_new_jlink_options(jdk) and vendor_info is not None:
for name, value in vendor_info.items():
jlink.append('--' + name + '=' + value)
release_file = join(jdk.home, 'release')
if isfile(release_file):
jlink.append('--release-info=' + release_file)
if exists(dst_jdk_dir):
if use_upgrade_module_path and _vm_options_match(vm_options, vm_options_path):
mx.logv('[Existing JDK image {} is up to date]'.format(dst_jdk_dir))
return False
mx.rmtree(dst_jdk_dir)
# TODO: investigate the options below used by OpenJDK to see if they should be used:
# --order-resources: specifies order of resources in generated lib/modules file.
# This is apparently not so important if a CDS archive is available.
# --generate-jli-classes: pre-generates a set of java.lang.invoke classes.
# See https://github.com/openjdk/jdk/blob/master/make/GenerateLinkOptData.gmk
mx.logv('[Creating JDK image in {}]'.format(dst_jdk_dir))
mx.run(jlink)
if use_upgrade_module_path:
# Move synthetic upgrade modules into final location
for jmd, jarpath in synthetic_modules.items():
mx.ensure_dir_exists(dirname(jarpath))
os.rename(jmd.jarpath, jarpath)
# Persist VM options cooked into image to be able to skip a subsequent
# jlink execution if the options do not change.
with open(vm_options_path, 'w') as fp:
fp.write(os.linesep.join(vm_options))
# Create src.zip in new JDK image
_copy_src_zip(jdk.home, dst_jdk_dir, modules, lambda jmd: not use_upgrade_module_path and with_source(jmd.dist))
mx.logv('[Copying static libraries]')
lib_directory = join(jdk.home, 'lib', 'static')
if exists(lib_directory):
dst_lib_directory = join(dst_jdk_dir, 'lib', 'static')
try:
mx.copytree(lib_directory, dst_lib_directory)
except shutil.Error as e:
# On AArch64, there can be a problem in the copystat part
# of copytree which occurs after file and directory copying
# has successfully completed. Since the metadata doesn't
# matter in this case, just ensure that the content was copied.
for root, _, lib_files in os.walk(lib_directory):
relative_root = os.path.relpath(root, dst_lib_directory)
for lib in lib_files:
src_lib_path = join(root, lib)
dst_lib_path = join(dst_lib_directory, relative_root, lib)
if not exists(dst_lib_path):
mx.abort('Error copying static libraries: {} missing in {}{}Original copytree error: {}'.format(
join(relative_root, lib), dst_lib_directory, os.linesep, e))
src_lib_hash = mx.sha1OfFile(src_lib_path)
dst_lib_hash = mx.sha1OfFile(dst_lib_path)
if src_lib_hash != dst_lib_hash:
mx.abort('Error copying static libraries: {} (hash={}) and {} (hash={}) differ{}Original copytree error: {}'.format(
src_lib_path, src_lib_hash,
dst_lib_path, dst_lib_hash,
os.linesep, e))
# Allow older JDK versions to work
else:
lib_prefix = mx.add_lib_prefix('')
lib_suffix = mx.add_static_lib_suffix('')
lib_directory = join(jdk.home, 'lib')
dst_lib_directory = join(dst_jdk_dir, 'lib')
for f in os.listdir(lib_directory):
if f.startswith(lib_prefix) and f.endswith(lib_suffix):
lib_path = join(lib_directory, f)
if isfile(lib_path):
shutil.copy2(lib_path, dst_lib_directory)
finally:
if not mx.get_opts().verbose:
# Preserve build directory so that javac command can be re-executed
# by cutting and pasting verbose output.
shutil.rmtree(build_dir)
if not use_upgrade_module_path:
# Create CDS archive (https://openjdk.java.net/jeps/341).
out = mx.OutputCapture()
mx.logv('[Creating CDS shared archive]')
if mx.run([mx.exe_suffix(join(dst_jdk_dir, 'bin', 'java')), '-Xshare:dump', '-Xmx128M', '-Xms128M'], out=out, err=out, nonZeroIsFatal=False) != 0:
mx.log(out.data)
mx.abort('Error generating CDS shared archive')
else:
# -Xshare is incompatible with --upgrade-module-path
pass
return True
def jlink_new_jdk(jdk, dst_jdk_dir, module_dists,
root_module_names=None,
missing_export_target_action='create',
with_source=lambda x: True,
vendor_info=None,
dedup_legal_notices=True):
"""
Uses jlink from `jdk` to create a new JDK image in `dst_jdk_dir` with `module_dists` and
their dependencies added to the JDK image, replacing any existing modules of the same name.
:param JDKConfig jdk: source JDK
:param str dst_jdk_dir: path to use for the jlink --output option
:param list module_dists: list of distributions defining modules
:param list root_module_names: list of strings naming the module root set for the new JDK image.
The named modules must either be in `module_dists` or in `jdk`. If None, then
the root set will be all the modules in ``module_dists` and `jdk`.
:param str missing_export_target_action: the action to perform for a qualifed export target that
is not present in `module_dists` and does not have a hash stored in java.base.
The choices are:
"create" - an empty module is created
"error" - raise an error
None - do nothing
:param lambda with_source: returns True if the sources of a module distribution must be included in the new JDK
:param dict vendor_info: values for the jlink vendor options added by JDK-8232080
"""
assert callable(with_source)
if jdk.javaCompliance < '9':
mx.abort('Cannot derive a new JDK from ' + jdk.home + ' with jlink since it is not JDK 9 or later')
exploded_java_base_module = join(jdk.home, 'modules', 'java.base')
if exists(exploded_java_base_module):
mx.abort('Cannot derive a new JDK from ' + jdk.home + ' since it appears to be a developer build with exploded modules')
jimage = join(jdk.home, 'lib', 'modules')
jmods_dir = join(jdk.home, 'jmods')
if not isfile(jimage):
mx.abort('Cannot derive a new JDK from ' + jdk.home + ' since ' + jimage + ' is missing or is not an ordinary file')
if not isdir(jmods_dir):
mx.abort('Cannot derive a new JDK from ' + jdk.home + ' since ' + jmods_dir + ' is missing or is not a directory')
# Exclude jdk.aot due to GR-10545 and JDK-8255616
jdk_modules = {jmd.name: jmd for jmd in jdk.get_modules() if jmd.name != 'jdk.aot'}
modules = [as_java_module(dist, jdk) for dist in module_dists]
all_module_names = frozenset(list(jdk_modules.keys()) + [m.name for m in modules])
# Read hashes stored in java.base (the only module in the JDK where hashes are stored)
out = mx.LinesOutputCapture()
mx.run([jdk.exe_path('jmod'), 'describe', jdk_modules['java.base'].get_jmod_path()], out=out)
lines = out.lines
hashes = {}
for line in lines:
if line.startswith('hashes'):
parts = line.split()
assert len(parts) == 4, 'expected hashes line to have 4 fields, got {} fields: {}'.format(len(parts), line)
_, module_name, algorithm, hash_value = parts
hashes[module_name] = (algorithm, hash_value)
build_dir = mx.ensure_dir_exists(join(dst_jdk_dir + ".build"))
try:
# Handle targets of qualified exports that are not present in `modules`
target_requires = {}
for jmd in modules:
for targets in jmd.exports.values():
for target in targets:
if target not in all_module_names and target not in hashes:
target_requires.setdefault(target, set()).add(jmd.name)
if target_requires and missing_export_target_action is not None:
if missing_export_target_action == 'error':
mx.abort('Target(s) of qualified exports cannot be resolved: ' + '.'.join(target_requires.keys()))
assert missing_export_target_action == 'create', 'invalid value for missing_export_target_action: ' + str(missing_export_target_action)
extra_modules = []
for name, requires in target_requires.items():
module_jar = join(build_dir, name + '.jar')
jmd = JavaModuleDescriptor(name, {}, requires={module: [] for module in requires}, uses=set(), provides={}, jarpath=module_jar)
extra_modules.append(jmd)
module_build_dir = mx.ensure_dir_exists(join(build_dir, name))
module_info_java = join(module_build_dir, 'module-info.java')
module_info_class = join(module_build_dir, 'module-info.class')
with open(module_info_java, 'w') as fp:
print(jmd.as_module_info(), file=fp)
mx.run([jdk.javac, '-d', module_build_dir,
'--limit-modules=java.base,' + ','.join(jmd.requires.keys()),
'--module-path=' + os.pathsep.join((m.jarpath for m in modules)),
module_info_java])
with ZipFile(module_jar, 'w') as zf:
zf.write(module_info_class, basename(module_info_class))
if exists(jmd.get_jmod_path()):
os.remove(jmd.get_jmod_path())
mx.run([jdk.javac.replace('javac', 'jmod'), 'create', '--class-path=' + module_build_dir, jmd.get_jmod_path()])
modules.extend(extra_modules)
all_module_names = frozenset(list(jdk_modules.keys()) + [m.name for m in modules])
# Extract src.zip from source JDK
jdk_src_zip = join(jdk.home, 'lib', 'src.zip')
dst_src_zip_contents = {}
if isfile(jdk_src_zip):
mx.logv('[Extracting ' + jdk_src_zip + ']')
with ZipFile(jdk_src_zip, 'r') as zf:
for name in zf.namelist():
if not name.endswith('/'):
dst_src_zip_contents[name] = zf.read(name)
else:
mx.warn("'{}' does not exist or is not a file".format(jdk_src_zip))
# Edit lib/security/default.policy in java.base
patched_java_base = join(build_dir, 'java.base.jmod')
with open(join(jmods_dir, 'java.base.jmod'), 'rb') as src_f, open(patched_java_base, 'wb') as dst_f:
jmod_header = src_f.read(4)
if len(jmod_header) != 4 or jmod_header != b'JM\x01\x00':
raise mx.abort("Unexpected jmod header: " + b2a_hex(jmod_header).decode('ascii'))
dst_f.write(jmod_header)
policy_result = 'not found'
with ZipFile(src_f, 'r') as src_zip, ZipFile(dst_f, 'w', src_zip.compression) as dst_zip:
for i in src_zip.infolist():
if i.filename[-1] == '/':
continue
src_member = src_zip.read(i)
if i.filename == 'lib/security/default.policy':
policy_result = 'unmodified'
if 'grant codeBase "jrt:/com.oracle.graal.graal_enterprise"'.encode('utf-8') not in src_member:
policy_result = 'modified'
src_member += """
grant codeBase "jrt:/com.oracle.graal.graal_enterprise" {
permission java.security.AllPermission;
};
""".encode('utf-8')
if 'grant codeBase "jrt:/org.graalvm.truffle"'.encode('utf-8') not in src_member:
policy_result = 'modified'
src_member += """
grant codeBase "jrt:/org.graalvm.truffle" {
permission java.security.AllPermission;
};
grant codeBase "jrt:/org.graalvm.sdk" {
permission java.security.AllPermission;
};
grant codeBase "jrt:/org.graalvm.locator" {
permission java.io.FilePermission "<<ALL FILES>>", "read";
permission java.util.PropertyPermission "*", "read,write";
permission java.lang.RuntimePermission "createClassLoader";
permission java.lang.RuntimePermission "getClassLoader";
permission java.lang.RuntimePermission "getenv.*";
};
grant codeBase "file:${java.home}/languages/-" {
permission java.security.AllPermission;
};
""".encode('utf-8')
dst_zip.writestr(i, src_member)
if policy_result == 'not found':
raise mx.abort("Couldn't find `lib/security/default.policy` in " + join(jmods_dir, 'java.base.jmod'))
for jmd in modules:
# Remove existing sources for all the modules that we include
dst_src_zip_contents = {key: dst_src_zip_contents[key] for key in dst_src_zip_contents if not key.startswith(jmd.name)}
if with_source(jmd.dist):
# Add the sources that we can share.
# Extract module sources
jmd_src_zip = jmd.jarpath[0:-len('.jar')] + '.src.zip'
if isfile(jmd_src_zip):
mx.logv('[Extracting ' + jmd_src_zip + ']')
with ZipFile(jmd_src_zip, 'r') as zf:
for name in zf.namelist():
if not name.endswith('/'):
dst_src_zip_contents[jmd.name + '/' + name] = zf.read(name)
# Add module-info.java to sources
dst_src_zip_contents[jmd.name + '/module-info.java'] = jmd.as_module_info(extras_as_comments=False)
# Now build the new JDK image with jlink
jlink = [jdk.javac.replace('javac', 'jlink')]
if jdk_enables_jvmci_by_default(jdk):
# On JDK 9+, +EnableJVMCI forces jdk.internal.vm.ci to be in the root set
jlink += ['-J-XX:-EnableJVMCI', '-J-XX:-UseJVMCICompiler']
if root_module_names is not None:
missing = frozenset(root_module_names) - all_module_names
if missing:
mx.abort('Invalid module(s): {}.\nAvailable modules: {}'.format(','.join(missing), ','.join(sorted(all_module_names))))
jlink.append('--add-modules=' + ','.join(root_module_names))
else:
jlink.append('--add-modules=' + ','.join(sorted(all_module_names)))
module_path = patched_java_base + os.pathsep + jmods_dir
if modules:
module_path = os.pathsep.join((m.get_jmod_path(respect_stripping=True) for m in modules)) + os.pathsep + module_path
jlink.append('--module-path=' + module_path)
jlink.append('--output=' + dst_jdk_dir)
# These options are derived from how OpenJDK runs jlink to produce the final runtime image.
jlink.extend(['-J-XX:+UseSerialGC', '-J-Xms32M', '-J-Xmx512M', '-J-XX:TieredStopAtLevel=1'])
jlink.append('-J-Dlink.debug=true')
if dedup_legal_notices:
jlink.append('--dedup-legal-notices=error-if-not-same-content')
jlink.append('--keep-packaged-modules=' + join(dst_jdk_dir, 'jmods'))
if jdk_has_new_jlink_options(jdk):
if jdk_omits_warning_for_jlink_set_ThreadPriorityPolicy(jdk):
thread_priority_policy_option = ' -XX:ThreadPriorityPolicy=1'
else:
mx.logv('[Creating JDK without -XX:ThreadPriorityPolicy=1]')
thread_priority_policy_option = ''
if jdk_supports_enablejvmciproduct(jdk):
if any((m.name == 'jdk.internal.vm.compiler' for m in modules)):
jlink.append('--add-options=-XX:+UnlockExperimentalVMOptions -XX:+EnableJVMCIProduct -XX:-UnlockExperimentalVMOptions' + thread_priority_policy_option)
else:
# Don't default to using JVMCI as JIT unless Graal is being updated in the image.
# This avoids unexpected issues with using the out-of-date Graal compiler in
# the JDK itself.
jlink.append('--add-options=-XX:+UnlockExperimentalVMOptions -XX:+EnableJVMCIProduct -XX:-UseJVMCICompiler -XX:-UnlockExperimentalVMOptions' + thread_priority_policy_option)
else:
mx.logv('[Creating JDK without -XX:+EnableJVMCIProduct]')
if thread_priority_policy_option:
jlink.append('--add-options=' + thread_priority_policy_option.strip())
if vendor_info is not None:
for name, value in vendor_info.items():
jlink.append('--' + name + '=' + value)
release_file = join(jdk.home, 'release')
if isfile(release_file):
jlink.append('--release-info=' + release_file)
# TODO: investigate the options below used by OpenJDK to see if they should be used:
# --order-resources: specifies order of resources in generated lib/modules file.
# This is apparently not so important if a CDS archive is available.
# --generate-jli-classes: pre-generates a set of java.lang.invoke classes.
# See https://github.com/openjdk/jdk/blob/master/make/GenerateLinkOptData.gmk
mx.logv('[Creating JDK image in {}]'.format(dst_jdk_dir))
mx.run(jlink)
dst_src_zip = join(dst_jdk_dir, 'lib', 'src.zip')
mx.logv('[Creating ' + dst_src_zip + ']')
with ZipFile(dst_src_zip, 'w', compression=ZIP_DEFLATED, allowZip64=True) as zf:
for name, contents in sorted(dst_src_zip_contents.items()):
zf.writestr(name, contents)
mx.logv('[Copying static libraries]')
lib_directory = join(jdk.home, 'lib', 'static')
if exists(lib_directory):
dst_lib_directory = join(dst_jdk_dir, 'lib', 'static')
try:
mx.copytree(lib_directory, dst_lib_directory)
except shutil.Error as e:
# On AArch64, there can be a problem in the copystat part
# of copytree which occurs after file and directory copying
# has successfully completed. Since the metadata doesn't
# matter in this case, just ensure that the content was copied.
for root, _, lib_files in os.walk(lib_directory):
relative_root = os.path.relpath(root, dst_lib_directory)
for lib in lib_files:
src_lib_path = join(root, lib)
dst_lib_path = join(dst_lib_directory, relative_root, lib)
if not exists(dst_lib_path):
mx.abort('Error copying static libraries: {} missing in {}{}Original copytree error: {}'.format(
join(relative_root, lib), dst_lib_directory, os.linesep, e))
src_lib_hash = mx.sha1OfFile(src_lib_path)
dst_lib_hash = mx.sha1OfFile(dst_lib_path)
if src_lib_hash != dst_lib_hash:
mx.abort('Error copying static libraries: {} (hash={}) and {} (hash={}) differ{}Original copytree error: {}'.format(
src_lib_path, src_lib_hash,
dst_lib_path, dst_lib_hash,
os.linesep, e))
# Allow older JDK versions to work
else:
lib_prefix = mx.add_lib_prefix('')
lib_suffix = mx.add_static_lib_suffix('')
lib_directory = join(jdk.home, 'lib')
dst_lib_directory = join(dst_jdk_dir, 'lib')
for f in os.listdir(lib_directory):
if f.startswith(lib_prefix) and f.endswith(lib_suffix):
lib_path = join(lib_directory, f)
if isfile(lib_path):
shutil.copy2(lib_path, dst_lib_directory)
finally:
if not mx.get_opts().verbose:
# Preserve build directory so that javac command can be re-executed
# by cutting and pasting verbose output.
shutil.rmtree(build_dir)
# Create CDS archive (https://openjdk.java.net/jeps/341).
out = mx.OutputCapture()
mx.logv('[Creating CDS shared archive]')
if mx.run([mx.exe_suffix(join(dst_jdk_dir, 'bin', 'java')), '-Xshare:dump', '-Xmx128M', '-Xms128M'], out=out, err=out, nonZeroIsFatal=False) != 0:
mx.log(out.data)
mx.abort('Error generating CDS shared archive')
def jlink_new_jdk(jdk, dst_jdk_dir, module_dists, root_module_names=None, missing_export_target_action='create', with_source=lambda x: True):
"""
Uses jlink from `jdk` to create a new JDK image in `dst_jdk_dir` with `module_dists` and
their dependencies added to the JDK image, replacing any existing modules of the same name.
:param JDKConfig jdk: source JDK
:param str dst_jdk_dir: path to use for the jlink --output option
:param list module_dists: list of distributions defining modules
:param list root_module_names: list of strings naming the module root set for the new JDK image.
The named modules must either be in `module_dists` or in `jdk`. If None, then
the root set will be all the modules in ``module_dists` and `jdk`.
:param str missing_export_target_action: the action to perform for a qualifed export target that
is not present in `module_dists` and does not have a hash stored in java.base.
The choices are:
"create" - an empty module is created
"error" - raise an error
None - do nothing
:param lambda with_source: returns True if the sources of a module distribution must be included in the new JDK
"""
assert callable(with_source)
if jdk.javaCompliance < '9':
mx.abort('Cannot derive a new JDK from ' + jdk.home + ' with jlink since it is not JDK 9 or later')
exploded_java_base_module = join(jdk.home, 'modules', 'java.base')
if exists(exploded_java_base_module):
mx.abort('Cannot derive a new JDK from ' + jdk.home + ' since it appears to be a developer build with exploded modules')
jimage = join(jdk.home, 'lib', 'modules')
jmods_dir = join(jdk.home, 'jmods')
if not isfile(jimage):
mx.abort('Cannot derive a new JDK from ' + jdk.home + ' since ' + jimage + ' is missing or is not an ordinary file')
if not isdir(jmods_dir):
mx.abort('Cannot derive a new JDK from ' + jdk.home + ' since ' + jmods_dir + ' is missing or is not a directory')
jdk_modules = {jmd.name : jmd for jmd in jdk.get_modules()}
modules = [as_java_module(dist, jdk) for dist in module_dists]
all_module_names = frozenset(list(jdk_modules.keys()) + [m.name for m in modules])
# Read hashes stored in java.base (the only module in the JDK where hashes are stored)
out = mx.LinesOutputCapture()
mx.run([jdk.exe_path('jmod'), 'describe', jdk_modules['java.base'].get_jmod_path()], out=out)
lines = out.lines
hashes = {}
for line in lines:
if line.startswith('hashes'):
parts = line.split()
assert len(parts) == 4, 'expected hashes line to have 4 fields, got {} fields: {}'.format(len(parts), line)
_, module_name, algorithm, hash_value = parts
hashes[module_name] = (algorithm, hash_value)
build_dir = mx.ensure_dir_exists(join(dst_jdk_dir + ".build"))
try:
# Handle targets of qualified exports that are not present in `modules`
target_requires = {}
for jmd in modules:
for targets in jmd.exports.values():
for target in targets:
if target not in all_module_names and target not in hashes:
target_requires.setdefault(target, set()).add(jmd.name)
if target_requires and missing_export_target_action is not None:
if missing_export_target_action == 'error':
mx.abort('Target(s) of qualified exports cannot be resolved: ' + '.'.join(target_requires.keys()))
assert missing_export_target_action == 'create', 'invalid value for missing_export_target_action: ' + str(missing_export_target_action)
extra_modules = []
for name, requires in target_requires.items():
module_jar = join(build_dir, name + '.jar')
jmd = JavaModuleDescriptor(name, {}, requires={module : [] for module in requires}, uses=set(), provides={}, jarpath=module_jar)
extra_modules.append(jmd)
module_build_dir = mx.ensure_dir_exists(join(build_dir, name))
module_info_java = join(module_build_dir, 'module-info.java')
module_info_class = join(module_build_dir, 'module-info.class')
with open(module_info_java, 'w') as fp:
print(jmd.as_module_info(), file=fp)
mx.run([jdk.javac, '-d', module_build_dir, \
'--limit-modules=java.base,' + ','.join(jmd.requires.keys()), \
'--module-path=' + os.pathsep.join((m.jarpath for m in modules)), \
module_info_java])
with ZipFile(module_jar, 'w') as zf:
zf.write(module_info_class, basename(module_info_class))
if exists(jmd.get_jmod_path()):
os.remove(jmd.get_jmod_path())
mx.run([jdk.javac.replace('javac', 'jmod'), 'create', '--class-path=' + module_build_dir, jmd.get_jmod_path()])
modules.extend(extra_modules)
all_module_names = frozenset(list(jdk_modules.keys()) + [m.name for m in modules])
# Extract src.zip from source JDK
jdk_src_zip = join(jdk.home, 'lib', 'src.zip')
dst_src_zip_contents = {}
if isfile(jdk_src_zip):
mx.logv('[Extracting ' + jdk_src_zip + ']')
with ZipFile(jdk_src_zip, 'r') as zf:
for name in zf.namelist():
if not name.endswith('/'):
dst_src_zip_contents[name] = zf.read(name)
else:
mx.warn("'{}' does not exist or is not a file".format(jdk_src_zip))
for jmd in modules:
# Remove existing sources for all the modules that we include
dst_src_zip_contents = {key : dst_src_zip_contents[key] for key in dst_src_zip_contents if not key.startswith(jmd.name)}
if with_source(jmd.dist):
# Add the sources that we can share.
# Extract module sources
jmd_src_zip = jmd.jarpath[0:-len('.jar')] + '.src.zip'
if isfile(jmd_src_zip):
mx.logv('[Extracting ' + jmd_src_zip + ']')
with ZipFile(jmd_src_zip, 'r') as zf:
for name in zf.namelist():
if not name.endswith('/'):
dst_src_zip_contents[jmd.name + '/' + name] = zf.read(name)
# Add module-info.java to sources
dst_src_zip_contents[jmd.name + '/module-info.java'] = jmd.as_module_info(extras_as_comments=False)
# Now build the new JDK image with jlink
jlink = [jdk.javac.replace('javac', 'jlink')]
if jdk_enables_jvmci_by_default(jdk):
# On JDK 9+, +EnableJVMCI forces jdk.internal.vm.ci to be in the root set
jlink.append('-J-XX:-EnableJVMCI')
if root_module_names is not None:
missing = frozenset(root_module_names) - all_module_names
if missing:
mx.abort('Invalid module(s): {}.\nAvailable modules: {}'.format(','.join(missing), ','.join(sorted(all_module_names))))
jlink.append('--add-modules=' + ','.join(root_module_names))
else:
jlink.append('--add-modules=' + ','.join(sorted(all_module_names)))
module_path = jmods_dir
if modules:
module_path = os.pathsep.join((m.get_jmod_path(respect_stripping=True) for m in modules)) + os.pathsep + module_path
jlink.append('--module-path=' + module_path)
jlink.append('--output=' + dst_jdk_dir)
# These options are inspired by how OpenJDK runs jlink to produce the final runtime image.
jlink.extend(['-J-XX:+UseSerialGC', '-J-Xms32M', '-J-Xmx512M', '-J-XX:TieredStopAtLevel=1'])
jlink.append('-J-Dlink.debug=true')
jlink.append('--dedup-legal-notices=error-if-not-same-content')
jlink.append('--keep-packaged-modules=' + join(dst_jdk_dir, 'jmods'))
# TODO: investigate the options below used by OpenJDK to see if they should be used:
# --release-info: this allow extra properties to be written to the <jdk>/release file
# --order-resources: specifies order of resources in generated lib/modules file.
# This is apparently not so important if a CDS archive is available.
# --generate-jli-classes: pre-generates a set of java.lang.invoke classes.
# See https://github.com/openjdk/jdk/blob/master/make/GenerateLinkOptData.gmk
mx.logv('[Creating JDK image]')
mx.run(jlink)
dst_src_zip = join(dst_jdk_dir, 'lib', 'src.zip')
mx.logv('[Creating ' + dst_src_zip + ']')
with ZipFile(dst_src_zip, 'w', compression=ZIP_DEFLATED, allowZip64=True) as zf:
for name, contents in sorted(dst_src_zip_contents.items()):
zf.writestr(name, contents)
mx.logv('[Copying static libraries]')
lib_prefix = mx.add_lib_prefix('')
lib_suffix = '.lib' if mx.is_windows() else '.a'
lib_directory = join(jdk.home, 'lib')
dst_lib_directory = join(dst_jdk_dir, 'lib')
for f in os.listdir(lib_directory):
if f.startswith(lib_prefix) and f.endswith(lib_suffix):
lib_path = join(lib_directory, f)
if isfile(lib_path):
shutil.copy2(lib_path, dst_lib_directory)
# Build the list of modules whose classes might have annotations
# to be processed by native-image (GR-15192).
with open(join(dst_jdk_dir, 'lib', 'native-image-modules.list'), 'w') as fp:
print('# Modules whose classes might have annotations processed by native-image', file=fp)
for m in modules:
print(m.name, file=fp)
finally:
if not mx.get_opts().verbose:
# Preserve build directory so that javac command can be re-executed
# by cutting and pasting verbose output.
shutil.rmtree(build_dir)
# Create CDS archive (https://openjdk.java.net/jeps/341).
out = mx.OutputCapture()
mx.logv('[Creating CDS shared archive]')
if mx.run([mx.exe_suffix(join(dst_jdk_dir, 'bin', 'java')), '-Xshare:dump', '-Xmx128M', '-Xms128M'], out=out, err=out, nonZeroIsFatal=False) != 0:
mx.log(out.data)
mx.abort('Error generating CDS shared archive')
def build(self):
source_dir = self.subject.getSourceDir()
output_dir = self.subject.getOutputDir()
if not emcc_dir:
mx.abort(
"No EMCC_DIR specified - the source programs will not be compiled to .wasm."
)
emcc_cmd = os.path.join(emcc_dir, "emcc")
gcc_cmd = os.path.join(gcc_dir, "gcc")
if mx.run([emcc_cmd, "-v"], nonZeroIsFatal=False) != 0:
mx.abort("Could not check the emcc version.")
if mx.run([gcc_cmd, "--version"], nonZeroIsFatal=False) != 0:
mx.abort("Could not check the gcc version.")
if not wabt_dir:
mx.abort(
"Set WABT_DIR if you want the binary to include .wat files.")
mx.log("Building files from the source dir: " + source_dir)
cc_flags = ["-g2", "-O3"]
include_flags = []
if hasattr(self.project, "includeset"):
include_flags = [
"-I",
os.path.join(_suite.dir, "includes", self.project.includeset)
]
emcc_flags = [
"-s", "EXIT_RUNTIME=1", "-s", "STANDALONE_WASM", "-s",
"WASM_BIGINT"
] + cc_flags
if self.project.isBenchmarkProject():
emcc_flags = emcc_flags + [
"-s", "EXPORTED_FUNCTIONS=" +
str(self.benchmark_methods()).replace("'", "\"") + ""
]
subdir_program_names = defaultdict(lambda: [])
for root, filename in self.subject.getProgramSources():
if filename.startswith("_"):
# Ignore files starting with an underscore
continue
subdir = os.path.relpath(root, self.subject.getSourceDir())
mx.ensure_dir_exists(os.path.join(output_dir, subdir))
basename = remove_extension(filename)
source_path = os.path.join(root, filename)
output_wasm_path = os.path.join(output_dir, subdir,
basename + ".wasm")
output_js_path = os.path.join(output_dir, subdir, basename + ".js")
timestampedSource = mx.TimeStampFile(source_path)
timestampedOutput = mx.TimeStampFile(output_wasm_path)
mustRebuild = timestampedSource.isNewerThan(
timestampedOutput) or not timestampedOutput.exists()
# Step 1: build the .wasm binary.
if mustRebuild:
if filename.endswith(".c"):
# This generates both a js file and a wasm file.
# See https://github.com/emscripten-core/emscripten/wiki/WebAssembly-Standalone
build_cmd_line = [emcc_cmd] + emcc_flags + [
source_path, "-o", output_js_path
] + include_flags
if mx.run(build_cmd_line, nonZeroIsFatal=False) != 0:
mx.abort("Could not build the wasm-only output of " +
filename + " with emcc.")
elif filename.endswith(".wat"):
# Step 1: compile the .wat file to .wasm.
wat2wasm_cmd = os.path.join(wabt_dir, "wat2wasm")
build_cmd_line = [
wat2wasm_cmd, "-o", output_wasm_path, source_path
]
if mx.run(build_cmd_line, nonZeroIsFatal=False) != 0:
mx.abort("Could not translate " + filename +
" to binary format.")
elif filename.endswith(".wasm"):
shutil.copyfile(source_path, output_wasm_path)
else:
mx.logv("skipping, file is up-to-date: " + source_path)
# Step 2: copy the result file if it exists.
result_path = os.path.join(root, basename + ".result")
if os.path.isfile(result_path):
result_output_path = os.path.join(output_dir, subdir,
basename + ".result")
shutil.copyfile(result_path, result_output_path)
# Step 3: copy the opts file if it exists.
opts_path = os.path.join(root, basename + ".opts")
if os.path.isfile(opts_path):
opts_output_path = os.path.join(output_dir, subdir,
basename + ".opts")
shutil.copyfile(opts_path, opts_output_path)
output_wat_path = os.path.join(output_dir, subdir,
basename + ".wat")
if mustRebuild:
if filename.endswith(".c"):
# Step 4: produce the .wat files, for easier debugging.
wasm2wat_cmd = os.path.join(wabt_dir, "wasm2wat")
if mx.run([
wasm2wat_cmd, "-o", output_wat_path,
output_wasm_path
],
nonZeroIsFatal=False) != 0:
mx.abort("Could not compile .wat file for " + filename)
elif filename.endswith(".wat"):
# Step 4: copy the .wat file, for easier debugging.
wat_path = os.path.join(root, basename + ".wat")
shutil.copyfile(wat_path, output_wat_path)
# Step 5: if this is a benchmark project, create native binaries too.
if mustRebuild:
if filename.endswith(".c"):
mx.ensure_dir_exists(
os.path.join(output_dir, subdir, NATIVE_BENCH_DIR))
output_path = os.path.join(output_dir, subdir,
NATIVE_BENCH_DIR,
mx.exe_suffix(basename))
link_flags = ["-lm"]
gcc_cmd_line = [gcc_cmd] + cc_flags + [
source_path, "-o", output_path
] + include_flags + link_flags
if mx.run(gcc_cmd_line, nonZeroIsFatal=False) != 0:
mx.abort("Could not build the native binary of " +
filename + ".")
os.chmod(output_path,
stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
elif filename.endswith(".wat"):
mx.warn(
"The .wat files are not translated to native binaries: "
+ filename)
# Remember the source name.
subdir_program_names[subdir].append(basename)
for subdir in subdir_program_names:
with open(os.path.join(output_dir, subdir, "wasm_test_index"),
"w") as f:
for name in subdir_program_names[subdir]:
f.write(name)
f.write("\n")
def build(self):
source_dir = self.subject.getSourceDir()
output_dir = self.subject.getOutputDir()
if not emcc_dir:
mx.abort("No EMCC_DIR specified - the source programs will not be compiled to .wasm.")
emcc_cmd = os.path.join(emcc_dir, "emcc")
gcc_cmd = os.path.join(gcc_dir, "gcc")
if mx.run([emcc_cmd, "-v"], nonZeroIsFatal=False) != 0:
mx.abort("Could not check the emcc version.")
if mx.run([gcc_cmd, "--version"], nonZeroIsFatal=False) != 0:
mx.abort("Could not check the gcc version.")
if not wabt_dir:
mx.abort("Set WABT_DIR if you want the binary to include .wat files.")
mx.log("Building files from the source dir: " + source_dir)
cc_flags = ["-O3", "-g2"]
include_flags = []
disable_test_api_flags = ["-DDISABLE_TEST_API"]
if hasattr(self.project, "includeset"):
include_flags = ["-I", os.path.join(_suite.dir, "includes", self.project.includeset)]
emcc_flags = cc_flags
if self.project.isBenchmarkProject():
emcc_flags = emcc_flags + ["-s", "EXPORTED_FUNCTIONS=" + str(benchmark_methods).replace("'", "\"") + ""]
subdir_program_names = defaultdict(lambda: [])
for root, filename in self.subject.getProgramSources():
subdir = os.path.relpath(root, self.subject.getSourceDir())
mx.ensure_dir_exists(os.path.join(output_dir, subdir))
basename = remove_extension(filename)
source_path = os.path.join(root, filename)
output_wasm_path = os.path.join(output_dir, subdir, basename + ".wasm")
timestampedSource = mx.TimeStampFile(source_path)
timestampedOutput = mx.TimeStampFile(output_wasm_path)
mustRebuild = timestampedSource.isNewerThan(timestampedOutput) or not timestampedOutput.exists()
# Step 1: build the .wasm binary.
if mustRebuild:
if filename.endswith(".c"):
# Step 1a: compile with the JS file, and store as files for running Node, if necessary.
output_js_path = os.path.join(output_dir, subdir, basename + ".js")
build_cmd_line = [emcc_cmd] + emcc_flags + disable_test_api_flags + [source_path, "-o", output_js_path] + include_flags
if mx.run(build_cmd_line, nonZeroIsFatal=False) != 0:
mx.abort("Could not build the JS output of " + filename + " with emcc.")
if self.subject.isBenchmarkProject():
node_dir = os.path.join(output_dir, subdir, NODE_BENCH_DIR)
mx.ensure_dir_exists(node_dir)
shutil.copyfile(output_js_path, os.path.join(node_dir, basename + ".js"))
shutil.copyfile(output_wasm_path, os.path.join(node_dir, basename + ".wasm"))
# Step 1b: extract the relevant information out of the JS file, and record it into an initialization file.
init_info = self.extractInitialization(output_js_path)
with open(os.path.join(output_dir, subdir, basename + ".init"), "w") as f:
f.write(init_info)
# Step 1c: compile to just a .wasm file, to avoid name mangling.
build_cmd_line = [emcc_cmd] + emcc_flags + ["-s", "ERROR_ON_UNDEFINED_SYMBOLS=0"] + [source_path, "-o", output_wasm_path] + include_flags
if mx.run(build_cmd_line, nonZeroIsFatal=False) != 0:
mx.abort("Could not build the wasm-only output of " + filename + " with emcc.")
elif filename.endswith(".wat"):
# Step 1: compile the .wat file to .wasm.
wat2wasm_cmd = os.path.join(wabt_dir, "wat2wasm")
build_cmd_line = [wat2wasm_cmd, "-o", output_wasm_path, source_path]
if mx.run(build_cmd_line, nonZeroIsFatal=False) != 0:
mx.abort("Could not translate " + filename + " to binary format.")
else:
mx.logv("skipping, file is up-to-date: " + source_path)
# Step 2: copy the result file if it exists.
result_path = os.path.join(root, basename + ".result")
if os.path.isfile(result_path):
result_output_path = os.path.join(output_dir, subdir, basename + ".result")
shutil.copyfile(result_path, result_output_path)
# Step 3: copy the opts file if it exists.
opts_path = os.path.join(root, basename + ".opts")
if os.path.isfile(opts_path):
opts_output_path = os.path.join(output_dir, subdir, basename + ".opts")
shutil.copyfile(opts_path, opts_output_path)
output_wat_path = os.path.join(output_dir, subdir, basename + ".wat")
if mustRebuild:
if filename.endswith(".c"):
# Step 4: produce the .wat files, for easier debugging.
wasm2wat_cmd = os.path.join(wabt_dir, "wasm2wat")
if mx.run([wasm2wat_cmd, "-o", output_wat_path, output_wasm_path], nonZeroIsFatal=False) != 0:
mx.abort("Could not compile .wat file for " + filename)
elif filename.endswith(".wat"):
# Step 4: copy the .wat file, for easier debugging.
wat_path = os.path.join(root, basename + ".wat")
shutil.copyfile(wat_path, output_wat_path)
# Step 5: if this is a benchmark project, create native binaries too.
if mustRebuild:
mx.ensure_dir_exists(os.path.join(output_dir, subdir, NATIVE_BENCH_DIR))
if filename.endswith(".c"):
output_path = os.path.join(output_dir, subdir, NATIVE_BENCH_DIR, mx.exe_suffix(basename))
link_flags = ["-lm"]
gcc_cmd_line = [gcc_cmd] + cc_flags + disable_test_api_flags + [source_path, "-o", output_path] + include_flags + link_flags
if mx.run(gcc_cmd_line, nonZeroIsFatal=False) != 0:
mx.abort("Could not build the native binary of " + filename + ".")
os.chmod(output_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
elif filename.endswith(".wat"):
mx.warn("The .wat files are not translated to native binaries: " + filename)
# Remember the source name.
subdir_program_names[subdir].append(basename)
for subdir in subdir_program_names:
with open(os.path.join(output_dir, subdir, "wasm_test_index"), "w") as f:
for name in subdir_program_names[subdir]:
f.write(name)
f.write("\n")
def wat2wasm_binary():
return mx.exe_suffix("wat2wasm")