def gate_svm_sl_tck(tasks):
with Task('SVM Truffle TCK', tasks, tags=[VmGateTasks.svm_sl_tck]) as t:
if t:
tools_suite = mx.suite('tools')
if not tools_suite:
mx.abort("Cannot resolve tools suite.")
native_image_context, svm = graalvm_svm()
with native_image_context(svm.IMAGE_ASSERTION_FLAGS) as native_image:
svmbuild = mkdtemp()
try:
import mx_compiler
unittest_deps = []
unittest_file = join(svmbuild, 'truffletck.tests')
mx_unittest._run_tests([], lambda deps, vm_launcher, vm_args: unittest_deps.extend(deps), mx_unittest._VMLauncher('dummy_launcher', None, mx_compiler.jdk), ['@Test', '@Parameters'], unittest_file, [], [re.compile('com.oracle.truffle.tck.tests')], None, mx.suite('truffle'))
if not exists(unittest_file):
mx.abort('TCK tests not found.')
unittest_deps.append(mx.dependency('truffle:TRUFFLE_SL_TCK'))
unittest_deps.append(mx.dependency('truffle:TRUFFLE_TCK_INSTRUMENTATION'))
vm_image_args = mx.get_runtime_jvm_args(unittest_deps, jdk=mx_compiler.jdk)
options = [
'--macro:truffle',
'--tool:all',
'-H:Path={}'.format(svmbuild),
'-H:Class=org.junit.runner.JUnitCore',
]
tests_image = native_image(vm_image_args + options)
with open(unittest_file) as f:
test_classes = [l.rstrip() for l in f.readlines()]
mx.run([tests_image] + test_classes)
finally:
mx.rmtree(svmbuild)
def update_import_cmd(args):
"""Update our mx imports"""
if not args:
args = ["truffle"]
if "sulong" in args:
args.append("regex")
if "regex" in args:
args.append("sulong")
if "truffle" in args:
args.remove("truffle")
args += ["sulong", "regex"]
if "sulong" in args:
join = os.path.join
callback = lambda: shutil.copy(
join(
mx.dependency("SULONG_LEGACY").output, "include", "truffle.h"),
join(SUITE.dir, "graalpython", "com.oracle.graal.python.cext",
"include", "truffle.h")
) and shutil.copy(
join(mx.dependency("SULONG_HOME").output, "include", "polyglot.h"),
join(SUITE.dir, "graalpython", "com.oracle.graal.python.cext",
"include", "polyglot.h"))
else:
callback = None
for name in set(args):
update_import(name, callback=callback)
def check_libgraal_dependencies():
if mx.get_os() == 'windows':
return 'libgraal is unsupported on Windows'
graal_hotspot_library = mx.dependency('substratevm:GRAAL_HOTSPOT_LIBRARY', fatalIfMissing=False)
if not graal_hotspot_library:
return 'libgraal dependency substratevm:GRAAL_HOTSPOT_LIBRARY is missing'
truffle_compiler_library = mx.dependency('compiler:GRAAL_TRUFFLE_COMPILER_LIBGRAAL', fatalIfMissing=False)
if not truffle_compiler_library:
return 'libgraal dependency compiler:GRAAL_TRUFFLE_COMPILER_LIBGRAAL is missing'
return None
def cinterfacetutorial(native_image, args=None):
"""Build and run the tutorial for the C interface"""
args = [] if args is None else args
tutorial_proj = mx.dependency('com.oracle.svm.tutorial')
cSourceDir = join(tutorial_proj.dir, 'native')
buildDir = join(svmbuild_dir(), tutorial_proj.name, 'build')
# clean / create output directory
if exists(buildDir):
remove_tree(buildDir)
mkpath(buildDir)
# Build the shared library from Java code
native_image(['--shared', '-H:Path=' + buildDir, '-H:Name=libcinterfacetutorial',
'-H:CLibraryPath=' + tutorial_proj.dir, '-cp', tutorial_proj.output_dir()] + args)
# Build the C executable
mx.run(['cc', '-g', join(cSourceDir, 'cinterfacetutorial.c'),
'-I' + buildDir,
'-L' + buildDir, '-lcinterfacetutorial',
'-ldl', '-Wl,-rpath,' + buildDir,
'-o', join(buildDir, 'cinterfacetutorial')])
# Start the C executable
mx.run([buildDir + '/cinterfacetutorial'])
def unpackIcuData(args):
"""populate ICU4J localization data from jar file dependency"""
icu4jDataDir = join(_suite.dir, 'lib', 'icu4j')
# clean up first
if isdir(icu4jDataDir):
shutil.rmtree(icu4jDataDir)
icu4jPackageDir = 'com/ibm/icu/impl/data'
# unpack the files
icu4jDep = mx.dependency('ICU4J')
icu4jPath = icu4jDep.get_path(resolve=True)
mx.log("ICU4J dependency found in %s" % (icu4jPath))
with zipfile.ZipFile(icu4jPath, 'r') as zf:
toExtract = [
e for e in zf.namelist() if e.startswith(icu4jPackageDir)
and not e.endswith(".class") and not e.endswith(".html")
]
zf.extractall(icu4jDataDir, toExtract)
mx.log("%d files extracted to %s" % (len(toExtract), icu4jDataDir))
# move the stuff such that the path is stable
for f in os.listdir(join(icu4jDataDir, icu4jPackageDir)):
if re.match('icudt.*', f):
icu4jUnzippedDataPath = join(icu4jDataDir, icu4jPackageDir, f)
icu4jDataPath = join(icu4jDataDir, "icudt")
shutil.move(icu4jUnzippedDataPath, icu4jDataPath)
shutil.rmtree(join(icu4jDataDir, "com"))
mx.log(
'Use the following parameters when invoking svm version of js to make ICU4J localization data available for the runtime:\n-Dpolyglot.js.intl-402=true -Dcom.ibm.icu.impl.ICUBinary.dataPath=%s'
% icu4jDataPath)
def _write_llvm_config_java(constants, file_comment=None):
package_name = "com.oracle.truffle.llvm.toolchain.config"
project_name = package_name
class_name = "LLVMConfig"
source_gen_dir = mx.dependency(project_name).source_dirs()[0]
rel_file = package_name.split(".") + [class_name + ".java"]
src_file = os.path.join(source_gen_dir, *rel_file)
mx.ensure_dir_exists(os.path.dirname(src_file))
with open(src_file, "w") as fp:
mx.log("Generating {}".format(src_file))
fp.write(
COPYRIGHT_HEADER_BSD.format(
"package {package};".format(package=package_name)))
if file_comment:
fp.write("\n/**\n * {}\n */\n".format(file_comment))
fp.write("public abstract class {class_name} {{\n".format(
class_name=class_name))
fp.write("\n private {class_name}() {{}}\n\n".format(
class_name=class_name))
for const_name, value, description in constants:
fp.write(" /** {} */\n".format(description))
if isinstance(value, int):
fp.write(" public static final int {} = {};\n".format(
const_name, value))
else:
fp.write(
" public static final String {} = \"{}\";\n".format(
const_name, value))
fp.write("}\n")
def _find_version_base_project(versioned_project):
base_project_name = getattr(versioned_project, 'overlayTarget', None)
if base_project_name:
return mx.dependency(base_project_name, context=versioned_project)
extended_packages = versioned_project.extended_java_packages()
if not extended_packages:
mx.abort(
'Project with a multiReleaseJarVersion attribute must have sources in a package defined by project without multiReleaseJarVersion attribute',
context=versioned_project)
base_project = None
base_package = None
for extended_package in extended_packages:
for p in mx.projects():
if versioned_project != p and p.isJavaProject() and not hasattr(
p, 'multiReleaseJarVersion'):
if extended_package in p.defined_java_packages():
if base_project is None:
base_project = p
base_package = extended_package
else:
if base_project != p:
mx.abort(
'Multi-release jar versioned project {} must extend packages from exactly one project but extends {} from {} and {} from {}'
.format(versioned_project, extended_package, p,
base_project, base_package))
if not base_project:
mx.abort(
'Multi-release jar versioned project {} must extend package(s) from another project'
.format(versioned_project))
return base_project
def _cinterfacetutorial(native_image, args=None):
"""Build and run the tutorial for the C interface"""
args = [] if args is None else args
tutorial_proj = mx.dependency('com.oracle.svm.tutorial')
c_source_dir = join(tutorial_proj.dir, 'native')
build_dir = join(svmbuild_dir(), tutorial_proj.name, 'build')
# clean / create output directory
if exists(build_dir):
remove_tree(build_dir)
mkpath(build_dir)
# Build the shared library from Java code
native_image(['--shared', '-H:Path=' + build_dir, '-H:Name=libcinterfacetutorial',
'-H:CLibraryPath=' + tutorial_proj.dir, '-cp', tutorial_proj.output_dir()] + args)
# Build the C executable
if mx.get_os() != 'windows':
mx.run(['cc', '-g', join(c_source_dir, 'cinterfacetutorial.c'),
'-I.', '-L.', '-lcinterfacetutorial',
'-ldl', '-Wl,-rpath,' + build_dir,
'-o', 'cinterfacetutorial'],
cwd=build_dir)
else:
mx.run(['cl', '-MD', join(c_source_dir, 'cinterfacetutorial.c'),
'-I.', 'libcinterfacetutorial.lib'],
cwd=build_dir)
# Start the C executable
mx.run([join(build_dir, 'cinterfacetutorial')])
def cinterfacetutorial(native_image, args=None):
"""Build and run the tutorial for the C interface"""
args = [] if args is None else args
tutorial_proj = mx.dependency('com.oracle.svm.tutorial')
cSourceDir = join(tutorial_proj.dir, 'native')
buildDir = join(svmbuild_dir(), tutorial_proj.name, 'build')
# clean / create output directory
if exists(buildDir):
remove_tree(buildDir)
mkpath(buildDir)
# Build the shared library from Java code
native_image([
'-H:Kind=SHARED_LIBRARY', '-H:Path=' +
buildDir, '-H:Name=libcinterfacetutorial', '-H:CLibraryPath=' +
tutorial_proj.dir, '-cp',
tutorial_proj.output_dir()
] + args)
# Build the C executable
mx.run([
'cc', '-g',
join(cSourceDir,
'cinterfacetutorial.c'), '-I' + buildDir, '-L' + buildDir,
'-lcinterfacetutorial', '-ldl', '-Wl,-rpath,' + buildDir, '-o',
join(buildDir, 'cinterfacetutorial')
])
# Start the C executable
mx.run([buildDir + '/cinterfacetutorial'])
def build(args, vm=None):
if any([opt in args for opt in ['-h', '--help']]):
orig_command_build(args, vm)
mx.log('build: Checking SubstrateVM requirements for building ...')
if not _host_os_supported():
mx.abort('build: SubstrateVM can be built only on Darwin, Linux and Windows platforms')
graal_compiler_flags_dir = join(mx.dependency('substratevm:com.oracle.svm.driver').dir, 'resources')
def update_if_needed(version_tag, graal_compiler_flags):
flags_filename = 'graal-compiler-flags-' + version_tag + '.config'
flags_path = join(graal_compiler_flags_dir, flags_filename)
flags_contents = '\n'.join(graal_compiler_flags)
needs_update = True
try:
with open(flags_path, 'r') as flags_file:
if flags_file.read() == flags_contents:
needs_update = False
except:
pass
if needs_update:
with open(flags_path, 'w') as f:
print('Write file ' + flags_path)
f.write(flags_contents)
for version_tag in GRAAL_COMPILER_FLAGS_MAP:
update_if_needed(version_tag, GRAAL_COMPILER_FLAGS_BASE + GRAAL_COMPILER_FLAGS_MAP[version_tag])
orig_command_build(args, vm)
def native_image_on_jvm(args, **kwargs):
save_args = []
for arg in args:
if arg == '--no-server' or arg.startswith('--server'):
mx.warn('Ignoring server-mode native-image argument ' + arg)
else:
save_args.append(arg)
driver_cp = [
join(suite_native_image_root(), 'lib', subdir, '*.jar')
for subdir in ['boot', 'jvmci', 'graalvm']
]
driver_cp += [
join(suite_native_image_root(), 'lib', 'svm', tail)
for tail in ['*.jar', join('builder', '*.jar')]
]
driver_cp = list(
itertools.chain.from_iterable(glob.glob(cp) for cp in driver_cp))
svm_version = suite.release_version(snapshotSuffix='SNAPSHOT')
run_java([
'-Dorg.graalvm.version=' + svm_version,
'-Dnative-image.root=' + suite_native_image_root(), '-cp',
os.pathsep.join(driver_cp),
mx.dependency('substratevm:SVM_DRIVER').mainClass
] + save_args, **kwargs)
def _run_fuzz_tool(tool_name, tool_args, *args, **kwargs):
dist = 'SULONG_TOOLS'
tool = os.path.join(mx.dependency(dist, fatalIfMissing=True).get_output(), 'bin', tool_name)
if not os.path.exists(tool):
msg = "The executable {} does not exist: {}{}".format(tool_name, tool, os.linesep)
msg += "This might be solved by running: mx build --dependencies={}".format(dist)
mx.abort(msg)
return mx.run([tool] + tool_args, *args, **kwargs)
def needsBuild(self, newestInput):
if not exists(self.icuDir):
return (True, self.icuDir + " not found")
icu4jDep = mx.dependency('ICU4J')
icu4jPath = icu4jDep.get_path(resolve=True)
if getmtime(icu4jPath) > getmtime(self.icuDir):
return (True, self.icuDir + " is older than " + icu4jPath)
return (False, None)
def find_jvmti_asm_agent():
"""Find the path the JVMTI agent that records the disassembly"""
d = mx.dependency('com.oracle.jvmtiasmagent')
for source_file, _ in d.getArchivableResults(single=True):
if not os.path.exists(source_file):
mx.abort('{} hasn\'t been built yet'.format(source_file))
return source_file
return None
def svm_gate_body(args, tasks):
build_native_image_image()
with native_image_context(IMAGE_ASSERTION_FLAGS) as native_image:
with Task('image demos', tasks, tags=[GraalTags.helloworld]) as t:
if t:
javac_image(['--output-path', svmbuild_dir()])
javac_command = ' '.join(javac_image_command(svmbuild_dir()))
helloworld(['--output-path', svmbuild_dir(), '--javac-command', javac_command])
helloworld(['--output-path', svmbuild_dir(), '--shared']) # Build and run helloworld as shared library
cinterfacetutorial([])
clinittest([])
with Task('native unittests', tasks, tags=[GraalTags.test]) as t:
if t:
native_unittest(['--build-args', '--initialize-at-build-time'])
with Task('Run Truffle NFI unittests with SVM image', tasks, tags=["svmjunit"]) as t:
if t:
testlib = mx_subst.path_substitutions.substitute('-Dnative.test.lib=<path:truffle:TRUFFLE_TEST_NATIVE>/<lib:nativetest>')
native_unittest_args = ['com.oracle.truffle.nfi.test', '--build-args', '--initialize-at-build-time', '--language:nfi',
'-H:MaxRuntimeCompileMethods=1500', '--run-args', testlib, '--very-verbose', '--enable-timing']
native_unittest(native_unittest_args)
with Task('Truffle TCK', tasks, tags=[GraalTags.truffletck]) as t:
if t:
junit_native_dir = join(svmbuild_dir(), platform_name(), 'junit')
mkpath(junit_native_dir)
junit_tmp_dir = tempfile.mkdtemp(dir=junit_native_dir)
try:
unittest_deps = []
unittest_file = join(junit_tmp_dir, 'truffletck.tests')
_run_tests([], lambda deps, vm_launcher, vm_args: unittest_deps.extend(deps), _VMLauncher('dummy_launcher', None, mx_compiler.jdk), ['@Test', '@Parameters'], unittest_file, [], [re.compile('com.oracle.truffle.tck.tests')], None, mx.suite('truffle'))
if not exists(unittest_file):
mx.abort('TCK tests not found.')
unittest_deps.append(mx.dependency('truffle:TRUFFLE_SL_TCK'))
vm_image_args = mx.get_runtime_jvm_args(unittest_deps, jdk=mx_compiler.jdk)
tests_image = native_image(vm_image_args + ['--macro:truffle', '--initialize-at-build-time',
'--features=com.oracle.truffle.tck.tests.TruffleTCKFeature',
'-H:Class=org.junit.runner.JUnitCore', '-H:IncludeResources=com/oracle/truffle/sl/tck/resources/.*',
'-H:MaxRuntimeCompileMethods=3000'])
with open(unittest_file) as f:
test_classes = [l.rstrip() for l in f.readlines()]
mx.run([tests_image, '-Dtck.inlineVerifierInstrument=false'] + test_classes)
finally:
remove_tree(junit_tmp_dir)
build_native_image_image(config=_graalvm_js_config)
with native_image_context(IMAGE_ASSERTION_FLAGS, config=_graalvm_js_config) as native_image:
with Task('JavaScript', tasks, tags=[GraalTags.js]) as t:
if t:
js = build_js(native_image)
test_run([js, '-e', 'print("hello:" + Array.from(new Array(10), (x,i) => i*i ).join("|"))'], 'hello:0|1|4|9|16|25|36|49|64|81\n')
test_js(js, [('octane-richards', 1000, 100, 300)])
with Task('maven plugin checks', tasks, tags=[GraalTags.maven]) as t:
if t:
maven_plugin_install(["--deploy-dependencies"])
maven_plugin_test([])
def _run_espresso_meta(args, nonZeroIsFatal=True):
"""Run Espresso (standalone) on Espresso (launcher)"""
return _run_espresso_launcher(
[
'--vm.Xss4m',
'-Dtruffle.class.path.append=' + mx.dependency('ESPRESSO').
path, # on GraalVM the EspressoLanguageProvider must be visible to the GraalVMLocator
] + _espresso_standalone_command(args),
nonZeroIsFatal=nonZeroIsFatal)
def build_libgraal(image_args):
msg = check_libgraal_dependencies()
if msg:
return msg
graal_hotspot_library = mx.dependency('substratevm:GRAAL_HOTSPOT_LIBRARY')
truffle_compiler_library = mx.dependency('compiler:GRAAL_TRUFFLE_COMPILER_LIBGRAAL')
truffle_api = mx.dependency('truffle:TRUFFLE_API')
libgraal_args = ['-H:Name=libjvmcicompiler', '--shared', '-cp', os.pathsep.join([graal_hotspot_library.classpath_repr(), truffle_compiler_library.classpath_repr()]),
'--features=com.oracle.svm.graal.hotspot.libgraal.HotSpotGraalLibraryFeature',
'-J-Xbootclasspath/a:' + truffle_api.classpath_repr(),
'-H:-UseServiceLoaderFeature',
'-H:+AllowFoldMethods',
'-Djdk.vm.ci.services.aot=true']
native_image_on_jvm(libgraal_args + image_args)
return None
def run_in_graalvm(self, cwd, args, extra_polyglot_args, host_vm):
# In GraalVM we run the Java benchmarks driver like one would run any other Java application
# that embeds GraalPython on GraalVM. We need to add the dependencies on class path, and since
# we use run_java, we need to do some output postprocessing that normally run_launcher would do
with environ(self._env or {}):
cp = self.get_classpath()
jhm = mx.dependency("mx:JMH_1_21")
cp_deps = [
mx.distribution('GRAALPYTHON_BENCH', fatalIfMissing=True),
jhm,
mx.dependency("sdk:LAUNCHER_COMMON")
] + jhm.deps
cp += [x.classpath_repr() for x in cp_deps]
java_args = ['-cp', ':'.join(cp)] + [self.launcher_class()]
out = mx.TeeOutputCapture(mx.OutputCapture())
code = host_vm.run_java(java_args + extra_polyglot_args + args, cwd=cwd, out=out, err=out)
out = out.underlying.data
dims = host_vm.dimensions(cwd, args, code, out)
return code, out, dims
def llvm_extra_tool(args=None, out=None, **kwargs):
if len(args) < 1:
mx.abort("usage: llvm-extra-tool <llvm-tool> [args...]")
program = args[0]
dep = mx.dependency(_LLVM_EXTRA_TOOL_DIST, fatalIfMissing=True)
llvm_program = os.path.join(dep.get_output(), 'bin', program)
try:
mx.run([llvm_program] + args[1:], out=out, nonZeroIsFatal=False, **kwargs)
except BaseException as e:
msg = "{}\n".format(e)
msg += "This might be solved by running: mx build --dependencies={}".format(_LLVM_EXTRA_TOOL_DIST)
mx.abort(msg)
def native_image_on_jvm(args, **kwargs):
save_args = []
for arg in args:
if arg == '--no-server' or arg.startswith('--server'):
mx.warn('Ignoring server-mode native-image argument ' + arg)
else:
save_args.append(arg)
driver_cp = [join(suite_native_image_root(), 'lib', subdir, '*.jar') for subdir in ['boot', 'jvmci', 'graalvm']]
driver_cp += [join(suite_native_image_root(), 'lib', 'svm', tail) for tail in ['*.jar', join('builder', '*.jar')]]
driver_cp = list(itertools.chain.from_iterable(glob.glob(cp) for cp in driver_cp))
run_java(['-Dnative-image.root=' + suite_native_image_root(), '-cp', ":".join(driver_cp),
mx.dependency('substratevm:SVM_DRIVER').mainClass] + save_args, **kwargs)
def native_image_on_jvm(args):
driver_cp = [
join(suite_native_image_root(), 'lib', subdir, '*.jar')
for subdir in ['boot', 'jvmci', 'graalvm']
]
driver_cp += [
join(suite_native_image_root(), 'lib', 'svm', tail)
for tail in ['*.jar', join('builder', '*.jar')]
]
driver_cp = list(
itertools.chain.from_iterable(glob.glob(cp) for cp in driver_cp))
run_java([
'-Dnative-image.root=' +
suite_native_image_root(), '-cp', ":".join(driver_cp),
mx.dependency('substratevm:SVM_DRIVER').mainClass
] + args)
def update_import_cmd(args):
for name in args:
callback = None
if name == "sulong":
join = os.path.join
callback=lambda: shutil.copy(
join(_sulong.dir, "include", "truffle.h"),
join(_suite.dir, "graalpython", "com.oracle.graal.python.cext", "include", "truffle.h")
) and shutil.copy(
join(mx.dependency("SULONG_LIBS").output, "polyglot.h"),
join(_suite.dir, "graalpython", "com.oracle.graal.python.cext", "include", "polyglot.h")
)
# make sure that sulong and regex are the same version
if name == "regex":
update_import("sulong", callback=callback)
elif name == "sulong":
update_import("regex", callback=callback)
update_import(name, callback=callback)
def maven_plugin_install(args):
# First install native-image-maven-plugin dependencies into local maven repository
deps = []
def visit(dep, edge):
if isinstance(dep, mx.Distribution):
deps.append(dep)
mx.walk_deps([mx.dependency('substratevm:SVM_DRIVER')],
visit=visit,
ignoredEdges=[mx.DEP_ANNOTATION_PROCESSOR, mx.DEP_BUILD])
svmVersion = '{0}-SNAPSHOT'.format(suite.vc.parent(suite.vc_dir))
mx.maven_deploy([
'--version-string', svmVersion, '--suppress-javadoc',
'--all-distributions', '--validate=none', '--all-suites',
'--skip-existing', '--only', ','.join(dep.qualifiedName()
for dep in deps)
])
deploy_native_image_maven_plugin(svmVersion)
success_message = [
'',
'Use the following plugin snippet to enable native-image building for your maven project:',
'',
'<plugin>',
' <groupId>com.oracle.substratevm</groupId>',
' <artifactId>native-image-maven-plugin</artifactId>',
' <version>' + svmVersion + '</version>',
' <executions>',
' <execution>',
' <goals>',
' <goal>native-image</goal>',
' </goals>',
' <phase>package</phase>',
' </execution>',
' </executions>',
'</plugin>',
'',
]
mx.log('\n'.join(success_message))
def maven_plugin_install(args):
# First install native-image-maven-plugin dependencies into local maven repository
deps = []
def visit(dep, edge):
if isinstance(dep, mx.Distribution):
deps.append(dep)
mx.walk_deps([mx.dependency('substratevm:SVM_DRIVER')], visit=visit, ignoredEdges=[mx.DEP_ANNOTATION_PROCESSOR, mx.DEP_BUILD])
svmVersion = '{0}-SNAPSHOT'.format(suite.vc.parent(suite.vc_dir))
mx.maven_deploy([
'--version-string', svmVersion,
'--suppress-javadoc',
'--all-distributions',
'--validate=none',
'--all-suites',
'--skip-existing',
'--only', ','.join(dep.qualifiedName() for dep in deps)
])
deploy_native_image_maven_plugin(svmVersion)
success_message = [
'',
'Use the following plugin snippet to enable native-image building for your maven project:',
'',
'<plugin>',
' <groupId>com.oracle.substratevm</groupId>',
' <artifactId>native-image-maven-plugin</artifactId>',
' <version>' + svmVersion + '</version>',
' <executions>',
' <execution>',
' <goals>',
' <goal>native-image</goal>',
' </goals>',
' <phase>package</phase>',
' </execution>',
' </executions>',
'</plugin>',
'',
]
mx.log('\n'.join(success_message))
def native_image_layout_dists(subdir, dists):
native_image_dists = [mx.dependency(dist_name) for dist_name in dists]
native_image_layout(native_image_dists, subdir, native_image_root)
def names_to_dists(dist_names):
deps = [mx.dependency(dist_name) for dist_name in dist_names]
return [
dep for dep in deps if not dep.isDistribution() or dep.exists()
]
def make_dist_rule(dist, mf):
def path_dist_relative(p):
return os.path.relpath(p, dist.suite.dir)
shortName = short_dist_name(dist.name)
jdkDeployedDists = get_jdk_deployed_dists()
jarName = os.path.basename(dist.path)
sourcesVariableName = shortName + "_SRC"
depJarVariableName = shortName + "_DEP_JARS"
sources = []
resources = []
sortedDeps = dist.sorted_deps(True, transitive=False, includeAnnotationProcessors=True)
projects = filter_projects(sortedDeps, mx.Project)
targetPathPrefix = "$(TARGET)/"
libraryDeps = [path_dist_relative(l.get_path(False)) for l in filter_projects(sortedDeps, mx.Library)]
annotationProcessorDeps = []
distDeps = dist.get_dist_deps(includeSelf=False, transitive=True)
distDepProjects = []
for d in distDeps:
update_list(distDepProjects, d.sorted_deps(includeLibs=False, transitive=True))
classPath = [targetPathPrefix + os.path.basename(d.path) for d in distDeps] + libraryDeps \
+ [path_dist_relative(mx.dependency(name).path) for name in dist.excludedDependencies]
for p in projects:
if p.definedAnnotationProcessors != None and p.definedAnnotationProcessorsDist != dist:
update_list(annotationProcessorDeps, [p])
for p in projects:
projectDir = path_dist_relative(p.dir)
if p not in distDepProjects and p not in annotationProcessorDeps:
for src in [projectDir + '/' + d for d in p.srcDirs]:
sources.append("$(shell find {} -type f 2> /dev/null)".format(src))
metaInf = src + "/META-INF"
if os.path.exists(metaInf):
resources.append(metaInf)
sourceLines = sourcesVariableName + " = " + ("\n" + sourcesVariableName + " += ").join(sources)
apPaths = []
apDistNames = []
apDistVariableNames = []
for p in annotationProcessorDeps:
apPaths.append(path_dist_relative(p.definedAnnotationProcessorsDist.path))
name = short_dist_name(p.definedAnnotationProcessorsDist.name)
apDistNames.append(name)
apDistVariableNames.append("$(" + name + "_JAR)")
shouldExport = dist.name in jdkDeployedDists
props = {
"name": shortName,
"jarName": targetPathPrefix + jarName,
"depJarsVariableAccess": "$(" + depJarVariableName + ")" if len(classPath) > 0 else "",
"depJarsVariable": depJarVariableName,
"sourceLines": sourceLines,
"sourcesVariableName": sourcesVariableName,
"annotationProcessors": " ".join(apDistVariableNames),
"cpAnnotationProcessors": ":".join(apDistVariableNames),
"jarDeps": " ".join(classPath),
"copyResources": " ".join(resources)
}
mf.add_definition(sourceLines)
mf.add_definition("{name}_JAR = {jarName}".format(**props))
if len(classPath) > 0: mf.add_definition("{depJarsVariable} = {jarDeps}".format(**props))
if shouldExport: mf.add_definition("EXPORTED_FILES += $({name}_JAR)".format(**props))
mf.add_rule("""$({name}_JAR): $({sourcesVariableName}) {annotationProcessors} {depJarsVariableAccess}
\t$(call build_and_jar,{cpAnnotationProcessors},$(subst $(space),:,{depJarsVariableAccess}),{copyResources},$({name}_JAR))
""".format(**props))
return
def jlink_new_jdk(jdk, dst_jdk_dir, module_dists, ignore_dists,
root_module_names=None,
missing_export_target_action='create',
with_source=lambda x: True,
vendor_info=None,
dedup_legal_notices=True,
use_upgrade_module_path=False):
"""
Uses jlink from `jdk` to create a new JDK image in `dst_jdk_dir` with `module_dists` and
their dependencies added to the JDK image, replacing any existing modules of the same name.
:param JDKConfig jdk: source JDK
:param str dst_jdk_dir: path to use for the jlink --output option
:param list module_dists: list of distributions defining modules
:param list ignore_dists: list of distributions that should be ignored for missing_export_target_action
:param list root_module_names: list of strings naming the module root set for the new JDK image.
The named modules must either be in `module_dists` or in `jdk`. If None, then
the root set will be all the modules in ``module_dists` and `jdk`.
:param str missing_export_target_action: the action to perform for a qualified export target that
is not present in `module_dists` and does not have a hash stored in java.base.
The choices are:
"create" - an empty module is created
"error" - raise an error
None - do nothing
:param lambda with_source: returns True if the sources of a module distribution must be included in the new JDK
:param dict vendor_info: values for the jlink vendor options added by JDK-8232080
:param bool use_upgrade_module_path: if True, then instead of linking `module_dists` into the image, resolve
them via --upgrade-module-path at image runtime
:return bool: False if use_upgrade_module_path == True and the existing image is up to date otherwise True
"""
assert callable(with_source)
if jdk.javaCompliance < '9':
mx.abort('Cannot derive a new JDK from ' + jdk.home + ' with jlink since it is not JDK 9 or later')
exploded_java_base_module = join(jdk.home, 'modules', 'java.base')
if exists(exploded_java_base_module):
mx.abort('Cannot derive a new JDK from ' + jdk.home + ' since it appears to be a developer build with exploded modules')
jimage = join(jdk.home, 'lib', 'modules')
jmods_dir = join(jdk.home, 'jmods')
if not isfile(jimage):
mx.abort('Cannot derive a new JDK from ' + jdk.home + ' since ' + jimage + ' is missing or is not an ordinary file')
if not isdir(jmods_dir):
mx.abort('Cannot derive a new JDK from ' + jdk.home + ' since ' + jmods_dir + ' is missing or is not a directory')
# Exclude jdk.aot due to GR-10545 and JDK-8255616
jdk_modules = {jmd.name: jmd for jmd in jdk.get_modules() if jmd.name != 'jdk.aot'}
modules = [as_java_module(dist, jdk) for dist in module_dists]
module_names = frozenset((m.name for m in modules))
all_module_names = frozenset(list(jdk_modules.keys())) | module_names
# Read hashes stored in java.base (the only module in the JDK where hashes are stored)
hashes = _read_java_base_hashes(jdk)
build_dir = mx.ensure_dir_exists(join(dst_jdk_dir + ".build"))
# Directory under dst_jdk_dir for artifacts related to use_upgrade_module_path
upgrade_dir = join(dst_jdk_dir, 'upgrade_modules_support')
# Map from JavaModuleDescriptors to post-jlink jar location.
synthetic_modules = OrderedDict()
try:
ignore_module_names = set(mx_javamodules.get_module_name(mx.dependency(ignore_dist)) for ignore_dist in ignore_dists)
# Synthesize modules for targets of qualified exports that are not present in `modules`.
# Without this, runtime module resolution will fail due to missing modules.
target_requires = {}
for jmd in modules:
for targets in jmd.exports.values():
for target in targets:
if target not in all_module_names and target not in ignore_module_names and target not in hashes:
target_requires.setdefault(target, set()).add(jmd.name)
if target_requires and missing_export_target_action is not None:
if missing_export_target_action == 'error':
mx.abort('Target(s) of qualified exports cannot be resolved: ' + '.'.join(target_requires.keys()))
assert missing_export_target_action == 'create', 'invalid value for missing_export_target_action: ' + str(missing_export_target_action)
for name, requires in sorted(target_requires.items()):
module_jar = join(build_dir, name + '.jar')
jmd = JavaModuleDescriptor(name, {}, requires={module: [] for module in requires}, uses=set(), provides={}, jarpath=module_jar)
module_build_dir = mx.ensure_dir_exists(join(build_dir, name))
module_info = jmd.as_module_info()
module_info_java = join(module_build_dir, 'module-info.java')
module_info_class = join(module_build_dir, 'module-info.class')
dst_module_jar = join(upgrade_dir, name + '.jar')
synthetic_modules[jmd] = dst_module_jar
if use_upgrade_module_path and exists(dst_module_jar):
with ZipFile(dst_module_jar, 'r') as zf:
previous_module_info = zf.read('module-info.java').decode()
if previous_module_info == module_info:
mx.logv('[Reusing synthetic module {}]'.format(name))
os.rename(dst_module_jar, module_jar)
continue
mx.logv('[Rebuilding synthetic module {} as module descriptor changed]'.format(name))
with open(module_info_java, 'w') as fp:
fp.write(module_info)
mx.run([jdk.javac, '-d', module_build_dir,
'--limit-modules=java.base,' + ','.join(jmd.requires.keys()),
'--module-path=' + os.pathsep.join((m.jarpath for m in modules)),
module_info_java])
with ZipFile(module_jar, 'w') as zf:
zf.write(module_info_java, 'module-info.java')
zf.write(module_info_class, 'module-info.class')
if exists(jmd.get_jmod_path()):
os.remove(jmd.get_jmod_path())
if not use_upgrade_module_path:
mx.run([jdk.javac.replace('javac', 'jmod'), 'create', '--class-path=' + module_build_dir, jmd.get_jmod_path()])
modules.extend(synthetic_modules.keys())
module_names = frozenset((m.name for m in modules))
all_module_names = frozenset(list(jdk_modules.keys())) | module_names
# Edit lib/security/default.policy in java.base
patched_java_base = _patch_default_security_policy(build_dir, jmods_dir, dst_jdk_dir)
# Now build the new JDK image with jlink
jlink = [jdk.javac.replace('javac', 'jlink')]
if jdk_enables_jvmci_by_default(jdk):
# On JDK 9+, +EnableJVMCI forces jdk.internal.vm.ci to be in the root set
jlink += ['-J-XX:-EnableJVMCI', '-J-XX:-UseJVMCICompiler']
jlink.append('--add-modules=' + ','.join(_get_image_root_modules(root_module_names, module_names, jdk_modules.keys(), use_upgrade_module_path)))
module_path = patched_java_base + os.pathsep + jmods_dir
if modules and not use_upgrade_module_path:
module_path = os.pathsep.join((m.get_jmod_path(respect_stripping=True) for m in modules)) + os.pathsep + module_path
jlink.append('--module-path=' + module_path)
jlink.append('--output=' + dst_jdk_dir)
# These options are derived from how OpenJDK runs jlink to produce the final runtime image.
jlink.extend(['-J-XX:+UseSerialGC', '-J-Xms32M', '-J-Xmx512M', '-J-XX:TieredStopAtLevel=1'])
jlink.append('-J-Dlink.debug=true')
if dedup_legal_notices:
jlink.append('--dedup-legal-notices=error-if-not-same-content')
jlink.append('--keep-packaged-modules=' + join(dst_jdk_dir, 'jmods'))
vm_options_path = join(upgrade_dir, 'vm_options')
vm_options = _get_image_vm_options(jdk, use_upgrade_module_path, modules, synthetic_modules)
if vm_options:
jlink.append('--add-options=' + ' '.join(vm_options))
if jdk_has_new_jlink_options(jdk) and vendor_info is not None:
for name, value in vendor_info.items():
jlink.append('--' + name + '=' + value)
release_file = join(jdk.home, 'release')
if isfile(release_file):
jlink.append('--release-info=' + release_file)
if exists(dst_jdk_dir):
if use_upgrade_module_path and _vm_options_match(vm_options, vm_options_path):
mx.logv('[Existing JDK image {} is up to date]'.format(dst_jdk_dir))
return False
mx.rmtree(dst_jdk_dir)
# TODO: investigate the options below used by OpenJDK to see if they should be used:
# --order-resources: specifies order of resources in generated lib/modules file.
# This is apparently not so important if a CDS archive is available.
# --generate-jli-classes: pre-generates a set of java.lang.invoke classes.
# See https://github.com/openjdk/jdk/blob/master/make/GenerateLinkOptData.gmk
mx.logv('[Creating JDK image in {}]'.format(dst_jdk_dir))
mx.run(jlink)
if use_upgrade_module_path:
# Move synthetic upgrade modules into final location
for jmd, jarpath in synthetic_modules.items():
mx.ensure_dir_exists(dirname(jarpath))
os.rename(jmd.jarpath, jarpath)
# Persist VM options cooked into image to be able to skip a subsequent
# jlink execution if the options do not change.
with open(vm_options_path, 'w') as fp:
fp.write(os.linesep.join(vm_options))
# Create src.zip in new JDK image
_copy_src_zip(jdk.home, dst_jdk_dir, modules, lambda jmd: not use_upgrade_module_path and with_source(jmd.dist))
mx.logv('[Copying static libraries]')
lib_directory = join(jdk.home, 'lib', 'static')
if exists(lib_directory):
dst_lib_directory = join(dst_jdk_dir, 'lib', 'static')
try:
mx.copytree(lib_directory, dst_lib_directory)
except shutil.Error as e:
# On AArch64, there can be a problem in the copystat part
# of copytree which occurs after file and directory copying
# has successfully completed. Since the metadata doesn't
# matter in this case, just ensure that the content was copied.
for root, _, lib_files in os.walk(lib_directory):
relative_root = os.path.relpath(root, dst_lib_directory)
for lib in lib_files:
src_lib_path = join(root, lib)
dst_lib_path = join(dst_lib_directory, relative_root, lib)
if not exists(dst_lib_path):
mx.abort('Error copying static libraries: {} missing in {}{}Original copytree error: {}'.format(
join(relative_root, lib), dst_lib_directory, os.linesep, e))
src_lib_hash = mx.sha1OfFile(src_lib_path)
dst_lib_hash = mx.sha1OfFile(dst_lib_path)
if src_lib_hash != dst_lib_hash:
mx.abort('Error copying static libraries: {} (hash={}) and {} (hash={}) differ{}Original copytree error: {}'.format(
src_lib_path, src_lib_hash,
dst_lib_path, dst_lib_hash,
os.linesep, e))
# Allow older JDK versions to work
else:
lib_prefix = mx.add_lib_prefix('')
lib_suffix = mx.add_static_lib_suffix('')
lib_directory = join(jdk.home, 'lib')
dst_lib_directory = join(dst_jdk_dir, 'lib')
for f in os.listdir(lib_directory):
if f.startswith(lib_prefix) and f.endswith(lib_suffix):
lib_path = join(lib_directory, f)
if isfile(lib_path):
shutil.copy2(lib_path, dst_lib_directory)
finally:
if not mx.get_opts().verbose:
# Preserve build directory so that javac command can be re-executed
# by cutting and pasting verbose output.
shutil.rmtree(build_dir)
if not use_upgrade_module_path:
# Create CDS archive (https://openjdk.java.net/jeps/341).
out = mx.OutputCapture()
mx.logv('[Creating CDS shared archive]')
if mx.run([mx.exe_suffix(join(dst_jdk_dir, 'bin', 'java')), '-Xshare:dump', '-Xmx128M', '-Xms128M'], out=out, err=out, nonZeroIsFatal=False) != 0:
mx.log(out.data)
mx.abort('Error generating CDS shared archive')
else:
# -Xshare is incompatible with --upgrade-module-path
pass
return True
def names_to_dists(dist_names):
return [mx.dependency(dist_name) for dist_name in dist_names]
def names_to_dists(dist_names):
return [mx.dependency(dist_name) for dist_name in dist_names]
def findBundledLLVMProgram(llvm_program):
llvm_dist = 'LLVM_TOOLCHAIN'
dep = mx.dependency(llvm_dist, fatalIfMissing=True)
return os.path.join(dep.get_output(), 'bin', mx.exe_suffix(llvm_program))
def typeFilter(project): # filters
if isinstance(project, str):
project = mx.dependency(project, True)
return isinstance(project, t)
def fuzz(args=None, out=None):
parser = ArgumentParser(prog='mx fuzz', description='')
parser.add_argument('--seed',
help='Seed used for randomness.',
metavar='<seed>',
type=int,
default=int(time.time()))
parser.add_argument(
'--size',
help=
'Approximate size for the generated testcases in lines of code. (default: %(default)s)',
metavar='<size>',
type=int,
default=30)
parser.add_argument(
'--timeout',
help=
'Timeout for running the generated program. (default: %(default)s)',
metavar='<timeout>',
type=int,
default=10)
parser.add_argument(
'--generator',
help='Tool used for generating the testcases. (default: %(default)s)',
choices=("llvm-stress", "csmith"),
default="llvm-stress")
parser.add_argument(
'--nrtestcases',
help='Number of testcases to be generated. (default: %(default)s)',
metavar='<nrtestcases>',
type=int,
default=10)
parser.add_argument('outdir',
help='The output directory.',
metavar='<outdir>')
parsed_args = parser.parse_args(args)
tmp_dir = None
try:
tmp_dir = tempfile.mkdtemp()
tmp_ll = os.path.join(tmp_dir, 'tmp.ll')
tmp_main_ll = os.path.join(tmp_dir, 'tmp.main.ll')
tmp_c = os.path.join(tmp_dir, 'tmp.c')
tmp_out = os.path.join(tmp_dir, 'tmp.out')
tmp_sulong_out = os.path.join(tmp_dir, 'tmp_sulong_out.txt')
tmp_bin_out = os.path.join(tmp_dir, 'tmp_bin_out.txt')
tmp_sulong_err = os.path.join(tmp_dir, 'tmp_sulong_err.txt')
tmp_bin_err = os.path.join(tmp_dir, 'tmp_bin_err.txt')
rand = Random(parsed_args.seed)
passed = 0
invalid = 0
gen = []
for _ in range(parsed_args.nrtestcases):
toolchain_clang = mx_sulong._get_toolchain_tool("native,CC")
if parsed_args.generator == "llvm-stress":
_run_fuzz_tool("llvm-stress", [
"-o", tmp_ll, "--size",
str(parsed_args.size), "--seed",
str(rand.randint(0, 10000000))
])
fuzz_main = os.path.join(
mx.dependency('SULONG_TOOLS',
fatalIfMissing=True).get_output(), "src",
"fuzzmain.c")
mx.run([
toolchain_clang, "-O0", "-Wno-everything", "-o", tmp_out,
tmp_ll, fuzz_main
])
mx_sulong.llvm_tool([
"clang", "-O0", "-Wno-everything", "-S", "-emit-llvm",
"-o", tmp_main_ll, fuzz_main
])
mx_sulong.llvm_tool(
["llvm-link", "-o", tmp_ll, tmp_ll, tmp_main_ll])
mx_sulong.llvm_tool(["llvm-dis", "-o", tmp_ll, tmp_ll])
else:
csmith_exe = "csmith"
csmith_headers = mx.get_env('CSMITH_HEADERS', None)
if not csmith_headers:
mx.abort("Environment variable `CSMITH_HEADERS` not set")
mx.run([
csmith_exe, "-o", tmp_c, "--seed",
str(rand.randint(0, 10000000))
])
mx.run([
toolchain_clang, "-O0", "-Wno-everything",
"-I" + csmith_headers, "-o", tmp_out, tmp_c
])
mx_sulong.llvm_tool([
"clang", "-O0", "-Wno-everything", "-S", "-emit-llvm",
"-I" + csmith_headers, "-o", tmp_ll, tmp_c
])
gen.append((tmp_c, 'autogen.c'))
timeout = parsed_args.timeout
with open(tmp_sulong_out, 'w') as o, open(tmp_sulong_err,
'w') as e:
mx_sulong.runLLVM([
'--llvm.llDebug', '--llvm.traceIR',
'--experimental-options', tmp_out
],
timeout=timeout,
nonZeroIsFatal=False,
out=o,
err=e)
with open(tmp_bin_out, 'w') as o, open(tmp_bin_err, 'w') as e:
try:
mx.run([tmp_out], timeout=timeout, out=o, err=e)
except SystemExit:
invalid += 1
continue
if all(
filecmp.cmp(sulong_f, bin_f, shallow=False)
for sulong_f, bin_f in ((tmp_sulong_out, tmp_bin_out),
(tmp_sulong_err, tmp_bin_err))):
passed += 1
else:
now = str(datetime.datetime.now())
now = now.replace(":", "_").replace(" ", "_")
current_out_dir = os.path.join(
parsed_args.outdir, now + "_" + parsed_args.generator)
os.makedirs(current_out_dir)
gen += [
(tmp_ll, 'autogen.ll'),
(tmp_out, 'autogen'),
(tmp_sulong_out, 'sulong_out.txt'),
(tmp_bin_out, 'bin_out.txt'),
(tmp_sulong_err, 'sulong_err.txt'),
(tmp_bin_err, 'bin_err.txt'),
]
for tmp_f, gen_f_name in gen:
shutil.copy(tmp_f, os.path.join(current_out_dir,
gen_f_name))
finally:
if tmp_dir:
shutil.rmtree(tmp_dir)
mx.log("Test report")
mx.log("total testcases: {} seed: {}".format(parsed_args.nrtestcases,
parsed_args.seed))
mx.log("interesting testcases: {} invalid testcases: {}".format(
parsed_args.nrtestcases - invalid - passed, invalid))
def jlink_new_jdk(jdk,
dst_jdk_dir,
module_dists,
ignore_dists,
root_module_names=None,
missing_export_target_action='create',
with_source=lambda x: True,
vendor_info=None,
dedup_legal_notices=True):
"""
Uses jlink from `jdk` to create a new JDK image in `dst_jdk_dir` with `module_dists` and
their dependencies added to the JDK image, replacing any existing modules of the same name.
:param JDKConfig jdk: source JDK
:param str dst_jdk_dir: path to use for the jlink --output option
:param list module_dists: list of distributions defining modules
:param list ignore_dists: list of distributions that should be ignored for missing_export_target_action
:param list root_module_names: list of strings naming the module root set for the new JDK image.
The named modules must either be in `module_dists` or in `jdk`. If None, then
the root set will be all the modules in ``module_dists` and `jdk`.
:param str missing_export_target_action: the action to perform for a qualified export target that
is not present in `module_dists` and does not have a hash stored in java.base.
The choices are:
"create" - an empty module is created
"error" - raise an error
None - do nothing
:param lambda with_source: returns True if the sources of a module distribution must be included in the new JDK
:param dict vendor_info: values for the jlink vendor options added by JDK-8232080
"""
assert callable(with_source)
if jdk.javaCompliance < '9':
mx.abort('Cannot derive a new JDK from ' + jdk.home +
' with jlink since it is not JDK 9 or later')
exploded_java_base_module = join(jdk.home, 'modules', 'java.base')
if exists(exploded_java_base_module):
mx.abort(
'Cannot derive a new JDK from ' + jdk.home +
' since it appears to be a developer build with exploded modules')
jimage = join(jdk.home, 'lib', 'modules')
jmods_dir = join(jdk.home, 'jmods')
if not isfile(jimage):
mx.abort('Cannot derive a new JDK from ' + jdk.home + ' since ' +
jimage + ' is missing or is not an ordinary file')
if not isdir(jmods_dir):
mx.abort('Cannot derive a new JDK from ' + jdk.home + ' since ' +
jmods_dir + ' is missing or is not a directory')
# Exclude jdk.aot due to GR-10545 and JDK-8255616
jdk_modules = {
jmd.name: jmd
for jmd in jdk.get_modules() if jmd.name != 'jdk.aot'
}
modules = [as_java_module(dist, jdk) for dist in module_dists]
all_module_names = frozenset(
list(jdk_modules.keys()) + [m.name for m in modules])
# Read hashes stored in java.base (the only module in the JDK where hashes are stored)
out = mx.LinesOutputCapture()
mx.run([
jdk.exe_path('jmod'), 'describe',
jdk_modules['java.base'].get_jmod_path()
],
out=out)
lines = out.lines
hashes = {}
for line in lines:
if line.startswith('hashes'):
parts = line.split()
assert len(
parts
) == 4, 'expected hashes line to have 4 fields, got {} fields: {}'.format(
len(parts), line)
_, module_name, algorithm, hash_value = parts
hashes[module_name] = (algorithm, hash_value)
build_dir = mx.ensure_dir_exists(join(dst_jdk_dir + ".build"))
synthetic_modules = []
try:
ignore_module_names = set(
mx_javamodules.get_module_name(mx.dependency(ignore_dist))
for ignore_dist in ignore_dists)
# Synthesize modules for targets of qualified exports that are not present in `modules`.
# Without this, runtime module resolution will fail due to missing modules.
target_requires = {}
for jmd in modules:
for targets in jmd.exports.values():
for target in targets:
if target not in all_module_names and target not in ignore_module_names and target not in hashes:
target_requires.setdefault(target, set()).add(jmd.name)
if target_requires and missing_export_target_action is not None:
if missing_export_target_action == 'error':
mx.abort(
'Target(s) of qualified exports cannot be resolved: ' +
'.'.join(target_requires.keys()))
assert missing_export_target_action == 'create', 'invalid value for missing_export_target_action: ' + str(
missing_export_target_action)
for name, requires in target_requires.items():
module_jar = join(build_dir, name + '.jar')
jmd = JavaModuleDescriptor(
name, {},
requires={module: []
for module in requires},
uses=set(),
provides={},
jarpath=module_jar)
synthetic_modules.append(jmd)
module_build_dir = mx.ensure_dir_exists(join(build_dir, name))
module_info_java = join(module_build_dir, 'module-info.java')
module_info_class = join(module_build_dir, 'module-info.class')
with open(module_info_java, 'w') as fp:
print(jmd.as_module_info(), file=fp)
mx.run([
jdk.javac, '-d', module_build_dir,
'--limit-modules=java.base,' +
','.join(jmd.requires.keys()),
'--module-path=' + os.pathsep.join(
(m.jarpath for m in modules)), module_info_java
])
with ZipFile(module_jar, 'w') as zf:
zf.write(module_info_class, basename(module_info_class))
if exists(jmd.get_jmod_path()):
os.remove(jmd.get_jmod_path())
mx.run([
jdk.javac.replace('javac', 'jmod'), 'create',
'--class-path=' + module_build_dir,
jmd.get_jmod_path()
])
modules.extend(synthetic_modules)
all_module_names = frozenset(
list(jdk_modules.keys()) + [m.name for m in modules])
# Extract src.zip from source JDK
jdk_src_zip = join(jdk.home, 'lib', 'src.zip')
dst_src_zip_contents = {}
if isfile(jdk_src_zip):
mx.logv('[Extracting ' + jdk_src_zip + ']')
with ZipFile(jdk_src_zip, 'r') as zf:
for name in zf.namelist():
if not name.endswith('/'):
dst_src_zip_contents[name] = zf.read(name)
else:
mx.warn("'{}' does not exist or is not a file".format(jdk_src_zip))
# Edit lib/security/default.policy in java.base
patched_java_base = join(build_dir, 'java.base.jmod')
with open(join(jmods_dir, 'java.base.jmod'),
'rb') as src_f, open(patched_java_base, 'wb') as dst_f:
jmod_header = src_f.read(4)
if len(jmod_header) != 4 or jmod_header != b'JM\x01\x00':
raise mx.abort("Unexpected jmod header: " +
b2a_hex(jmod_header).decode('ascii'))
dst_f.write(jmod_header)
policy_result = 'not found'
with ZipFile(src_f, 'r') as src_zip, ZipFile(
dst_f, 'w', src_zip.compression) as dst_zip:
for i in src_zip.infolist():
if i.filename[-1] == '/':
continue
src_member = src_zip.read(i)
if i.filename == 'lib/security/default.policy':
policy_result = 'unmodified'
if 'grant codeBase "jrt:/com.oracle.graal.graal_enterprise"'.encode(
'utf-8') not in src_member:
policy_result = 'modified'
src_member += """
grant codeBase "jrt:/com.oracle.graal.graal_enterprise" {
permission java.security.AllPermission;
};
""".encode('utf-8')
if 'grant codeBase "jrt:/org.graalvm.truffle"'.encode(
'utf-8') not in src_member:
policy_result = 'modified'
src_member += """
grant codeBase "jrt:/org.graalvm.truffle" {
permission java.security.AllPermission;
};
grant codeBase "jrt:/org.graalvm.sdk" {
permission java.security.AllPermission;
};
grant codeBase "jrt:/org.graalvm.locator" {
permission java.io.FilePermission "<<ALL FILES>>", "read";
permission java.util.PropertyPermission "*", "read,write";
permission java.lang.RuntimePermission "createClassLoader";
permission java.lang.RuntimePermission "getClassLoader";
permission java.lang.RuntimePermission "getenv.*";
};
grant codeBase "file:${java.home}/languages/-" {
permission java.security.AllPermission;
};
""".encode('utf-8')
dst_zip.writestr(i, src_member)
if policy_result == 'not found':
raise mx.abort(
"Couldn't find `lib/security/default.policy` in " +
join(jmods_dir, 'java.base.jmod'))
for jmd in modules:
# Remove existing sources for all the modules that we include
dst_src_zip_contents = {
key: dst_src_zip_contents[key]
for key in dst_src_zip_contents if not key.startswith(jmd.name)
}
if with_source(jmd.dist):
# Add the sources that we can share.
# Extract module sources
jmd_src_zip = jmd.jarpath[0:-len('.jar')] + '.src.zip'
if isfile(jmd_src_zip):
mx.logv('[Extracting ' + jmd_src_zip + ']')
with ZipFile(jmd_src_zip, 'r') as zf:
for name in zf.namelist():
if not name.endswith('/'):
dst_src_zip_contents[jmd.name + '/' +
name] = zf.read(name)
# Add module-info.java to sources
dst_src_zip_contents[jmd.name +
'/module-info.java'] = jmd.as_module_info(
extras_as_comments=False)
# Now build the new JDK image with jlink
jlink = [jdk.javac.replace('javac', 'jlink')]
if jdk_enables_jvmci_by_default(jdk):
# On JDK 9+, +EnableJVMCI forces jdk.internal.vm.ci to be in the root set
jlink += ['-J-XX:-EnableJVMCI', '-J-XX:-UseJVMCICompiler']
if root_module_names is not None:
missing = frozenset(root_module_names) - all_module_names
if missing:
mx.abort(
'Invalid module(s): {}.\nAvailable modules: {}'.format(
','.join(missing), ','.join(sorted(all_module_names))))
jlink.append('--add-modules=' + ','.join(root_module_names))
else:
jlink.append('--add-modules=' + ','.join(sorted(all_module_names)))
module_path = patched_java_base + os.pathsep + jmods_dir
if modules:
module_path = os.pathsep.join(
(m.get_jmod_path(respect_stripping=True)
for m in modules)) + os.pathsep + module_path
jlink.append('--module-path=' + module_path)
jlink.append('--output=' + dst_jdk_dir)
# These options are derived from how OpenJDK runs jlink to produce the final runtime image.
jlink.extend([
'-J-XX:+UseSerialGC', '-J-Xms32M', '-J-Xmx512M',
'-J-XX:TieredStopAtLevel=1'
])
jlink.append('-J-Dlink.debug=true')
if dedup_legal_notices:
jlink.append('--dedup-legal-notices=error-if-not-same-content')
jlink.append('--keep-packaged-modules=' + join(dst_jdk_dir, 'jmods'))
if jdk_has_new_jlink_options(jdk):
if jdk_omits_warning_for_jlink_set_ThreadPriorityPolicy(jdk):
thread_priority_policy_option = ' -XX:ThreadPriorityPolicy=1'
else:
mx.logv('[Creating JDK without -XX:ThreadPriorityPolicy=1]')
thread_priority_policy_option = ''
if jdk_supports_enablejvmciproduct(jdk):
non_synthetic_modules = [
m.name for m in modules if m not in synthetic_modules
]
if 'jdk.internal.vm.compiler' in non_synthetic_modules:
jlink.append(
'--add-options=-XX:+UnlockExperimentalVMOptions -XX:+EnableJVMCIProduct -XX:-UnlockExperimentalVMOptions'
+ thread_priority_policy_option)
else:
# Don't default to using JVMCI as JIT unless Graal is being updated in the image.
# This avoids unexpected issues with using the out-of-date Graal compiler in
# the JDK itself.
jlink.append(
'--add-options=-XX:+UnlockExperimentalVMOptions -XX:+EnableJVMCIProduct -XX:-UseJVMCICompiler -XX:-UnlockExperimentalVMOptions'
+ thread_priority_policy_option)
else:
mx.logv('[Creating JDK without -XX:+EnableJVMCIProduct]')
if thread_priority_policy_option:
jlink.append('--add-options=' +
thread_priority_policy_option.strip())
if vendor_info is not None:
for name, value in vendor_info.items():
jlink.append('--' + name + '=' + value)
release_file = join(jdk.home, 'release')
if isfile(release_file):
jlink.append('--release-info=' + release_file)
# TODO: investigate the options below used by OpenJDK to see if they should be used:
# --order-resources: specifies order of resources in generated lib/modules file.
# This is apparently not so important if a CDS archive is available.
# --generate-jli-classes: pre-generates a set of java.lang.invoke classes.
# See https://github.com/openjdk/jdk/blob/master/make/GenerateLinkOptData.gmk
mx.logv('[Creating JDK image in {}]'.format(dst_jdk_dir))
mx.run(jlink)
dst_src_zip = join(dst_jdk_dir, 'lib', 'src.zip')
mx.logv('[Creating ' + dst_src_zip + ']')
with ZipFile(dst_src_zip,
'w',
compression=ZIP_DEFLATED,
allowZip64=True) as zf:
for name, contents in sorted(dst_src_zip_contents.items()):
zf.writestr(name, contents)
mx.logv('[Copying static libraries]')
lib_directory = join(jdk.home, 'lib', 'static')
if exists(lib_directory):
dst_lib_directory = join(dst_jdk_dir, 'lib', 'static')
try:
mx.copytree(lib_directory, dst_lib_directory)
except shutil.Error as e:
# On AArch64, there can be a problem in the copystat part
# of copytree which occurs after file and directory copying
# has successfully completed. Since the metadata doesn't
# matter in this case, just ensure that the content was copied.
for root, _, lib_files in os.walk(lib_directory):
relative_root = os.path.relpath(root, dst_lib_directory)
for lib in lib_files:
src_lib_path = join(root, lib)
dst_lib_path = join(dst_lib_directory, relative_root,
lib)
if not exists(dst_lib_path):
mx.abort(
'Error copying static libraries: {} missing in {}{}Original copytree error: {}'
.format(join(relative_root, lib),
dst_lib_directory, os.linesep, e))
src_lib_hash = mx.sha1OfFile(src_lib_path)
dst_lib_hash = mx.sha1OfFile(dst_lib_path)
if src_lib_hash != dst_lib_hash:
mx.abort(
'Error copying static libraries: {} (hash={}) and {} (hash={}) differ{}Original copytree error: {}'
.format(src_lib_path, src_lib_hash,
dst_lib_path, dst_lib_hash, os.linesep,
e))
# Allow older JDK versions to work
else:
lib_prefix = mx.add_lib_prefix('')
lib_suffix = mx.add_static_lib_suffix('')
lib_directory = join(jdk.home, 'lib')
dst_lib_directory = join(dst_jdk_dir, 'lib')
for f in os.listdir(lib_directory):
if f.startswith(lib_prefix) and f.endswith(lib_suffix):
lib_path = join(lib_directory, f)
if isfile(lib_path):
shutil.copy2(lib_path, dst_lib_directory)
finally:
if not mx.get_opts().verbose:
# Preserve build directory so that javac command can be re-executed
# by cutting and pasting verbose output.
shutil.rmtree(build_dir)
# Create CDS archive (https://openjdk.java.net/jeps/341).
out = mx.OutputCapture()
mx.logv('[Creating CDS shared archive]')
if mx.run([
mx.exe_suffix(join(dst_jdk_dir, 'bin', 'java')), '-Xshare:dump',
'-Xmx128M', '-Xms128M'
],
out=out,
err=out,
nonZeroIsFatal=False) != 0:
mx.log(out.data)
mx.abort('Error generating CDS shared archive')
def findBundledLLVMProgram(llvm_program):
llvm_dist = 'SULONG_LLVM_ORG'
dep = mx.dependency(llvm_dist, fatalIfMissing=True)
return os.path.join(dep.get_output(), 'bin', llvm_program)
def do_build_makefile(mf, selectedDists):
jdk = mx.get_jdk()
bootClassPath = jdk.bootclasspath(filtered=False)
bootClassPath = bootClassPath.replace(os.path.realpath(jdk.home), "$(ABS_BOOTDIR)")
jdkBootClassPathVariableName = "JDK_BOOTCLASSPATH"
mf.add_definition("""# This Makefile is generated automatically, do not edit
TARGET=.
# Bootstrap JDK to be used (for javac and jar)
ABS_BOOTDIR=
JAVAC=$(ABS_BOOTDIR)/bin/javac -g -target """ + str(jdk.javaCompliance) + """
JAR=$(ABS_BOOTDIR)/bin/jar
HS_COMMON_SRC=.
# Directories, where the generated property-files reside within the JAR files
PROVIDERS_INF=/META-INF/jvmci.providers
SERVICES_INF=/META-INF/jvmci.services
OPTIONS_INF=/META-INF/jvmci.options
JARS = $(foreach dist,$(DISTRIBUTIONS),$($(dist)_JAR))
ifeq ($(ABS_BOOTDIR),)
$(error Variable ABS_BOOTDIR must be set to a JDK installation.)
endif
ifeq ($(MAKE_VERBOSE),)
QUIETLY=@
endif
# Required to construct a whitespace for use with subst
space :=
space +=
# Takes the provider files created by ServiceProviderProcessor (the processor
# for the @ServiceProvider annotation) and merges them into a single file.
# Arguments:
# 1: directory with contents of the JAR file
define process_providers
$(eval providers := $(1)/$(PROVIDERS_INF))
$(eval services := $(1)/$(SERVICES_INF))
$(QUIETLY) test -d $(services) || mkdir -p $(services)
$(QUIETLY) test ! -d $(providers) || (cd $(providers) && for i in $$(ls); do c=$$(cat $$i); echo $$i >> $(abspath $(services))/$$c; rm $$i; done)
@# Since all projects are built together with one javac call we cannot determine
@# which project contains HotSpotVMConfig.inline.hpp so we hardcode it.
$(eval vmconfig := $(1)/hotspot/HotSpotVMConfig.inline.hpp)
$(eval vmconfigDest := $(HS_COMMON_SRC)/../mxbuild/jvmci/jdk.vm.ci.hotspot/src_gen/hotspot)
$(QUIETLY) test ! -f $(vmconfig) || (mkdir -p $(vmconfigDest) && cp $(vmconfig) $(vmconfigDest))
endef
# Finds the *_OptionsDescriptors classes created by OptionProcessor (the processor for the @Option annotation)
# and appends their names to services/jdk.vm.ci.options.OptionDescriptors.
# Arguments:
# 1: directory with contents of the JAR file
define process_options
$(eval services := $(1)/META-INF/services)
$(QUIETLY) test -d $(services) || mkdir -p $(services)
$(eval optionDescriptors := $(1)/META-INF/services/jdk.vm.ci.options.OptionDescriptors)
$(QUIETLY) cd $(1) && for i in $$(find . -name '*_OptionDescriptors.class' 2>/dev/null); do echo $${i} | sed 's:\\./\\(.*\\)\\.class:\\1:g' | tr '/' '.' >> $(abspath $(optionDescriptors)); done
endef
# Extracts META-INF/jvmci.services from a JAR file into a given directory
# Arguments:
# 1: JAR file to extract
# 2: target directory (which already exists)
define extract
$(eval TMP := $(shell mktemp -d $(TARGET)/tmp_XXXXX))
$(QUIETLY) cd $(TMP) && $(JAR) xf $(abspath $(1)) && \\
(test ! -d .$(SERVICES_INF) || cp -r .$(SERVICES_INF) $(abspath $(2)));
$(QUIETLY) rm -r $(TMP);
$(QUIETLY) cp $(1) $(2)
endef
# Calls $(JAVAC) with the boot class path $(JDK_BOOTCLASSPATH) and sources taken from the automatic variable $^
# Arguments:
# 1: processorpath
# 2: classpath
# 3: resources to copy
# 4: target JAR file
define build_and_jar
$(info Building $(4))
$(eval TMP := $(shell mkdir -p $(TARGET) && mktemp -d $(TARGET)/tmp_XXXXX))
$(QUIETLY) $(JAVAC) -d $(TMP) -processorpath :$(1) -bootclasspath $(JDK_BOOTCLASSPATH) -cp :$(2) $(filter %.java,$^)
$(QUIETLY) test "$(3)" = "" || cp -r $(3) $(TMP)
$(QUIETLY) $(call process_options,$(TMP))
$(QUIETLY) $(call process_providers,$(TMP))
$(QUIETLY) mkdir -p $(shell dirname $(4))
$(QUIETLY) $(JAR) -0cf $(4) -C $(TMP) .
$(QUIETLY) rm -r $(TMP)
endef
# Verifies that make/defs.make contains an appropriate line for each JVMCI service
# and that only existing JVMCI services are exported.
# Arguments:
# 1: list of service files
# 2: variable name for directory of service files
define verify_defs_make
$(eval defs := make/defs.make)
$(eval uncondPattern := EXPORT_LIST += $$$$($(2))/)
$(eval condPattern := CONDITIONAL_EXPORT_LIST += $$$$($(2))/)
$(eval unconditionalExports := $(shell grep '^EXPORT_LIST += $$($2)' make/defs.make | sed 's:.*($(2))/::g'))
$(eval conditionalExports := $(shell grep '^CONDITIONAL_EXPORT_LIST += $$($2)' make/defs.make | sed 's:.*($(2))/::g'))
$(eval allExports := $(unconditionalExports) $(conditionalExports))
$(foreach file,$(1),$(if $(findstring $(file),$(allExports)), ,$(error "Line matching '$(uncondPattern)$(file)' or '$(condPattern)$(file)' not found in $(defs)")))
$(foreach export,$(unconditionalExports),$(if $(findstring $(export),$(1)), ,$(error "The line '$(uncondPattern)$(export)' should not be in $(defs)")))
endef
all: default
\t$(info Put $(EXPORTED_FILES) into SHARED_DIR $(SHARED_DIR))
\t$(shell mkdir -p $(SHARED_DIR))
\t$(foreach export,$(EXPORTED_FILES),$(call extract,$(export),$(SHARED_DIR)))
export: all
\t$(call verify_defs_make,$(notdir $(wildcard $(SHARED_DIR)/jvmci.services/*)),EXPORT_JRE_LIB_JVMCI_SERVICES_DIR)
.PHONY: export
clean:
\t$(QUIETLY) rm $(JARS) 2> /dev/null || true
\t$(QUIETLY) rmdir -p $(dir $(JARS)) 2> /dev/null || true
.PHONY: export clean
""")
assert selectedDists
selectedDists = [mx.dependency(s) for s in selectedDists]
dists = []
def _visit(dep, edge):
if dep.isDistribution():
dists.append(dep)
mx.walk_deps(roots=selectedDists, visit=_visit, ignoredEdges=[mx.DEP_EXCLUDED])
mf.add_definition(jdkBootClassPathVariableName + " = " + bootClassPath)
for dist in dists: make_dist_rule(dist, mf)
mf.add_definition("DISTRIBUTIONS = " + ' '.join([dist.name for dist in dists]))
mf.add_rule("default: $({}_JAR)\n.PHONY: default\n".format("_JAR) $(".join([d.name for d in selectedDists])))
return True
