def _test_libgraal_basic(extra_vm_arguments): """ Tests basic libgraal execution by running a DaCapo benchmark, ensuring it has a 0 exit code and that the output for -DgraalShowConfiguration=info describes a libgraal execution. """ expect = r"Using compiler configuration '[^']+' provided by [\.\w]+ loaded from[ \w]* JVMCI native library" compiler_log_file = abspath('graal-compiler.log') args = [ '-Dgraal.ShowConfiguration=info', '-Dgraal.LogFile=' + compiler_log_file, '-jar', mx.library('DACAPO').get_path(True), 'avrora', '-n', '1' ] # Verify execution via raw java launcher in `mx graalvm-home`. try: mx.run([join(mx_sdk_vm_impl.graalvm_home(), 'bin', 'java')] + args) finally: _check_compiler_log(compiler_log_file, expect) # Verify execution via `mx vm`. import mx_compiler try: mx_compiler.run_vm(extra_vm_arguments + args) finally: _check_compiler_log(compiler_log_file, expect)
def _brainfck_launcher_command(args): """Brainf*ck launcher embedded in GraalVM + arguments""" import mx_sdk_vm_impl return [ os.path.join(mx_sdk_vm_impl.graalvm_home(fatalIfMissing=True), 'bin', mx.cmd_suffix('bf')) ] + args
def lli(args=None, out=None): """run lli via the current GraalVM""" debug_args = mx.java_debug_args() if debug_args and not mx.is_debug_disabled(): args = [_java_to_graalvm_arg(d) for d in debug_args] + args # on Windows <GRAALVM_HOME>/bin/lli is always a .cmd file because it is a "fake symlink" mx.run([os.path.join(mx_sdk_vm_impl.graalvm_home(fatalIfMissing=True), 'bin', mx_subst.path_substitutions.substitute('<cmd:lli>'))] + args, out=out)
def _prepare_svm_env(): import mx_vm if hasattr(mx_vm, 'graalvm_home'): graalvm_home = mx_vm.graalvm_home() else: import mx_sdk_vm_impl graalvm_home = mx_sdk_vm_impl.graalvm_home() libgraal_nodejs_filename = mx.add_lib_suffix( mx.add_lib_prefix('graal-nodejs')) candidates = [ join(graalvm_home, directory, libgraal_nodejs_filename) for directory in [ join('jre', 'languages', 'nodejs', 'lib'), join('languages', 'nodejs', 'lib') ] ] libgraal_nodejs = None for candidate in candidates: if exists(candidate): libgraal_nodejs = candidate if libgraal_nodejs is None: mx.abort( "Cannot find graal-nodejs library in '{}'.\nDid you forget to build it (e.g., using 'mx --env svm build')?" .format(candidates)) _setEnvVar('NODE_JVM_LIB', libgraal_nodejs) _setEnvVar('ICU4J_DATA_PATH', join(mx.suite('graal-js').dir, 'lib', 'icu4j', 'icudt'))
def run_stage_agent(self, config, stages): profile_path = config.profile_path_no_extension + '-agent' + config.profile_file_extension hotspot_vm_args = [ '-ea', '-esa' ] if self.is_gate and not config.skip_agent_assertions else [] hotspot_run_args = [] hotspot_vm_args += [ '-agentlib:native-image-agent=config-output-dir=' + str(config.config_dir), '-XX:-UseJVMCINativeLibrary' ] if self.hotspot_pgo: hotspot_vm_args += ['-Dgraal.PGOInstrument=' + profile_path] if self.hotspot_pgo and not self.is_gate and config.extra_agent_profile_run_args: hotspot_run_args += config.extra_agent_profile_run_args elif config.extra_agent_run_args: hotspot_run_args += config.extra_agent_run_args else: hotspot_run_args += config.image_run_args hotspot_args = hotspot_vm_args + config.classpath_arguments + config.executable + config.system_properties + hotspot_run_args java_command = os.path.join( mx_sdk_vm_impl.graalvm_home(fatalIfMissing=True), 'bin', 'java') with stages.set_command([java_command] + hotspot_args) as s: s.execute_command() if self.hotspot_pgo and s.exit_code == 0: mx.copyfile(profile_path, config.latest_profile_path)
def _espresso_command(launcher, args): import mx_sdk_vm_impl bin_dir = join(mx_sdk_vm_impl.graalvm_home(fatalIfMissing=True), 'bin') exe = join(bin_dir, mx.exe_suffix(launcher)) if not os.path.exists(exe): exe = join(bin_dir, mx.cmd_suffix(launcher)) return [exe] + args
def _espresso_java_command(args): """Java launcher using libespresso in GraalVM + arguments""" import mx_sdk_vm_impl bin_dir = os.path.join(mx_sdk_vm_impl.graalvm_home(fatalIfMissing=True), 'bin') exe = os.path.join(bin_dir, mx.exe_suffix('java')) if not os.path.exists(exe): exe = os.path.join(bin_dir, mx.cmd_suffix('java')) return [exe, '-truffle'] + args
def _espresso_launcher_command(args): """Espresso launcher embedded in GraalVM + arguments""" import mx_sdk_vm_impl bin_dir = os.path.join(mx_sdk_vm_impl.graalvm_home(fatalIfMissing=True), 'bin') exe = os.path.join(bin_dir, mx.exe_suffix('espresso')) if not os.path.exists(exe): exe = os.path.join(bin_dir, mx.cmd_suffix('espresso')) return [exe] + args
def run_launcher(self, cmd, args, cwd): """Run the 'cmd' command in the 'bin' directory.""" args = self.post_process_launcher_command_line_args(args) self.extract_vm_info(args) mx.log("Running '{}' on '{}' with args: '{}'".format(cmd, self.name(), " ".join(args))) out = mx.TeeOutputCapture(mx.OutputCapture()) code = mx.run([os.path.join(mx_sdk_vm_impl.graalvm_home(fatalIfMissing=True), 'bin', cmd)] + args, out=out, err=out, cwd=cwd, nonZeroIsFatal=False) out = out.underlying.data dims = self.dimensions(cwd, args, code, out) return code, out, dims
def lli(args=None, out=None): """run lli via the current GraalVM""" debug_args = mx.java_debug_args() if debug_args and not mx.is_debug_disabled(): args = [_java_to_graalvm_arg(d) for d in debug_args] + args mx.run([ os.path.join(mx_sdk_vm_impl.graalvm_home(fatalIfMissing=True), 'bin', 'lli') ] + args, out=out)
def run_java(self, args, out=None, err=None, cwd=None, nonZeroIsFatal=False): """Run 'java' workloads.""" self.extract_vm_info(args) return mx.run([ os.path.join(mx_sdk_vm_impl.graalvm_home(fatalIfMissing=True), 'bin', 'java') ] + args, out=out, err=err, cwd=cwd, nonZeroIsFatal=nonZeroIsFatal)
def __init__(self, vm, bm_suite, args): self.bmSuite = bm_suite self.benchmark_suite_name = bm_suite.benchSuiteName(args) if len(inspect.getargspec(bm_suite.benchSuiteName).args) > 1 else bm_suite.benchSuiteName() # pylint: disable=deprecated-method self.benchmark_name = bm_suite.benchmarkName() self.extra_image_build_arguments = bm_suite.extra_image_build_argument(self.benchmark_name, args) self.extra_run_args = bm_suite.extra_run_arg(self.benchmark_name, args) self.extra_agent_run_args = bm_suite.extra_agent_run_arg(self.benchmark_name, args) self.extra_profile_run_args = bm_suite.extra_profile_run_arg(self.benchmark_name, args) self.extra_agent_profile_run_args = bm_suite.extra_agent_profile_run_arg(self.benchmark_name, args) self.benchmark_output_dir = bm_suite.benchmark_output_dir(self.benchmark_name, args) self.pgo_iteration_num = None self.params = ['extra-image-build-argument', 'extra-run-arg', 'extra-agent-run-arg', 'extra-profile-run-arg', 'extra-agent-profile-run-arg', 'benchmark-output-dir', 'stages', 'skip-agent-assertions'] self.profile_file_extension = '.iprof' self.stages = bm_suite.stages(args) self.last_stage = self.stages[-1] self.skip_agent_assertions = bm_suite.skip_agent_assertions(self.benchmark_name, args) self.root_dir = self.benchmark_output_dir if self.benchmark_output_dir else mx.join(mx.suite('vm').dir, 'mxbuild') self.executable_suffix = ('-' + self.benchmark_name) if self.benchmark_name else '' self.executable, self.classpath_arguments, self.system_properties, self.image_run_args = NativeImageVM.extract_benchmark_arguments(args) self.executable_name = (os.path.splitext(os.path.basename(self.executable[1]))[0] + self.executable_suffix if self.executable[0] == '-jar' else self.executable[0] + self.executable_suffix).lower() self.final_image_name = self.executable_name + '-' + vm.config_name() self.output_dir = mx.join(os.path.abspath(self.root_dir), 'native-image-bench-' + self.executable_name + '-' + vm.config_name()) self.profile_path_no_extension = os.path.join(self.output_dir, self.executable_name) self.latest_profile_path = self.profile_path_no_extension + '-latest' + self.profile_file_extension self.config_dir = os.path.join(self.output_dir, 'config') self.log_dir = self.output_dir self.base_image_build_args = [os.path.join(mx_sdk_vm_impl.graalvm_home(fatalIfMissing=True), 'bin', 'native-image')] self.base_image_build_args += ['--no-fallback', '--no-server', '-g', '--allow-incomplete-classpath'] self.base_image_build_args += ['-J-ea', '-J-esa', '-H:+VerifyGraalGraphs', '-H:+VerifyPhases'] if vm.is_gate else [] self.base_image_build_args += self.system_properties self.base_image_build_args += self.classpath_arguments self.base_image_build_args += self.executable self.base_image_build_args += ['-H:Path=' + self.output_dir] self.base_image_build_args += ['-H:ConfigurationFileDirectories=' + self.config_dir] if vm.is_llvm: self.base_image_build_args += ['-H:CompilerBackend=llvm', '-H:Features=org.graalvm.home.HomeFinderFeature', '-H:DeadlockWatchdogInterval=0'] self.base_image_build_args += self.extra_image_build_arguments
def gate_body(args, tasks): # all mx_sdk_vm_impl gate tasks can also be run as vm gate tasks mx_sdk_vm_impl.gate_body(args, tasks) with Task('Vm: Basic GraalVM Tests', tasks, tags=[VmGateTasks.compiler]) as t: if t and mx_sdk_vm_impl.has_component('GraalVM compiler'): # 1. the build must be a GraalVM # 2. the build must be JVMCI-enabled since the 'GraalVM compiler' component is registered mx_sdk_vm_impl.check_versions( mx_sdk_vm_impl.graalvm_output(), graalvm_version_regex=mx_sdk_vm_impl.graalvm_version_regex, expect_graalvm=True, check_jvmci=True) if mx_sdk_vm_impl.has_component('LibGraal'): libgraal_location = mx_sdk_vm_impl.get_native_image_locations( 'LibGraal', 'jvmcicompiler') if libgraal_location is None: mx.warn( "Skipping libgraal tests: no library enabled in the LibGraal component" ) else: extra_vm_arguments = [ '-XX:+UseJVMCICompiler', '-XX:+UseJVMCINativeLibrary', '-XX:JVMCILibPath=' + dirname(libgraal_location) ] if args.extra_vm_argument: extra_vm_arguments += args.extra_vm_argument import mx_compiler # run avrora on the GraalVM binary itself with Task('LibGraal Compiler:GraalVM DaCapo-avrora', tasks, tags=[VmGateTasks.libgraal]) as t: if t: mx.run([ join(mx_sdk_vm_impl.graalvm_home(), 'bin', 'java'), '-XX:+UseJVMCICompiler', '-XX:+UseJVMCINativeLibrary', '-jar', mx.library('DACAPO').get_path(True), 'avrora' ]) with Task('LibGraal Compiler:CTW', tasks, tags=[VmGateTasks.libgraal]) as t: if t: mx_compiler.ctw([ '-DCompileTheWorld.Config=Inline=false CompilationFailureAction=ExitVM', '-esa', '-XX:+EnableJVMCI', '-DCompileTheWorld.MultiThreaded=true', '-Dgraal.InlineDuringParsing=false', '-Dgraal.TrackNodeSourcePosition=true', '-DCompileTheWorld.Verbose=false', '-XX:ReservedCodeCacheSize=300m', ], extra_vm_arguments) mx_compiler.compiler_gate_benchmark_runner( tasks, extra_vm_arguments, prefix='LibGraal Compiler:') with Task('LibGraal Truffle:unittest', tasks, tags=[VmGateTasks.libgraal]) as t: if t: def _unittest_config_participant(config): vmArgs, mainClass, mainClassArgs = config def is_truffle_fallback(arg): fallback_args = [ "-Dtruffle.TruffleRuntime=com.oracle.truffle.api.impl.DefaultTruffleRuntime", "-Dgraalvm.ForcePolyglotInvalid=true" ] return arg in fallback_args newVmArgs = [ arg for arg in vmArgs if not is_truffle_fallback(arg) ] return (newVmArgs, mainClass, mainClassArgs) mx_unittest.add_config_participant( _unittest_config_participant) excluded_tests = environ.get("TEST_LIBGRAAL_EXCLUDE") if excluded_tests: with NamedTemporaryFile(prefix='blacklist.', mode='w', delete=False) as fp: fp.file.writelines( [l + '\n' for l in excluded_tests.split()]) unittest_args = ["--blacklist", fp.name] else: unittest_args = [] unittest_args = unittest_args + [ "--enable-timing", "--verbose" ] compiler_log_file = "graal-compiler.log" mx_unittest.unittest(unittest_args + extra_vm_arguments + [ "-Dgraal.TruffleCompileImmediately=true", "-Dgraal.TruffleBackgroundCompilation=false", "-Dgraal.TraceTruffleCompilation=true", "-Dgraalvm.locatorDisabled=true", "-Dgraal.PrintCompilation=true", "-Dgraal.LogFile={0}". format(compiler_log_file), "truffle" ]) if exists(compiler_log_file): remove(compiler_log_file) else: mx.warn("Skipping libgraal tests: component not enabled") gate_substratevm(tasks) gate_sulong(tasks) gate_python(tasks) gate_svm_sl_tck(tasks) gate_svm_truffle_tck_js(tasks)
def run_java(self, args, out=None, err=None, cwd=None, nonZeroIsFatal=False): if '-version' in args: return super(NativeImageVM, self).run_java(args, out=out, err=err, cwd=cwd, nonZeroIsFatal=nonZeroIsFatal) else: image_cwd = os.path.abspath(cwd if cwd else os.getcwd()) non_zero_is_fatal = self.is_gate or nonZeroIsFatal config = NativeImageVM.BenchmarkConfig() original_java_run_args = config.parse(args) executable, classpath_arguments, system_properties, image_run_args = NativeImageVM.extract_benchmark_arguments( original_java_run_args) executable_suffix = ( '-' + config.benchmark_name) if config.benchmark_name else '' executable_name = ( os.path.splitext(os.path.basename(executable[1]))[0] + executable_suffix if executable[0] == '-jar' else executable[0] + executable_suffix).lower() non_tmp_dir = os.path.abspath( config.benchmark_output_dir ) if config.benchmark_output_dir else None config.output_dir = mx.mkdtemp(suffix='bench-' + executable_name, prefix='native-image', dir=non_tmp_dir) config.profile_dir = config.output_dir config.log_dir = config.output_dir profile_path = os.path.join(config.profile_dir, executable_name + '.iprof') image_path = os.path.join(config.output_dir, executable_name) # Agent configuration and/or HotSpot profiling needs_config = (config.config_dir is None) and config.needs_config if needs_config or self.hotspot_pgo: hotspot_vm_args = ['-ea', '-esa'] if self.is_gate else [] hotspot_run_args = [] if needs_config: config.config_dir = os.path.join(config.output_dir, 'config') hotspot_vm_args += [ '-agentlib:native-image-agent=config-output-dir=' + str(config.config_dir), '-XX:-UseJVMCINativeLibrary' ] if self.hotspot_pgo: hotspot_vm_args += [ '-Dgraal.PGOInstrument=' + profile_path ] if config.extra_agent_run_args: hotspot_run_args += config.extra_profile_run_args if self.hotspot_pgo and not self.is_gate else config.extra_agent_run_args else: hotspot_run_args += image_run_args hotspot_args = hotspot_vm_args + classpath_arguments + executable + system_properties + hotspot_run_args hs_stdout_path = os.path.abspath( os.path.join(config.log_dir, executable_name + '-hot-spot-stdout.log')) hs_stderr_path = os.path.abspath( os.path.join(config.log_dir, executable_name + '-hot-spot-stderr.log')) with open(hs_stdout_path, 'a') as hs_stdout, open(hs_stderr_path, 'a') as hs_stderr: mx.log( 'Running with HotSpot to get the configuration files and profiles. This could take a while...' ) mx.log('Command: ' + ' '.join(['java'] + hotspot_args)) mx.log('The standard output saved to ' + hs_stdout_path) mx.log('The standard error saved to ' + hs_stderr_path) exit_code = super(NativeImageVM, self).run_java( hotspot_args, out=hs_stdout.write, err=hs_stderr.write, cwd=image_cwd, nonZeroIsFatal=non_zero_is_fatal) mx.log("Hotspot run finished with exit code " + str(exit_code) + ".") with open( os.path.join(config.config_dir, 'reflect-config.json'), 'r') as reflect_config: mx.log("The content of reflect-config.json is: ") mx.log(reflect_config.read()) base_image_build_args = [ os.path.join(mx_sdk_vm_impl.graalvm_home(fatalIfMissing=True), 'bin', 'native-image') ] base_image_build_args += ['--no-fallback'] base_image_build_args += [ '--no-server', '-J-ea', '-J-esa', '-H:+VerifyGraalGraphs', '-H:+VerifyPhases', '-H:+TraceClassInitialization' ] if self.is_gate else [] base_image_build_args += system_properties base_image_build_args += classpath_arguments base_image_build_args += executable base_image_build_args += [ '-H:Name=' + executable_name, '-H:Path=' + config.output_dir ] if needs_config: base_image_build_args += [ '-H:ConfigurationFileDirectories=' + config.config_dir ] if self.is_llvm: base_image_build_args += ['-H:CompilerBackend=llvm'] base_image_build_args += config.extra_image_build_arguments # PGO instrumentation i = 0 while i < self.pgo_instrumented_iterations: instrument_args = ['--pgo-instrument'] + ( [] if i == 0 and not self.hotspot_pgo else ['--pgo']) instrument_image_build_args = base_image_build_args + instrument_args mx.log('Building the instrumentation image with: ') mx.log(' ' + ' '.join([ pipes.quote(str(arg)) for arg in instrument_image_build_args ])) mx.run(instrument_image_build_args, out=None, err=None, cwd=image_cwd, nonZeroIsFatal=non_zero_is_fatal) image_run_cmd = [image_path] image_run_cmd += ['-XX:ProfilesDumpFile=' + profile_path] if config.extra_profile_run_args: image_run_cmd += config.extra_profile_run_args else: image_run_cmd += image_run_args + config.extra_run_args mx.log('Running the instrumented image with: ') mx.log( ' ' + ' '.join([pipes.quote(str(arg)) for arg in image_run_cmd])) inst_stdout_path = os.path.abspath( os.path.join( config.log_dir, executable_name + '-instrument-' + str(i) + '-stdout.log')) inst_stderr_path = os.path.abspath( os.path.join( config.log_dir, executable_name + '-instrument-' + str(i) + '-stderr.log')) with open(inst_stdout_path, 'a') as inst_stdout, open(inst_stderr_path, 'a') as inst_stderr: mx.log('The standard output saved to ' + inst_stdout_path) mx.log('The standard error saved to ' + inst_stderr_path) mx.run(image_run_cmd, out=inst_stdout.write, err=inst_stderr.write, cwd=image_cwd, nonZeroIsFatal=non_zero_is_fatal) i += 1 # Build the final image pgo_verification_output_path = os.path.join( config.output_dir, executable_name + '-probabilities.log') pgo_args = ['--pgo=' + profile_path, '-H:+VerifyPGOProfiles', '-H:VerificationDumpFile=' + pgo_verification_output_path] if self.pgo_instrumented_iterations > 0 or self.hotspot_pgo else [] final_image_args = base_image_build_args + pgo_args mx.log('Building the final image with: ') mx.log( ' ' + ' '.join([pipes.quote(str(arg)) for arg in final_image_args])) mx.run(final_image_args, out=None, err=None, cwd=image_cwd, nonZeroIsFatal=non_zero_is_fatal) # Execute the benchmark image_run_cmd = [image_path ] + image_run_args + config.extra_run_args mx.log('Running the produced native executable with: ') mx.log(' ' + ' '.join([pipes.quote(str(arg)) for arg in image_run_cmd])) mx.run(image_run_cmd, out=out, err=err, cwd=image_cwd, nonZeroIsFatal=non_zero_is_fatal)
def gate_body(args, tasks): # all mx_sdk_vm_impl gate tasks can also be run as vm gate tasks if not args.all_suites: mx_sdk_vm_impl.gate_body(args, tasks) with Task('Vm: Basic GraalVM Tests', tasks, tags=[VmGateTasks.compiler]) as t: if t and mx_sdk_vm_impl.has_component('GraalVM compiler'): # 1. the build must be a GraalVM # 2. the build must be JVMCI-enabled since the 'GraalVM compiler' component is registered mx_sdk_vm_impl.check_versions( mx_sdk_vm_impl.graalvm_output(), graalvm_version_regex=mx_sdk_vm_impl.graalvm_version_regex, expect_graalvm=True, check_jvmci=True) if mx_sdk_vm_impl.has_component('LibGraal'): libgraal_location = mx_sdk_vm_impl.get_native_image_locations( 'LibGraal', 'jvmcicompiler') if libgraal_location is None: mx.warn( "Skipping libgraal tests: no library enabled in the LibGraal component" ) else: extra_vm_arguments = [ '-XX:+UseJVMCICompiler', '-XX:+UseJVMCINativeLibrary', '-XX:JVMCILibPath=' + dirname(libgraal_location) ] if args.extra_vm_argument: extra_vm_arguments += args.extra_vm_argument import mx_compiler # run avrora on the GraalVM binary itself with Task('LibGraal Compiler:GraalVM DaCapo-avrora', tasks, tags=[VmGateTasks.libgraal]) as t: if t: java_exe = join(mx_sdk_vm_impl.graalvm_home(), 'bin', 'java') mx.run([ java_exe, '-XX:+UseJVMCICompiler', '-XX:+UseJVMCINativeLibrary', '-jar', mx.library('DACAPO').get_path(True), 'avrora' ]) # Ensure that fatal errors in libgraal route back to HotSpot testdir = mkdtemp() try: cmd = [ java_exe, '-XX:+UseJVMCICompiler', '-XX:+UseJVMCINativeLibrary', '-Dlibgraal.CrashAt=length,hashCode', '-Dlibgraal.CrashAtIsFatal=true', '-jar', mx.library('DACAPO').get_path(True), 'avrora' ] out = mx.OutputCapture() exitcode = mx.run(cmd, cwd=testdir, nonZeroIsFatal=False, out=out) if exitcode == 0: if 'CrashAtIsFatal: no fatalError function pointer installed' in out.data: # Executing a VM that does not configure fatal errors handling # in libgraal to route back through the VM. pass else: mx.abort( 'Expected following command to result in non-zero exit code: ' + ' '.join(cmd)) else: hs_err = None testdir_entries = listdir(testdir) for name in testdir_entries: if name.startswith('hs_err_pid' ) and name.endswith('.log'): hs_err = join(testdir, name) if hs_err is None: mx.abort( 'Expected a file starting with "hs_err_pid" in test directory. Entries found=' + str(testdir_entries)) with open(join(testdir, hs_err)) as fp: contents = fp.read() if 'Fatal error in JVMCI' not in contents: mx.abort( 'Expected "Fatal error in JVMCI" to be in contents of ' + hs_err + ':' + linesep + contents) finally: mx.rmtree(testdir) with Task('LibGraal Compiler:CTW', tasks, tags=[VmGateTasks.libgraal]) as t: if t: mx_compiler.ctw([ '-DCompileTheWorld.Config=Inline=false CompilationFailureAction=ExitVM', '-esa', '-XX:+EnableJVMCI', '-DCompileTheWorld.MultiThreaded=true', '-Dgraal.InlineDuringParsing=false', '-Dgraal.TrackNodeSourcePosition=true', '-DCompileTheWorld.Verbose=false', '-DCompileTheWorld.HugeMethodLimit=4000', '-DCompileTheWorld.MaxCompiles=150000', '-XX:ReservedCodeCacheSize=300m', ], extra_vm_arguments) mx_compiler.compiler_gate_benchmark_runner( tasks, extra_vm_arguments, prefix='LibGraal Compiler:') with Task('LibGraal Truffle:unittest', tasks, tags=[VmGateTasks.libgraal]) as t: if t: def _unittest_config_participant(config): vmArgs, mainClass, mainClassArgs = config def is_truffle_fallback(arg): fallback_args = [ "-Dtruffle.TruffleRuntime=com.oracle.truffle.api.impl.DefaultTruffleRuntime", "-Dgraalvm.ForcePolyglotInvalid=true" ] return arg in fallback_args newVmArgs = [ arg for arg in vmArgs if not is_truffle_fallback(arg) ] return (newVmArgs, mainClass, mainClassArgs) mx_unittest.add_config_participant( _unittest_config_participant) excluded_tests = environ.get("TEST_LIBGRAAL_EXCLUDE") if excluded_tests: with NamedTemporaryFile(prefix='blacklist.', mode='w', delete=False) as fp: fp.file.writelines( [l + '\n' for l in excluded_tests.split()]) unittest_args = ["--blacklist", fp.name] else: unittest_args = [] unittest_args = unittest_args + [ "--enable-timing", "--verbose" ] compiler_log_file = "graal-compiler.log" mx_unittest.unittest(unittest_args + extra_vm_arguments + [ "-Dpolyglot.engine.AllowExperimentalOptions=true", "-Dpolyglot.engine.CompileImmediately=true", "-Dpolyglot.engine.BackgroundCompilation=false", "-Dpolyglot.engine.TraceCompilation=true", "-Dpolyglot.log.file={0}".format(compiler_log_file), "-Dgraalvm.locatorDisabled=true", "truffle" ]) if exists(compiler_log_file): remove(compiler_log_file) else: mx.warn("Skipping libgraal tests: component not enabled") gate_substratevm(tasks) gate_sulong(tasks) gate_python(tasks) gate_svm_sl_tck(tasks) gate_svm_truffle_tck_js(tasks)
def run_java(self, args, out=None, err=None, cwd=None, nonZeroIsFatal=False): if '-version' in args: return super(NativeImageVM, self).run_java(args, out=out, err=err, cwd=cwd, nonZeroIsFatal=nonZeroIsFatal) else: image_cwd = os.path.abspath(cwd if cwd else os.getcwd()) non_zero_is_fatal = self.is_gate or nonZeroIsFatal config = NativeImageVM.BenchmarkConfig() original_java_run_args = config.parse(args) executable, classpath_arguments, system_properties, image_run_args = NativeImageVM.extract_benchmark_arguments(original_java_run_args) executable_suffix = ('-' + config.benchmark_name) if config.benchmark_name else '' executable_name = (os.path.splitext(os.path.basename(executable[1]))[0] + executable_suffix if executable[0] == '-jar' else executable[0] + executable_suffix).lower() final_image_name = executable_name + '-' + self.config_name() non_tmp_dir = os.path.abspath(config.benchmark_output_dir) if config.benchmark_output_dir else None if config.only_prepare_native_image or config.only_run_prepared_image: bench_suite = mx.suite('vm-enterprise') root_dir = mx.join(bench_suite.dir, "mxbuild") output_dir_path = mx.join(os.path.abspath(root_dir), 'native-image-bench-' + executable_name + '-' + self.config_name()) if config.only_prepare_native_image: if os.path.exists(output_dir_path): os.rmdir(output_dir_path) os.mkdir(output_dir_path) config.output_dir = output_dir_path else: config.output_dir = mx.mkdtemp(suffix='bench-' + executable_name, prefix='native-image', dir=non_tmp_dir) if not config.only_run_prepared_image: config.profile_dir = config.output_dir config.log_dir = config.output_dir profile_path_no_extension = os.path.join(config.profile_dir, executable_name) extension = '.iprof' profile_path = profile_path_no_extension + extension # Agent configuration and/or HotSpot profiling needs_config = (config.config_dir is None) and config.needs_config if needs_config or self.hotspot_pgo: hotspot_vm_args = ['-ea', '-esa'] if self.is_gate else [] hotspot_run_args = [] if needs_config: config.config_dir = os.path.join(config.output_dir, 'config') hotspot_vm_args += ['-agentlib:native-image-agent=config-output-dir=' + str(config.config_dir), '-XX:-UseJVMCINativeLibrary'] if self.hotspot_pgo: hotspot_vm_args += ['-Dgraal.PGOInstrument=' + profile_path] if self.hotspot_pgo and not self.is_gate and config.extra_agent_profile_run_args: hotspot_run_args += config.extra_agent_profile_run_args elif config.extra_agent_run_args: hotspot_run_args += config.extra_agent_run_args else: hotspot_run_args += image_run_args hotspot_args = hotspot_vm_args + classpath_arguments + executable + system_properties + hotspot_run_args hs_stdout_path = os.path.abspath(os.path.join(config.log_dir, executable_name + '-hot-spot-stdout.log')) hs_stderr_path = os.path.abspath(os.path.join(config.log_dir, executable_name + '-hot-spot-stderr.log')) with open(hs_stdout_path, 'a') as hs_stdout, open(hs_stderr_path, 'a') as hs_stderr: mx.log('Running with HotSpot to get the configuration files and profiles. This could take a while...') mx.log('Command: ' + ' '.join(['java'] + hotspot_args)) mx.log('The standard output saved to ' + hs_stdout_path) mx.log('The standard error saved to ' + hs_stderr_path) exit_code = super(NativeImageVM, self).run_java( hotspot_args, out=hs_stdout.write, err=hs_stderr.write, cwd=image_cwd, nonZeroIsFatal=non_zero_is_fatal) mx.log("Hotspot run finished with exit code " + str(exit_code) + ".") with open(os.path.join(config.config_dir, 'reflect-config.json'), 'r') as reflect_config: mx.log("The content of reflect-config.json is: ") mx.log(reflect_config.read()) base_image_build_args = [os.path.join(mx_sdk_vm_impl.graalvm_home(fatalIfMissing=True), 'bin', 'native-image')] base_image_build_args += ['--no-fallback', '--no-server'] base_image_build_args += ['-J-ea', '-J-esa', '-H:+VerifyGraalGraphs', '-H:+VerifyPhases', '-H:+TraceClassInitialization'] if self.is_gate else [] base_image_build_args += system_properties base_image_build_args += classpath_arguments base_image_build_args += executable base_image_build_args += ['-H:Path=' + config.output_dir] if needs_config: base_image_build_args += ['-H:ConfigurationFileDirectories=' + config.config_dir] if self.is_llvm: base_image_build_args += ['-H:CompilerBackend=llvm', '-H:Features=org.graalvm.home.HomeFinderFeature', '-H:DeadlockWatchdogInterval=0'] base_image_build_args += config.extra_image_build_arguments # PGO instrumentation i = 0 instrumented_iterations = self.pgo_instrumented_iterations if config.pgo_iteration_num is None else int(config.pgo_iteration_num) if not self.hotspot_pgo: while i < instrumented_iterations: instrumentation_image_name = executable_name + '-instrument' + (str(i) if i > 0 else '') executable_name_args = ['-H:Name=' + instrumentation_image_name] image_path = os.path.join(config.output_dir, instrumentation_image_name) pgo_verification_output_path = os.path.join(config.output_dir, instrumentation_image_name + '-probabilities.log') pgo_args = ['--pgo=' + profile_path, '-H:+VerifyPGOProfiles', '-H:VerificationDumpFile=' + pgo_verification_output_path] instrument_args = ['--pgo-instrument'] + ([] if i == 0 else pgo_args) instrument_args += ['-H:+InlineAllExplored'] if self.pgo_inline_explored else [] instrument_args += ['-H:' + ('+' if self.pgo_context_sensitive else '-') + 'EnablePGOContextSensitivity'] instrument_image_build_args = base_image_build_args + executable_name_args + instrument_args mx.log('Building the instrumentation image with: ') mx.log(' ' + ' '.join([pipes.quote(str(arg)) for arg in instrument_image_build_args])) mx.run(instrument_image_build_args, out=None, err=None, cwd=image_cwd, nonZeroIsFatal=non_zero_is_fatal) image_run_cmd = [image_path] profile_path = profile_path_no_extension + (str(i) + extension if i > 0 else extension) image_run_cmd += ['-XX:ProfilesDumpFile=' + profile_path] if config.extra_profile_run_args: image_run_cmd += config.extra_profile_run_args else: image_run_cmd += image_run_args + config.extra_run_args mx.log('Running the instrumented image with: ') mx.log(' ' + ' '.join([pipes.quote(str(arg)) for arg in image_run_cmd])) inst_stdout_path = os.path.abspath(os.path.join(config.log_dir, executable_name + '-instrument-' + str(i) + '-stdout.log')) inst_stderr_path = os.path.abspath(os.path.join(config.log_dir, executable_name + '-instrument-' + str(i) + '-stderr.log')) with open(inst_stdout_path, 'a') as inst_stdout, open(inst_stderr_path, 'a') as inst_stderr: mx.log('The standard output saved to ' + inst_stdout_path) mx.log('The standard error saved to ' + inst_stderr_path) mx.run(image_run_cmd, out=inst_stdout.write, err=inst_stderr.write, cwd=image_cwd, nonZeroIsFatal=non_zero_is_fatal) image_size = os.stat(image_path).st_size mx.log('Produced image size is ' + str(image_size) + ' B') i += 1 # Build the final image executable_name_args = ['-H:Name=' + final_image_name] pgo_verification_output_path = os.path.join(config.output_dir, final_image_name + '-probabilities.log') pgo_args = ['--pgo=' + profile_path, '-H:+VerifyPGOProfiles', '-H:VerificationDumpFile=' + pgo_verification_output_path] if self.pgo_instrumented_iterations > 0 or self.hotspot_pgo else [] final_image_args = base_image_build_args + executable_name_args + pgo_args mx.log('Building the final image with: ') mx.log(' ' + ' '.join([pipes.quote(str(arg)) for arg in final_image_args])) mx.run(final_image_args, out=None, err=None, cwd=image_cwd, nonZeroIsFatal=non_zero_is_fatal) # Execute the benchmark if not config.only_prepare_native_image: image_path = os.path.join(config.output_dir, final_image_name) if os.path.exists(image_path): image_run_cmd = [image_path] + image_run_args + config.extra_run_args mx.log('Running the produced native executable with: ') mx.log(' ' + ' '.join([pipes.quote(str(arg)) for arg in image_run_cmd])) mx.run(image_run_cmd, out=out, err=err, cwd=image_cwd, nonZeroIsFatal=non_zero_is_fatal) image_path = mx.join(config.output_dir, final_image_name) image_size = os.stat(image_path).st_size mx.log('Final image size is ' + str(image_size) + ' B') else: mx.log('\n\n\nImage ' + image_path + ' doesn\'t exist\n\n\n')
def home(self): return mx_sdk_vm_impl.graalvm_home(fatalIfMissing=True)
def run_java(self, args, out=None, err=None, cwd=None, nonZeroIsFatal=False): if '-version' in args: return super(NativeImageVM, self).run_java(args, out=out, err=err, cwd=cwd, nonZeroIsFatal=nonZeroIsFatal) else: # never fatal, we handle it ourselves config = NativeImageVM.BenchmarkConfig() original_java_run_args = config.parse(args) executable, classpath_arguments, system_properties, image_run_args = NativeImageVM.extract_benchmark_arguments( original_java_run_args) executable_suffix = ( '-' + config.benchmark_name) if config.benchmark_name else '' executable_name = ( os.path.splitext(os.path.basename(executable[1]))[0] + executable_suffix if executable[0] == '-jar' else executable[0] + executable_suffix).lower() final_image_name = executable_name + '-' + self.config_name() stages = NativeImageVM.Stages( config, out, err, final_image_name, self.is_gate, True if self.is_gate else nonZeroIsFatal, os.path.abspath(cwd if cwd else os.getcwd())) bench_suite = mx.suite('vm') root_dir = config.benchmark_output_dir if config.benchmark_output_dir else mx.join( bench_suite.dir, 'mxbuild') config.output_dir = mx.join( os.path.abspath(root_dir), 'native-image-bench-' + executable_name + '-' + self.config_name()) if not os.path.exists(config.output_dir): os.makedirs(config.output_dir) config.profile_dir = config.output_dir profile_path_no_extension = os.path.join(config.profile_dir, executable_name) profile_file_extension = '.iprof' latest_profile_path = profile_path_no_extension + '-latest' + profile_file_extension config.config_dir = os.path.join(config.output_dir, 'config') if not os.path.exists(config.config_dir): os.makedirs(config.config_dir) config.log_dir = config.output_dir if stages.change_stage('agent'): profile_path = profile_path_no_extension + '-agent' + profile_file_extension hotspot_vm_args = [ '-ea', '-esa' ] if self.is_gate and not config.skip_agent_assertions else [] hotspot_run_args = [] hotspot_vm_args += [ '-agentlib:native-image-agent=config-output-dir=' + str(config.config_dir), '-XX:-UseJVMCINativeLibrary' ] if self.hotspot_pgo: hotspot_vm_args += [ '-Dgraal.PGOInstrument=' + profile_path ] if self.hotspot_pgo and not self.is_gate and config.extra_agent_profile_run_args: hotspot_run_args += config.extra_agent_profile_run_args elif config.extra_agent_run_args: hotspot_run_args += config.extra_agent_run_args else: hotspot_run_args += image_run_args hotspot_args = hotspot_vm_args + classpath_arguments + executable + system_properties + hotspot_run_args java_command = os.path.join( mx_sdk_vm_impl.graalvm_home(fatalIfMissing=True), 'bin', 'java') with stages.set_command([java_command] + hotspot_args) as s: s.execute_command() if self.hotspot_pgo and s.exit_code == 0: mx.copyfile(profile_path, latest_profile_path) base_image_build_args = [ os.path.join(mx_sdk_vm_impl.graalvm_home(fatalIfMissing=True), 'bin', 'native-image') ] base_image_build_args += [ '--no-fallback', '--no-server', '-g', '--allow-incomplete-classpath' ] base_image_build_args += [ '-J-ea', '-J-esa', '-H:+VerifyGraalGraphs', '-H:+VerifyPhases', '-H:+TraceClassInitialization' ] if self.is_gate else [] base_image_build_args += system_properties base_image_build_args += classpath_arguments base_image_build_args += executable base_image_build_args += ['-H:Path=' + config.output_dir] base_image_build_args += [ '-H:ConfigurationFileDirectories=' + config.config_dir ] if self.is_llvm: base_image_build_args += [ '-H:CompilerBackend=llvm', '-H:Features=org.graalvm.home.HomeFinderFeature', '-H:DeadlockWatchdogInterval=0' ] base_image_build_args += config.extra_image_build_arguments if not self.hotspot_pgo: # Native Image profile collection i = 0 instrumented_iterations = self.pgo_instrumented_iterations if config.pgo_iteration_num is None else int( config.pgo_iteration_num) while i < instrumented_iterations: profile_path = profile_path_no_extension + '-' + str( i) + profile_file_extension instrumentation_image_name = executable_name + '-instrument-' + str( i) instrumentation_image_latest = executable_name + '-instrument-latest' image_path = os.path.join(config.output_dir, instrumentation_image_name) image_path_latest = os.path.join( config.output_dir, instrumentation_image_latest) if stages.change_stage('instrument-image', str(i)): executable_name_args = [ '-H:Name=' + instrumentation_image_name ] pgo_verification_output_path = os.path.join( config.output_dir, instrumentation_image_name + '-probabilities.log') pgo_args = [ '--pgo=' + latest_profile_path, '-H:+VerifyPGOProfiles', '-H:VerificationDumpFile=' + pgo_verification_output_path ] instrument_args = ['--pgo-instrument' ] + ([] if i == 0 else pgo_args) instrument_args += [ '-H:+InlineAllExplored' ] if self.pgo_inline_explored else [] instrument_args += [ '-H:' + ('+' if self.pgo_context_sensitive else '-') + 'EnablePGOContextSensitivity' ] with stages.set_command(base_image_build_args + executable_name_args + instrument_args) as s: s.execute_command() if s.exit_code == 0: mx.copyfile(image_path, image_path_latest) if i + 1 == instrumented_iterations and s.exit_code == 0: image_size = os.stat(image_path).st_size out('Instrumented image size: ' + str(image_size) + ' B') if stages.change_stage('instrument-run', str(i)): image_run_cmd = [image_path] image_run_cmd += [ '-XX:ProfilesDumpFile=' + profile_path ] if config.extra_profile_run_args: image_run_cmd += config.extra_profile_run_args else: image_run_cmd += image_run_args + config.extra_run_args with stages.set_command(image_run_cmd) as s: s.execute_command() if s.exit_code == 0: mx.copyfile(profile_path, latest_profile_path) i += 1 image_path = mx.join(config.output_dir, final_image_name) # Build the final image if stages.change_stage('image'): executable_name_args = ['-H:Name=' + final_image_name] pgo_verification_output_path = os.path.join( config.output_dir, final_image_name + '-probabilities.log') pgo_args = ['--pgo=' + latest_profile_path, '-H:+VerifyPGOProfiles', '-H:VerificationDumpFile=' + pgo_verification_output_path] if self.pgo_instrumented_iterations > 0 or self.hotspot_pgo else [] final_image_command = base_image_build_args + executable_name_args + pgo_args with stages.set_command(final_image_command) as s: s.execute_command() if s.exit_code == 0: image_size = os.stat(image_path).st_size out('Final image size: ' + str(image_size) + ' B') # Execute the benchmark if stages.change_stage('run'): image_path = os.path.join(config.output_dir, final_image_name) image_run_cmd = [image_path ] + image_run_args + config.extra_run_args with stages.set_command(image_run_cmd) as s: s.execute_command(True)
def gate_body(args, tasks): with Task('Vm: GraalVM dist names', tasks, tags=['names']) as t: if t: mx_sdk_vm.verify_graalvm_configs(suites=['vm', 'vm-enterprise']) with Task('Vm: Basic GraalVM Tests', tasks, tags=[VmGateTasks.compiler]) as t: if t and mx_sdk_vm_impl.has_component('GraalVM compiler'): # 1. the build must be a GraalVM # 2. the build must be JVMCI-enabled since the 'GraalVM compiler' component is registered mx_sdk_vm_impl.check_versions( mx_sdk_vm_impl.graalvm_output(), graalvm_version_regex=mx_sdk_vm_impl.graalvm_version_regex, expect_graalvm=True, check_jvmci=True) if mx_sdk_vm_impl.has_component('LibGraal'): libgraal_location = mx_sdk_vm_impl.get_native_image_locations( 'LibGraal', 'jvmcicompiler') if libgraal_location is None: mx.warn( "Skipping libgraal tests: no library enabled in the LibGraal component" ) else: extra_vm_arguments = [ '-XX:+UseJVMCICompiler', '-XX:+UseJVMCINativeLibrary', '-XX:JVMCILibPath=' + dirname(libgraal_location) ] if args.extra_vm_argument: extra_vm_arguments += args.extra_vm_argument import mx_compiler # run avrora on the GraalVM binary itself with Task('LibGraal Compiler:GraalVM DaCapo-avrora', tasks, tags=[VmGateTasks.libgraal]) as t: if t: java_exe = join(mx_sdk_vm_impl.graalvm_home(), 'bin', 'java') mx.run([ java_exe, '-XX:+UseJVMCICompiler', '-XX:+UseJVMCINativeLibrary', '-jar', mx.library('DACAPO').get_path(True), 'avrora', '-n', '1' ]) # Ensure that fatal errors in libgraal route back to HotSpot vmargs = [ '-XX:+UseJVMCICompiler', '-XX:+UseJVMCINativeLibrary', '-XX:+PrintFlagsFinal', '-Dlibgraal.CrashAt=length,hashCode', '-Dlibgraal.CrashAtIsFatal=true' ] cmd = ["dacapo:avrora", "--tracker=none", "--" ] + vmargs + ["--", "--preserve"] out = mx.OutputCapture() exitcode, bench_suite, _ = mx_benchmark.gate_mx_benchmark( cmd, nonZeroIsFatal=False, out=out, err=out) if exitcode == 0: if 'CrashAtIsFatal: no fatalError function pointer installed' in out.data: # Executing a VM that does not configure fatal errors handling # in libgraal to route back through the VM. pass else: mx.abort( 'Expected following benchmark to result in non-zero exit code: ' + ' '.join(cmd)) else: if len(bench_suite.scratchDirs()) == 0: mx.abort( "No scratch dir found despite error being expected!" ) latest_scratch_dir = bench_suite.scratchDirs()[-1] seen_libjvmci_log = False hs_errs = glob.glob( join(latest_scratch_dir, 'hs_err_pid*.log')) if not hs_errs: mx.abort( 'Expected a file starting with "hs_err_pid" in test directory. Entries found=' + str(listdir(latest_scratch_dir))) for hs_err in hs_errs: mx.log("Verifying content of {}".format( join(latest_scratch_dir, hs_err))) with open(join(latest_scratch_dir, hs_err)) as fp: contents = fp.read() if 'libjvmci' in hs_err: seen_libjvmci_log = True if 'Fatal error: Forced crash' not in contents: mx.abort( 'Expected "Fatal error: Forced crash" to be in contents of ' + hs_err + ':' + linesep + contents) else: if 'Fatal error in JVMCI' not in contents: mx.abort( 'Expected "Fatal error in JVMCI" to be in contents of ' + hs_err + ':' + linesep + contents) if 'JVMCINativeLibraryErrorFile' in out.data and not seen_libjvmci_log: mx.abort( 'Expected a file matching "hs_err_pid*_libjvmci.log" in test directory. Entries found=' + str(listdir(latest_scratch_dir))) # Only clean up scratch dir on success for scratch_dir in bench_suite.scratchDirs(): mx.log( "Cleaning up scratch dir after gate task completion: {}" .format(scratch_dir)) mx.rmtree(scratch_dir) with Task('LibGraal Compiler:CTW', tasks, tags=[VmGateTasks.libgraal]) as t: if t: mx_compiler.ctw([ '-DCompileTheWorld.Config=Inline=false CompilationFailureAction=ExitVM', '-esa', '-XX:+EnableJVMCI', '-DCompileTheWorld.MultiThreaded=true', '-Dgraal.InlineDuringParsing=false', '-Dgraal.TrackNodeSourcePosition=true', '-DCompileTheWorld.Verbose=false', '-DCompileTheWorld.HugeMethodLimit=4000', '-DCompileTheWorld.MaxCompiles=150000', '-XX:ReservedCodeCacheSize=300m', ], extra_vm_arguments) mx_compiler.compiler_gate_benchmark_runner( tasks, extra_vm_arguments, prefix='LibGraal Compiler:') with Task('LibGraal Truffle:unittest', tasks, tags=[VmGateTasks.libgraal]) as t: if t: def _unittest_config_participant(config): vmArgs, mainClass, mainClassArgs = config def is_truffle_fallback(arg): fallback_args = [ "-Dtruffle.TruffleRuntime=com.oracle.truffle.api.impl.DefaultTruffleRuntime", "-Dgraalvm.ForcePolyglotInvalid=true" ] return arg in fallback_args newVmArgs = [ arg for arg in vmArgs if not is_truffle_fallback(arg) ] return (newVmArgs, mainClass, mainClassArgs) mx_unittest.add_config_participant( _unittest_config_participant) excluded_tests = environ.get("TEST_LIBGRAAL_EXCLUDE") if excluded_tests: with NamedTemporaryFile(prefix='blacklist.', mode='w', delete=False) as fp: fp.file.writelines( [l + '\n' for l in excluded_tests.split()]) unittest_args = ["--blacklist", fp.name] else: unittest_args = [] unittest_args = unittest_args + [ "--enable-timing", "--verbose" ] compiler_log_file = "graal-compiler.log" mx_unittest.unittest(unittest_args + extra_vm_arguments + [ "-Dpolyglot.engine.AllowExperimentalOptions=true", "-Dpolyglot.engine.CompileImmediately=true", "-Dpolyglot.engine.BackgroundCompilation=false", "-Dpolyglot.engine.TraceCompilation=true", "-Dpolyglot.log.file={0}".format(compiler_log_file), "-Dgraalvm.locatorDisabled=true", "truffle" ]) if exists(compiler_log_file): remove(compiler_log_file) else: mx.warn("Skipping libgraal tests: component not enabled") gate_substratevm(tasks) gate_sulong(tasks) gate_python(tasks) gate_svm_sl_tck(tasks) gate_svm_truffle_tck_js(tasks)
def run_java(self, args, out=None, err=None, cwd=None, nonZeroIsFatal=False): if '-version' in args: return super(NativeImageVM, self).run_java(args, out=out, err=err, cwd=cwd, nonZeroIsFatal=nonZeroIsFatal) else: image_cwd = os.path.abspath(cwd if cwd else os.getcwd()) non_zero_is_fatal = self.is_gate or nonZeroIsFatal config = NativeImageVM.BenchmarkConfig() original_java_run_args = config.parse(args) executable, classpath_arguments, system_properties, image_run_args = NativeImageVM.extract_benchmark_arguments( original_java_run_args) executable_name = (os.path.splitext(os.path.basename( executable[1]))[0] if executable[0] == '-jar' else executable[0]).lower() image_path = os.path.join(image_cwd, executable_name) config.profile_dir = mx.mkdtemp(suffix='profile', prefix='native-image') profile_path = os.path.join(config.profile_dir, executable_name + '.iprof') # Agent configuration and/or HotSpot profiling needs_config = (config.config_dir is None) and config.needs_config if needs_config or self.hotspot_pgo: hotspot_vm_args = ['-ea', '-esa'] if self.is_gate else [] hotspot_run_args = [] if needs_config: config.config_dir = mx.mkdtemp(suffix='config', prefix='native-image') hotspot_vm_args += [ '-agentlib:native-image-agent=config-output-dir=' + str(config.config_dir), '-XX:-UseJVMCINativeLibrary' ] if self.hotspot_pgo: hotspot_vm_args += [ '-Dgraal.PGOInstrument=' + profile_path ] if config.extra_agent_run_args: hotspot_run_args += config.extra_profile_run_args if self.hotspot_pgo and not self.is_gate else config.extra_agent_run_args else: hotspot_run_args += image_run_args hotspot_args = hotspot_vm_args + classpath_arguments + executable + system_properties + hotspot_run_args mx.log( 'Running with HotSpot to get the configuration files and profiles. This could take a while:' ) super(NativeImageVM, self).run_java(hotspot_args, out=None, err=None, cwd=image_cwd, nonZeroIsFatal=non_zero_is_fatal) base_image_build_args = [ os.path.join(mx_sdk_vm_impl.graalvm_home(fatalIfMissing=True), 'bin', 'native-image') ] base_image_build_args += ['--no-fallback'] base_image_build_args += [ '-J-ea', '-J-esa', '-H:+VerifyGraalGraphs', '-H:+VerifyPhases', '-H:+TraceClassInitialization' ] if self.is_gate else [] base_image_build_args += system_properties base_image_build_args += classpath_arguments base_image_build_args += executable base_image_build_args += [ '-H:Name=' + executable_name, '-H:Path=' + image_cwd ] if needs_config: base_image_build_args += [ '-H:ConfigurationFileDirectories=' + config.config_dir ] if self.is_llvm: base_image_build_args += [ '-H:CompilerBackend=llvm', '-H:-SpawnIsolates' ] base_image_build_args += config.extra_image_build_arguments # PGO instrumentation i = 0 while i < self.pgo_instrumented_iterations: instrument_args = ['--pgo-instrument'] + ( [] if i == 0 and not self.hotspot_pgo else ['--pgo']) instrument_image_build_args = base_image_build_args + instrument_args mx.log('Building the instrumentation image with: ') mx.log(' ' + ' '.join([ pipes.quote(str(arg)) for arg in instrument_image_build_args ])) mx.run(instrument_image_build_args, out=None, err=None, cwd=image_cwd, nonZeroIsFatal=non_zero_is_fatal) image_run_cmd = [image_path] image_run_cmd += ['-XX:ProfilesDumpFile=' + profile_path] if config.extra_profile_run_args: image_run_cmd += config.extra_profile_run_args else: image_run_cmd += image_run_args + config.extra_run_args mx.log('Running the instrumented image with: ') mx.log( ' ' + ' '.join([pipes.quote(str(arg)) for arg in image_run_cmd])) mx.run(image_run_cmd, out=out, err=err, cwd=image_cwd, nonZeroIsFatal=non_zero_is_fatal) i += 1 # Build the final image pgo_args = ['--pgo=' + profile_path] if self.pgo_instrumented_iterations > 0 or self.hotspot_pgo else [] final_image_args = base_image_build_args + pgo_args mx.log('Building the final image with: ') mx.log( ' ' + ' '.join([pipes.quote(str(arg)) for arg in final_image_args])) mx.run(final_image_args, out=None, err=None, cwd=image_cwd, nonZeroIsFatal=non_zero_is_fatal) # Execute the benchmark image_run_cmd = [image_path ] + image_run_args + config.extra_run_args mx.log('Running the produced native executable with: ') mx.log(' ' + ' '.join([pipes.quote(str(arg)) for arg in image_run_cmd])) mx.run(image_run_cmd, out=out, err=err, cwd=image_cwd, nonZeroIsFatal=non_zero_is_fatal)