def harness(unittestDeps, vmLauncher, vmArgs): prefixArgs = ['-esa', '-ea'] if gc_after_test: prefixArgs.append('-XX:-DisableExplicitGC') with open(testfile) as fp: testclasses = [l.rstrip() for l in fp.readlines()] jdk = vmLauncher.jdk() vmArgs += mx.get_runtime_jvm_args(unittestDeps, cp_prefix=prefixCp+coreCp, jdk=jdk) # suppress menubar and dock when running on Mac vmArgs = prefixArgs + ['-Djava.awt.headless=true'] + vmArgs if jdk.javaCompliance > '1.8': # This is required to access jdk.internal.module.Modules for supporting # the @AddExports annotation. vmArgs = vmArgs + ['--add-exports=java.base/jdk.internal.module=ALL-UNNAMED'] # Execute Junit directly when one test is being run. This simplifies # replaying the VM execution in a native debugger (e.g., gdb). mainClassArgs = coreArgs + (testclasses if len(testclasses) == 1 else ['@' + mx._cygpathU2W(testfile)]) config = (vmArgs, mainClass, mainClassArgs) for p in _config_participants: config = p(config) vmLauncher.launcher(*config)
def microbench(self, args): """run JMH microbenchmark projects""" parser = ArgumentParser(prog='mx microbench', description=microbench.__doc__, usage="%(prog)s [command options|VM options] [-- [JMH options]]") parser.add_argument('--jar', help='Explicitly specify micro-benchmark location') self.add_arguments(parser) mx.warn("`mx microbench` is deprecated! Consider moving to `mx_benchmark.JMHRunnerBenchmarkSuite`") known_args, args = parser.parse_known_args(args) vmArgs, jmhArgs = mx.extract_VM_args(args, useDoubleDash=True) vmArgs = self.parseVmArgs(vmArgs) # look for -f in JMH arguments forking = True for i in range(len(jmhArgs)): arg = jmhArgs[i] if arg.startswith('-f'): if arg == '-f' and (i+1) < len(jmhArgs): arg += jmhArgs[i+1] try: if int(arg[2:]) == 0: forking = False except ValueError: pass if known_args.jar: # use the specified jar args = ['-jar', known_args.jar] if not forking: args += vmArgs # we do not know the compliance level of the jar - assuming 1.8 self.javaCompliance = mx.JavaCompliance('1.8') else: # find all projects with a direct JMH dependency projects_dict = mx_benchmark.JMHRunnerBenchmarkSuite.get_jmh_projects_dict() jmhProjects = projects_dict['JMH'] if 'JMH' in projects_dict else [] # get java compliance - 1.8 is minimum since we build jmh-runner with java 8 self.javaCompliance = max([p.javaCompliance for p in jmhProjects] + [mx.JavaCompliance('1.8')]) cpArgs = mx.get_runtime_jvm_args([p.name for p in jmhProjects], jdk=mx.get_jdk(self.javaCompliance)) # execute JMH runner if forking: args, cpVmArgs = self.filterVmArgs(cpArgs) vmArgs += cpVmArgs else: args = cpArgs + vmArgs args += ['org.openjdk.jmh.Main'] if forking: def quoteSpace(s): if " " in s: return '"' + s + '"' return s forkedVmArgs = map(quoteSpace, self.parseForkedVmArgs(vmArgs)) args += ['--jvmArgsPrepend', ' '.join(forkedVmArgs)] self.run_java(args + jmhArgs)
def rbcheck(args): '''Checks FastR builtins against GnuR gnur-only: GnuR builtins not implemented in FastR (i.e. TODO list). fastr-only: FastR builtins not implemented in GnuR both-diff: implemented in both GnuR and FastR, but with difference in signature (e.g. visibility) both: implemented in both GnuR and FastR with matching signature If the option --filter is not given, shows all groups. Multiple groups can be combined: e.g. "--filter gnur-only,fastr-only"''' vmArgs = mx.get_runtime_jvm_args('com.oracle.truffle.r.test') args.append("--suite-path") args.append(mx.primary_suite().dir) vmArgs += ['com.oracle.truffle.r.test.tools.RBuiltinCheck'] mx.run_java(vmArgs + args)
def run(self, cwd, args): _check_vm_args(self.name(), args) truffle_options = [ # '-Dgraal.TruffleCompilationExceptionsAreFatal=true' ] dists = ["GRAALPYTHON", "TRUFFLE_NFI", "GRAALPYTHON-LAUNCHER"] # add configuration specified distributions if self._distributions: assert isinstance( self._distributions, list), "distributions must be either None or a list" dists += self._distributions extra_polyglot_args = self._extra_polyglot_args if isinstance( self._extra_polyglot_args, list) else [] if mx.suite("tools", fatalIfMissing=False): dists.append('CHROMEINSPECTOR') if mx.suite("sulong", fatalIfMissing=False): dists.append('SULONG') if mx.suite("sulong-managed", fatalIfMissing=False): dists.append('SULONG_MANAGED') extra_polyglot_args += ["--experimental-options"] else: extra_polyglot_args += ["--experimental-options"] vm_args = mx.get_runtime_jvm_args(dists, cp_suffix=self._cp_suffix, cp_prefix=self._cp_prefix) if isinstance(self._extra_vm_args, list): vm_args += self._extra_vm_args vm_args += [ "-Dpython.home=%s" % join(SUITE.dir, "graalpython"), "com.oracle.graal.python.shell.GraalPythonMain" ] cmd = truffle_options + vm_args + extra_polyglot_args + args host_vm = self.host_vm() with environ(self._env): if hasattr(host_vm, 'run_lang'): return host_vm.run_lang('graalpython', extra_polyglot_args + args, cwd) else: return host_vm.run(cwd, cmd)
def testgen(args): '''generate the expected output for unit tests''' # check we are in the home directory if os.getcwd() != _fastr_suite.dir: mx.abort('must run rtestgen from FastR home directory') def need_version_check(): vardef = 'FASTR_TESTGEN_GNUR' in os.environ varval = os.environ['FASTR_TESTGEN_GNUR'] if vardef else None version_check = vardef and varval != 'internal' if version_check: rpath = join(varval, 'bin', 'R') else: rpath = None return version_check, rpath version_check, rpath = need_version_check() if version_check: # check the version of GnuR against FastR try: fastr_version = subprocess.check_output([ mx.get_jdk().java, mx.get_runtime_jvm_args('com.oracle.truffle.r.runtime'), 'com.oracle.truffle.r.runtime.RVersionNumber' ]) gnur_version = subprocess.check_output([rpath, '--version']) if not gnur_version.startswith(fastr_version): mx.abort( 'R version is incompatible with FastR, please update to ' + fastr_version) except subprocess.CalledProcessError: mx.abort('RVersionNumber.main failed') tests = _all_generated_unit_tests() # now just invoke unittst with the appropriate options mx.log("generating expected output for packages: ") for pkg in tests: mx.log(" " + str(pkg)) os.environ["TZDIR"] = "/usr/share/zoneinfo/" _unset_conflicting_envs() mx_unittest.unittest([ '-Dfastr.test.gen.expected=' + _test_srcdir(), '-Dfastr.test.gen.expected.quiet', '-Dfastr.test.project.output.dir=' + mx.project('com.oracle.truffle.r.test').output_dir() ] + tests)
def create_parser(grammar_project, grammar_package, grammar_name, copyright_template, args=None, out=None): """create the DSL expression parser using antlr""" grammar_dir = mx.project(grammar_project).source_dirs()[0] + "/" + grammar_package.replace(".", "/") + "/" mx.run_java(mx.get_runtime_jvm_args(['ANTLR4_COMPLETE']) + ["org.antlr.v4.Tool", "-package", grammar_package, "-no-listener"] + args + [grammar_dir + grammar_name + ".g4"], out=out) for filename in [grammar_dir + grammar_name + "Lexer.java", grammar_dir + grammar_name + "Parser.java"]: with open(filename, 'r') as content_file: content = content_file.read() # remove first line content = "\n".join(content.split("\n")[1:]) # modify SuppressWarnings to remove useless entries content = PTRN_SUPPRESS_WARNINGS.sub('@SuppressWarnings("all")', content) # remove useless casts content = PTRN_LOCALCTXT_CAST.sub('_localctx', content) content = PTRN_TOKEN_CAST.sub('_errHandler.recoverInline(this)', content) # add copyright header content = copyright_template.format(content) with open(filename, 'w') as content_file: content_file.write(content)
def native_junit(native_image, unittest_args, build_args=None, run_args=None): build_args = build_args if not None else [] run_args = run_args if not None else [] junit_native_dir = join(svmbuild_dir(), platform_subdir(), 'junit') mkpath(junit_native_dir) junit_tmp_dir = tempfile.mkdtemp(dir=junit_native_dir) try: unittest_deps = [] def dummy_harness(test_deps, vm_launcher, vm_args): unittest_deps.extend(test_deps) unittest_file = join(junit_tmp_dir, 'svmjunit.tests') _run_tests(unittest_args, dummy_harness, _VMLauncher('dummy_launcher', None, mx_compiler.jdk), ['@Test', '@Parameters'], unittest_file, None, None, None, None) extra_image_args = mx.get_runtime_jvm_args(unittest_deps, jdk=mx_compiler.jdk) native_image(build_args + extra_image_args + ['--tool:junit=' + unittest_file, '-H:Path=' + junit_tmp_dir]) unittest_image = join(junit_tmp_dir, 'svmjunit') mx.run([unittest_image] + run_args) finally: remove_tree(junit_tmp_dir)
def create_parser(grammar_project, grammar_package, grammar_name, copyright_template, args=None, out=None): """create the DSL expression parser using antlr""" grammar_dir = os.path.join(mx.project(grammar_project).source_dirs()[0], *grammar_package.split(".")) + os.path.sep mx.run_java(mx.get_runtime_jvm_args(['ANTLR4_COMPLETE']) + ["org.antlr.v4.Tool", "-package", grammar_package, "-no-listener"] + args + [grammar_dir + grammar_name + ".g4"], out=out) for filename in [grammar_dir + grammar_name + "Lexer.java", grammar_dir + grammar_name + "Parser.java"]: with open(filename, 'r') as content_file: content = content_file.read() # remove first line content = "\n".join(content.split("\n")[1:]) # modify SuppressWarnings to remove useless entries content = PTRN_SUPPRESS_WARNINGS.sub('@SuppressWarnings("all")', content) # remove useless casts content = PTRN_LOCALCTXT_CAST.sub('_localctx', content) content = PTRN_TOKEN_CAST.sub('_errHandler.recoverInline(this)', content) # add copyright header content = copyright_template.format(content) with open(filename, 'w') as content_file: content_file.write(content)
def _toolchain_helper(self, args=None, out=None): parser = ArgumentParser( prog='mx ' + self.mx_command, description='launch toolchain commands', epilog= 'Additional arguments are forwarded to the LLVM image command.', add_help=False) parser.add_argument('command', help='toolchain command', metavar='<command>', choices=self._supported_exes()) parsed_args, tool_args = parser.parse_known_args(args) main = self._tool_to_main(self.exe_map[parsed_args.command]) if "JACOCO" in os.environ: mx_gate._jacoco = os.environ["JACOCO"] return mx.run_java(mx.get_runtime_jvm_args([self.dist]) + [main] + tool_args, out=out)
def run(self, cwd, args): _check_vm_args(self.name(), args) truffle_options = [ # '-Dgraal.TruffleCompilationExceptionsAreFatal=true' ] dists = ["GRAALPYTHON", "TRUFFLE_NFI", "GRAALPYTHON-LAUNCHER"] # add configuration specified distributions if self._distributions: assert isinstance( self._distributions, list), "distributions must be either None or a list" dists += self._distributions extra_polyglot_args = self._extra_polyglot_args if isinstance( self._extra_polyglot_args, list) else [] if mx.suite("sulong", fatalIfMissing=False): dists.append('SULONG') if mx.suite("sulong-managed", fatalIfMissing=False): dists.append('SULONG_MANAGED') extra_polyglot_args += [ "--experimental-options", mx_subst.path_substitutions.substitute( '--llvm.libraryPath=<path:SULONG_MANAGED_LIBS>') ] else: extra_polyglot_args += [ "--experimental-options", mx_subst.path_substitutions.substitute( '--llvm.libraryPath=<path:SULONG_LIBS>') ] vm_args = mx.get_runtime_jvm_args(dists, cp_suffix=self._cp_suffix, cp_prefix=self._cp_prefix) if isinstance(self._extra_vm_args, list): vm_args += self._extra_vm_args vm_args += [ "-Dpython.home=%s" % join(SUITE.dir, "graalpython"), "com.oracle.graal.python.shell.GraalPythonMain" ] cmd = truffle_options + vm_args + extra_polyglot_args + args return self.host_vm().run(cwd, cmd)
def doImage(args, projects, imageVmArgs, imageNormalArgs, extraClassPath="", timeout=None): projects += [svmDistribution] if imageVmArgs is None: imageVmArgs = [] if imageNormalArgs is None: imageNormalArgs = [] extraVmArgs, extraNormalArgs = _parse_standard_arguments(False, args) server, port = processServerArguments(args) compilerClasspath = _classpath([svmDistribution]) cpVmArgs = mx.get_runtime_jvm_args([librarySupportDistribution] + projects, cp_suffix=extraClassPath, jdk=mx_compiler.jdk) (idx, classpath) = mx.find_classpath_arg(cpVmArgs) imageVmArgs += cpVmArgs[:idx - 1] + cpVmArgs[idx + 1:] classpathlist = list(set(classpath.split(os.pathsep))) classpath = os.pathsep.join(classpathlist) vmArgs, normalArgs = mx.extract_VM_args(args, useDoubleDash=True, defaultAllVMArgs=False) imageGenArgs = extraNormalArgs + imageNormalArgs + normalArgs if mx._opts.strip_jars: imageGenArgs += ['-H:-VerifyNamingConventions'] run_executable(server, port, imageVmArgs + extraVmArgs + vmArgs, 'com.oracle.svm.hosted.NativeImageGeneratorRunner', compilerClasspath, classpath, imageGenArgs, timeout=timeout)
def do_run_r(args, command, extraVmArgs=None, jdk=None, **kwargs): ''' This is the basic function that runs a FastR process, where args have already been parsed. Args: args: a list of command arguments command: e.g. 'R', implicitly defines the entry class (can be None for AOT) extraVmArgs: additional vm arguments jdk: jdk (an mx.JDKConfig instance) to use **kwargs other keyword args understood by run_java nonZeroIsFatal: whether to terminate the execution run fails out,err possible redirects to collect output By default a non-zero return code will cause an mx.abort, unless nonZeroIsFatal=False The assumption is that the VM is already built and available. ''' env = kwargs['env'] if 'env' in kwargs else os.environ setREnvironment(env) if not jdk: jdk = get_default_jdk() dists = ['FASTR'] if mx.suite("sulong", fatalIfMissing=False): dists.append('SULONG') vmArgs = mx.get_runtime_jvm_args(dists, jdk=jdk) vmArgs += set_graal_options() vmArgs += _sulong_options() args = _sulong_args() + args if not "FASTR_NO_ASSERTS" in os.environ and (extraVmArgs is None or not '-da' in extraVmArgs): # unless explicitly disabled we enable assertion checking vmArgs += ['-ea', '-esa'] if extraVmArgs: vmArgs += extraVmArgs vmArgs = _sanitize_vmArgs(jdk, vmArgs) if command: vmArgs.extend(_command_class_dict[command.lower()]) return mx.run_java(vmArgs + args, jdk=jdk, **kwargs)
def gate_svm_sl_tck(tasks): with Task('SVM Truffle TCK', tasks, tags=[VmGateTasks.svm_sl_tck]) as t: if t: tools_suite = mx.suite('tools') if not tools_suite: mx.abort("Cannot resolve tools suite.") native_image_context, svm = graalvm_svm() with native_image_context( svm.IMAGE_ASSERTION_FLAGS) as native_image: svmbuild = mkdtemp() try: import mx_compiler unittest_deps = [] unittest_file = join(svmbuild, 'truffletck.tests') mx_unittest._run_tests( [], lambda deps, vm_launcher, vm_args: unittest_deps. extend(deps), mx_unittest._VMLauncher('dummy_launcher', None, mx_compiler.jdk), ['@Test', '@Parameters'], unittest_file, [], [re.compile('com.oracle.truffle.tck.tests')], None, mx.suite('truffle')) if not exists(unittest_file): mx.abort('TCK tests not found.') unittest_deps.append( mx.dependency('truffle:TRUFFLE_SL_TCK')) unittest_deps.append( mx.dependency('truffle:TRUFFLE_TCK_INSTRUMENTATION')) vm_image_args = mx.get_runtime_jvm_args( unittest_deps, jdk=mx_compiler.jdk) options = [ '--macro:truffle', '--tool:all', '-H:Path={}'.format(svmbuild), '-H:+TruffleCheckBlackListedMethods', '-H:Class=org.junit.runner.JUnitCore', ] tests_image = native_image(vm_image_args + options) with open(unittest_file) as f: test_classes = [l.rstrip() for l in f.readlines()] mx.run([tests_image] + test_classes) finally: mx.rmtree(svmbuild)
def createCommandLineArgs(self, benchmarks, bmSuiteArgs): vmArgs = self.vmArgs(bmSuiteArgs) dists = ["GRAALPYTHON", "GRAALPYTHON-LAUNCHER"] if mx.suite("tools", fatalIfMissing=False): dists.extend(('CHROMEINSPECTOR', 'TRUFFLE_PROFILER')) if mx.suite("sulong", fatalIfMissing=False): dists.append('SULONG_NATIVE') if mx.suite("sulong-managed", fatalIfMissing=False): dists.append('SULONG_MANAGED') vmArgs += [ "-Dorg.graalvm.language.python.home=%s" % join(SUITE.dir, "graalpython"), ] vmArgs += mx.get_runtime_jvm_args(dists + ['com.oracle.graal.python.benchmarks'], jdk=mx.get_jdk()) jmh_entry = ["com.oracle.graal.python.benchmarks.parser.ParserBenchRunner"] runArgs = self.runArgs(bmSuiteArgs) bench_name = benchmarks[0] bench_args = self._benchmarks[bench_name] return vmArgs + jmh_entry + runArgs + [bench_name] + bench_args
def run(self, cwd, args): _check_vm_args(self.name(), args) truffle_options = [ # '-Dgraal.TruffleCompilationExceptionsAreFatal=true' ] dists = ["GRAALPYTHON", "GRAALPYTHON-LAUNCHER"] if mx.suite("sulong", fatalIfMissing=False): dists.append('SULONG') if mx.suite("sulong-managed", fatalIfMissing=False): dists.append('SULONG_MANAGED') vm_args = mx.get_runtime_jvm_args(dists) + [ "-Dpython.home=%s" % join(_graalpython_suite.dir, "graalpython"), "com.oracle.graal.python.shell.GraalPythonMain" ] cmd = truffle_options + vm_args + args return self.host_vm().run(cwd, cmd)
def runLLVMUnittests(unittest_runner): """runs the interop unit tests with a different unittest runner (e.g. AOT-based)""" libpath = mx_subst.path_substitutions.substitute( '-Dpolyglot.llvm.libraryPath=<path:SULONG_TEST_NATIVE>') libs = mx_subst.path_substitutions.substitute( '-Dpolyglot.llvm.libraries=<lib:sulongtest>') test_harness_dist = mx.distribution('SULONG_TEST') java_run_props = [ x for x in mx.get_runtime_jvm_args(test_harness_dist) if x.startswith('-D') ] test_suite = 'SULONG_TEST_SUITES' mx_testsuites.compileTestSuite(test_suite, extra_build_args=[]) run_args = [libpath, libs] + java_run_props build_args = ['--language:llvm'] + java_run_props unittest_runner(['com.oracle.truffle.llvm.test.interop', '--run-args'] + run_args + ['--build-args'] + build_args)
def contents(self, result): java = mx.get_jdk().java classpath_deps = [ dep for dep in self.subject.buildDependencies if isinstance(dep, mx.ClasspathDependency) ] jvm_args = [ pipes.quote(arg) for arg in mx.get_runtime_jvm_args(classpath_deps) ] jvm_args.append('-Dorg.graalvm.language.ruby.home=' + root) main_class = 'org.truffleruby.launcher.RubyLauncher' ruby_options = [ '--experimental-options', '--building-core-cexts', '--launcher=' + result, '--disable-gems', '--disable-rubyopt', ] command = [java] + jvm_args + [main_class] + ruby_options + ['"$@"'] return "#!/usr/bin/env bash\n" + "exec " + " ".join(command) + "\n"
def do_run_r(args, command, extraVmArgs=None, jdk=None, **kwargs): ''' This is the basic function that runs a FastR process, where args have already been parsed. Args: args: a list of command arguments command: e.g. 'R', implicitly defines the entry class (can be None for AOT) extraVmArgs: additional vm arguments jdk: jdk (an mx.JDKConfig instance) to use **kwargs other keyword args understood by run_java nonZeroIsFatal: whether to terminate the execution run fails out,err possible redirects to collect output By default a non-zero return code will cause an mx.abort, unless nonZeroIsFatal=False The assumption is that the VM is already built and available. ''' env = kwargs['env'] if 'env' in kwargs else os.environ setREnvironment(env) if not jdk: jdk = get_default_jdk() dists = ['FASTR'] if _mx_sulong: dists.append('SULONG') vmArgs = mx.get_runtime_jvm_args(dists, jdk=jdk) vmArgs += set_graal_options() vmArgs += _sulong_options() if extraVmArgs is None or not '-da' in extraVmArgs: # unless explicitly disabled we enable assertion checking vmArgs += ['-ea', '-esa'] if extraVmArgs: vmArgs += extraVmArgs vmArgs = _sanitize_vmArgs(jdk, vmArgs) if command: vmArgs.append(_command_class_dict[command.lower()]) return mx.run_java(vmArgs + args, jdk=jdk, **kwargs)
def contents(self, tool, exe): # platform support all_params = '"%*"' if mx.is_windows() else '"$@"' _quote = _quote_windows if mx.is_windows() else pipes.quote # build command line java = mx.get_jdk().java classpath_deps = [dep for dep in self.subject.buildDependencies if isinstance(dep, mx.ClasspathDependency)] extra_props = ['-Dorg.graalvm.launcher.executablename="{}"'.format(exe)] main_class = self.subject.suite.toolchain._tool_to_main(tool) # add jvm args from dependencies jvm_args = [_quote(arg) for arg in mx.get_runtime_jvm_args(classpath_deps)] # add properties from the project if hasattr(self.subject, "getJavaProperties"): for key, value in sorted(self.subject.getJavaProperties().items()): jvm_args.append("-D" + key + "=" + value) command = [java] + jvm_args + extra_props + [main_class, all_params] # create script if mx.is_windows(): return "@echo off\n" + " ".join(command) + "\n" else: return "#!/usr/bin/env bash\n" + "exec " + " ".join(command) + "\n"
def run(self, cwd, args): jar = mx.get_env(ENV_JYTHON_JAR) if jar: _check_vm_args(self.name(), args) host_vm = self.host_vm() vm_args = mx.get_runtime_jvm_args([]) vm_args += ["-jar", jar] for a in args[:]: if a.startswith("-D") or a.startswith("-XX"): vm_args.insert(0, a) args.remove(a) args = self._override_iterations_args(args) cmd = vm_args + args if not self._env: self._env = dict() with environ(self._env): return host_vm.run(cwd, cmd) else: return AbstractPythonIterationsControlVm.run(self, cwd, args)
def _native_junit(native_image, unittest_args, build_args=None, run_args=None, blacklist=None, whitelist=None): unittest_args = unittest_args build_args = build_args or [] run_args = run_args or ['--verbose'] junit_native_dir = join(svmbuild_dir(), platform_name(), 'junit') mkpath(junit_native_dir) junit_tmp_dir = tempfile.mkdtemp(dir=junit_native_dir) try: unittest_deps = [] def dummy_harness(test_deps, vm_launcher, vm_args): unittest_deps.extend(test_deps) unittest_file = join(junit_tmp_dir, 'svmjunit.tests') _run_tests(unittest_args, dummy_harness, _VMLauncher('dummy_launcher', None, mx_compiler.jdk), ['@Test', '@Parameters'], unittest_file, blacklist, whitelist, None, None) if not exists(unittest_file): mx.abort('No matching unit tests found. Skip image build and execution.') with open(unittest_file, 'r') as f: mx.log('Building junit image for matching: ' + ' '.join(l.rstrip() for l in f)) extra_image_args = mx.get_runtime_jvm_args(unittest_deps, jdk=mx_compiler.jdk) unittest_image = native_image(build_args + extra_image_args + ['--tool:junit=' + unittest_file, '-H:Path=' + junit_tmp_dir]) mx.run([unittest_image] + run_args) finally: remove_tree(junit_tmp_dir)
def native_junit(native_image, unittest_args=None, build_args=None, run_args=None): unittest_args = unittest_args or ['com.oracle.svm.test'] build_args = build_args or [] run_args = run_args or ['--verbose'] junit_native_dir = join(svmbuild_dir(), platform_name(), 'junit') mkpath(junit_native_dir) junit_tmp_dir = tempfile.mkdtemp(dir=junit_native_dir) try: unittest_deps = [] def dummy_harness(test_deps, vm_launcher, vm_args): unittest_deps.extend(test_deps) unittest_file = join(junit_tmp_dir, 'svmjunit.tests') _run_tests(unittest_args, dummy_harness, _VMLauncher('dummy_launcher', None, mx_compiler.jdk), ['@Test', '@Parameters'], unittest_file, None, None, None, None) if not exists(unittest_file): mx.abort('No matching unit tests found. Skip image build and execution.') with open(unittest_file, 'r') as f: mx.log('Building junit image for matching: ' + ' '.join(l.rstrip() for l in f)) extra_image_args = mx.get_runtime_jvm_args(unittest_deps, jdk=mx_compiler.jdk) unittest_image = native_image(build_args + extra_image_args + ['--tool:junit=' + unittest_file, '-H:Path=' + junit_tmp_dir]) mx.run([unittest_image] + run_args) finally: remove_tree(junit_tmp_dir)
def harness(unittestDeps, vmLauncher, vmArgs): prefixArgs = ['-esa', '-ea'] if '-JUnitGCAfterTest' in junit_args: prefixArgs.append('-XX:-DisableExplicitGC') jdk = vmLauncher.jdk() vmArgs += mx.get_runtime_jvm_args(unittestDeps, cp_prefix=prefixCp + coreCp, jdk=jdk) # suppress menubar and dock when running on Mac vmArgs = prefixArgs + ['-Djava.awt.headless=true'] + vmArgs if jdk.javaCompliance > '1.8': # This is required to access jdk.internal.module.Modules for supporting # the @AddExports annotation. vmArgs = vmArgs + [ '--add-exports=java.base/jdk.internal.module=ALL-UNNAMED' ] with open(testfile) as fp: testclass = None for l in fp: if testclass: testclass = None break testclass = l.rstrip() if testclass: # Execute Junit directly when one test is being run. This simplifies # replaying the VM execution in a native debugger (e.g., gdb). mainClassArgs = junit_args + [testclass] else: mainClassArgs = junit_args + ['@' + mx._cygpathU2W(testfile)] config = (vmArgs, mainClass, mainClassArgs) for p in _config_participants: config = p(config) vmLauncher.launcher(*config)
def contents(self, tool, exe): # platform support all_params = '"%*"' if mx.is_windows() else '"$@"' _quote = _quote_windows if mx.is_windows() else pipes.quote # build command line java = mx.get_jdk().java classpath_deps = [ dep for dep in self.subject.buildDependencies if isinstance(dep, mx.ClasspathDependency) ] extra_props = [ '-Dorg.graalvm.launcher.executablename="{}"'.format(exe) ] main_class = self.subject.suite.toolchain._tool_to_main(tool) jvm_args = [ _quote(arg) for arg in mx.get_runtime_jvm_args(classpath_deps) ] command = [java] + jvm_args + extra_props + [main_class, all_params] # create script if mx.is_windows(): return "@echo off\n" + " ".join(command) + "\n" else: return "#!/usr/bin/env bash\n" + "exec " + " ".join(command) + "\n"
def store_jvm_args(self): distributions = [dep.name for dep in self.subject.buildDependencies] jvm_args = mx.get_runtime_jvm_args(distributions) properties = [] while jvm_args: arg = jvm_args.pop(0) if arg == '-cp': classpath = self.relativize(jvm_args.pop(0)).split(':') else: properties.append(self.relativize(arg)) sulong_libs = mx_subst.path_substitutions.substitute('-Dpolyglot.llvm.libraryPath=<path:SULONG_LIBS>') properties.append(self.relativize(sulong_libs)) boot_dists = ['GRAAL_SDK', 'TRUFFLE_API', 'LAUNCHER_COMMON'] bootclasspath = self.relativize(mx.classpath(boot_dists)).split(':') for jar in bootclasspath: classpath.remove(jar) with open(self.jvm_args_file, 'w') as f: f.write('bootclasspath=\\\n' + ':\\\n'.join(bootclasspath) + '\n\n') f.write('classpath=\\\n' + ':\\\n'.join(classpath) + '\n\n') f.write('properties=(\n"' + '"\n"'.join(properties) + '"\n)\n')
def harness(unittestDeps, vmLauncher, vmArgs): prefixArgs = ['-esa', '-ea'] if gc_after_test: prefixArgs.append('-XX:-DisableExplicitGC') with open(testfile) as fp: testclasses = [l.rstrip() for l in fp.readlines()] vmArgs += mx.get_runtime_jvm_args(unittestDeps, cp_prefix=prefixCp + coreCp, jdk=vmLauncher.jdk()) # suppress menubar and dock when running on Mac vmArgs = prefixArgs + ['-Djava.awt.headless=true'] + vmArgs # Execute Junit directly when one test is being run. This simplifies # replaying the VM execution in a native debugger (e.g., gdb). mainClassArgs = coreArgs + (testclasses if len(testclasses) == 1 else ['@' + mx._cygpathU2W(testfile)]) config = (vmArgs, mainClass, mainClassArgs) for p in _config_participants: config = p(config) vmLauncher.launcher(*config)
def ruby_check_heap_dump(input_args, out=None): print("mx ruby_check_heap_dump " + " ".join(input_args)) args = input_args args.insert(0, "--experimental-options") dists = ['TRUFFLERUBY', 'TRUFFLE_NFI', 'SULONG_NATIVE', 'TRUFFLERUBY-TEST'] vm_args, truffleruby_args = mx.extract_VM_args(args, useDoubleDash=True, defaultAllVMArgs=False) vm_args += mx.get_runtime_jvm_args(dists) # vm_args.append("-agentlib:jdwp=transport=dt_socket,server=y,address=8000,suspend=y") vm_args.append("org.truffleruby.LeakTest") out = mx.OutputCapture() if out is None else out retval = mx.run_java(vm_args + truffleruby_args, jdk=jdk, nonZeroIsFatal=False, out=out) if retval == 0: print("PASSED") print(out.data) elif os.environ.get("CI") and "--keep-dump" not in input_args: # rerun once with heap dumping enabled out = mx.OutputCapture() ruby_check_heap_dump(["--keep-dump"] + input_args, out=out) path = out.data.strip().partition("Dump file:")[2].strip() if path: save_path = os.path.join(root, "dumps", "leak_test") try: os.makedirs(save_path) except OSError: pass dest = shutil.copy(path, save_path) # pylint: disable=assignment-from-no-return print("Heapdump file kept in " + dest) raise Exception("heap dump check failed") else: print("FAILED") print(out.data) raise Exception("heap dump check failed")
def testgen(args): '''generate the expected output for unit tests, and All/Failing test classes''' parser = ArgumentParser(prog='r testgen') parser.add_argument('--tests', action='store', default=_all_unit_tests(), help='pattern to match test classes') args = parser.parse_args(args) # check we are in the home directory if os.getcwd() != _fastr_suite.dir: mx.abort('must run rtestgen from FastR home directory') def need_version_check(): vardef = os.environ.has_key('FASTR_TESTGEN_GNUR') varval = os.environ['FASTR_TESTGEN_GNUR'] if vardef else None version_check = vardef and varval != 'internal' if version_check: rpath = join(varval, 'bin', 'R') else: rpath = None return version_check, rpath version_check, rpath = need_version_check() if version_check: # check the version of GnuR against FastR try: fastr_version = subprocess.check_output([mx.get_jdk().java, mx.get_runtime_jvm_args('com.oracle.truffle.r.runtime'), 'com.oracle.truffle.r.runtime.RVersionNumber']) gnur_version = subprocess.check_output([rpath, '--version']) if not gnur_version.startswith(fastr_version): mx.abort('R version is incompatible with FastR, please update to ' + fastr_version) except subprocess.CalledProcessError: mx.abort('RVersionNumber.main failed') # now just invoke junit with the appropriate options mx.log("generating expected output for packages: ") for pkg in args.tests.split(','): mx.log(" " + str(pkg)) os.environ["TZDIR"] = "/usr/share/zoneinfo/" _unset_conflicting_envs() junit(['--tests', args.tests, '--gen-expected-output', '--gen-expected-quiet'])
def rbdiag(args): '''Diagnoses FastR builtins -v Verbose output including the list of unimplemented specializations -n Ignore RNull as an argument type -m Ignore RMissing as an argument type --mnonly Uses the RMissing and RNull values as the only samples for the chimney-sweeping --noSelfTest Does not perform the pipeline self-test using the generated samples as the intro to each chimney-sweeping. It has no effect when --mnonly is specified as the self-test is never performed in that case. --sweep Performs the 'chimney-sweeping'. The sample combination selection method is determined automatically. --sweep=lite Performs the 'chimney-sweeping'. The diagonal sample selection method is used. --sweep=total Performs the 'chimney-sweeping'. The total sample selection method is used. --matchLevel=same Outputs produced by FastR and GnuR must be same (default) --matchLevel=error Outputs are considered matching if none or both outputs contain an error --maxSweeps=N Sets the maximum number of sweeps --outMaxLev=N Sets the maximum output detail level for report messages. Use 0 for the basic messages only. If no builtin is specified, all registered builtins are diagnosed. An external builtin is specified by the fully qualified name of its node class. Examples: mx rbdiag mx rbdiag colSums colMeans -v mx rbdiag scan -m -n mx rbdiag colSums --sweep mx rbdiag com.oracle.truffle.r.library.stats.Rnorm ''' vmArgs = mx.get_runtime_jvm_args('com.oracle.truffle.r.nodes.test') setREnvironment() os.environ["FASTR_TESTGEN_GNUR"] = "internal" # this should work for Linux and Mac: os.environ["TZDIR"] = "/usr/share/zoneinfo/" vmArgs += ['com.oracle.truffle.r.nodes.test.RBuiltinDiagnostics'] mx.run_java(vmArgs + args)
def llirtestgen(args=None, out=None): return mx.run_java(mx.get_runtime_jvm_args(["LLIR_TEST_GEN"]) + ["com.oracle.truffle.llvm.tests.llirtestgen.LLIRTestGen"] + args, out=out)
def getClasspathOptions(): """gets the classpath of the Sulong distributions""" return mx.get_runtime_jvm_args('SULONG')
def junit(args): '''run ZipPy Junit tests''' parser = ArgumentParser(prog='mx junit') parser.add_argument( '--tests', action='store', help= 'patterns to match test classes (specify multiple patterns using \',\')' ) parser.add_argument('--J', dest='vm_args', action='append', help='Java VM arguments (e.g. --J @-dsa)', metavar='@<args>') parser.add_argument('--jdk', action='store', help='JDK to use for the "java" command') args = parser.parse_args(args) vmArgs = ['-ea', '-esa'] # enable when necessary # vmArgs += ['-Xss12m'] if args.vm_args: vmArgs = vmArgs + mx_fastr.split_j_args(args.vm_args) testfile = os.environ.get('MX_TESTFILE', None) if testfile is None: (_, testfile) = tempfile.mkstemp(".testclasses", "mx") os.close(_) if args.jdk: jdk = mx.get_jdk(tag=args.jdk) if not jdk: mx.abort("JDK '" + args.jdk + "' not found!") else: tag = 'jvmci' if _mx_graal else None jdk = mx.get_jdk(tag=tag) candidates = [] for p in mx.projects(opt_limit_to_suite=True, limit_to_primary=True): if not p.isJavaProject() or jdk.javaCompliance < p.javaCompliance: continue candidates += mx_unittest._find_classes_with_annotations( p, None, ['@Test']).keys() tests = [] if args.tests is None else [ name for name in args.tests.split(',') ] classes = [] if len(tests) == 0: classes = candidates else: for test in tests: exists = False for candidate in candidates: if test in candidate: exists = True classes.append(candidate) if not exists: mx.warn('no tests matched by substring "' + test + '"') vmArgs += mx.get_runtime_jvm_args(['ZIPPY', 'ZIPPY_UNIT_TESTS'], jdk=jdk) if len(classes) != 0: if len(classes) == 1: testClassArgs = ['--testclass', classes[0]] else: with open(testfile, 'w') as f: for c in classes: f.write(c + '\n') testClassArgs = ['--testsfile', testfile] junitArgs = ['edu.uci.python.test.ZipPyJUnitRunner'] + testClassArgs rc = mx.run_java(vmArgs + junitArgs, nonZeroIsFatal=False, jdk=jdk) return rc else: return 0
def do_run_python(args, extra_vm_args=None, env=None, jdk=None, **kwargs): if not env: env = os.environ check_vm_env = env.get('GRAALPYTHON_MUST_USE_GRAAL', False) if check_vm_env: if check_vm_env == '1': check_vm(must_be_jvmci=True) elif check_vm_env == '0': check_vm() dists = ['GRAALPYTHON'] vm_args, graalpython_args = mx.extract_VM_args(args, useDoubleDash=True, defaultAllVMArgs=False) internal_graalpython_args, graalpython_args, additional_dists = _extract_graalpython_internal_options( graalpython_args) dists += additional_dists if '--python.WithJavaStacktrace' not in graalpython_args: graalpython_args.insert(0, '--python.WithJavaStacktrace') if _sulong: dists.append('SULONG') if mx.suite("sulong-managed", fatalIfMissing=False): dists.append('SULONG_MANAGED') vm_args.append( mx_subst.path_substitutions.substitute( '-Dpolyglot.llvm.libraryPath=<path:SULONG_LIBS>:<path:SULONG_MANAGED_LIBS>' )) else: vm_args.append( mx_subst.path_substitutions.substitute( '-Dpolyglot.llvm.libraryPath=<path:SULONG_LIBS>')) # Try eagerly to include tools on Tim's computer if not mx.suite("/tools", fatalIfMissing=False): def _is_user(user, home=None): if home: return os.environ.get("USER") == user and os.environ.get(home) return os.environ.get("USER") == user if _is_user("tim", "MAGLEV_HOME") or _is_user("cbasca") or _is_user("fa"): _suite.import_suite("tools", version=None, urlinfos=None, in_subdir=True) dists.append('CHROMEINSPECTOR') if _sulong: vm_args.append("-Dpolyglot.llvm.enableLVI=true") vm_args += mx.get_runtime_jvm_args(dists, jdk=jdk) vm_args += internal_graalpython_args if not jdk: jdk = get_jdk() # default: assertion checking is enabled if extra_vm_args is None or '-da' not in extra_vm_args: vm_args += ['-ea', '-esa'] if extra_vm_args: vm_args += extra_vm_args vm_args.append("com.oracle.graal.python.shell.GraalPythonMain") return mx.run_java(vm_args + graalpython_args, jdk=jdk, **kwargs)
def run_grid_server(args, **kwargs): vmArgs = mx.get_runtime_jvm_args(['GRID_DEVICE_REMOTE_SERVER'], jdk=get_default_jdk()) vmArgs.append('com.oracle.truffle.r.library.fastrGrid.device.remote.server.RemoteDeviceServer') return mx.run_java(vmArgs + args, jdk=get_default_jdk(), **kwargs)
def getClasspathOptions(): """gets the classpath of the Sulong distributions""" return mx.get_runtime_jvm_args(['SULONG', 'SULONG_LAUNCHER'])
def getClasspathOptions(extra_dists=None): """gets the classpath of the Sulong distributions""" return mx.get_runtime_jvm_args( ['SULONG', 'SULONG_LAUNCHER', 'TRUFFLE_NFI'] + (extra_dists or []))
def extract_bitcode(args=None, out=None): """Extract embedded LLVM bitcode from object files""" return mx.run_java( mx.get_runtime_jvm_args(["com.oracle.truffle.llvm.tools"]) + ["com.oracle.truffle.llvm.tools.ExtractBitcode"] + args, out=out)
def extract_bitcode(args=None, out=None): return mx.run_java( mx.get_runtime_jvm_args(["com.oracle.truffle.llvm.tools"]) + ["com.oracle.truffle.llvm.tools.ExtractBitcode"] + args, out=out)
def junit(args, harness, parser=None, jdk_default=None): """run Junit tests""" suppliedParser = parser is not None parser = parser if suppliedParser else ArgumentParser(prog='mx junit') parser.add_argument('--tests', action='store', help='pattern to match test classes') parser.add_argument('--J', dest='vm_args', action='append', help='target VM arguments (e.g. --J @-dsa)', metavar='@<args>') parser.add_argument('--jdk', action='store', help='jdk to use') if suppliedParser: parser.add_argument('remainder', nargs=REMAINDER, metavar='...') args = parser.parse_args(args) vmArgs = ['-ea', '-esa'] if args.vm_args: vmArgs = vmArgs + mx_fastr.split_j_args(args.vm_args) testfile = os.environ.get('MX_TESTFILE', None) if testfile is None: (_, testfile) = tempfile.mkstemp(".testclasses", "mx") os.close(_) candidates = [] if args.jdk: jdk = mx.get_jdk(tag=args.jdk) if not jdk: mx.abort("jdk '" + args.jdk + "' not found") else: if not jdk_default: jdk = mx.get_jdk() else: jdk = jdk_default for p in mx.projects(opt_limit_to_suite=True): if not p.isJavaProject() or jdk.javaCompliance < p.javaCompliance: continue candidates += _find_classes_with_annotations(p, None, ['@Test']).keys() tests = [] if args.tests is None else [name for name in args.tests.split(',')] classes = [] if len(tests) == 0: classes = candidates else: for t in tests: found = False for c in candidates: if t in c: found = True classes.append(c) if not found: mx.warn('no tests matched by substring "' + t + '"') vmArgs += mx.get_runtime_jvm_args([pcp.name for pcp in mx.projects(opt_limit_to_suite=True) if pcp.isJavaProject() and pcp.javaCompliance <= jdk.javaCompliance], jdk=jdk) if len(classes) != 0: if len(classes) == 1: testClassArgs = ['--testclass', classes[0]] else: with open(testfile, 'w') as f: for c in classes: f.write(c + '\n') testClassArgs = ['--testsfile', testfile] junitArgs = ['com.oracle.truffle.r.test.FastRJUnitWrapper'] + testClassArgs rc = harness(args, vmArgs, jdk, junitArgs) return rc else: return 0