def harness(unittestDeps, vmLauncher, vmArgs):
prefixArgs = ['-esa', '-ea']
if gc_after_test:
prefixArgs.append('-XX:-DisableExplicitGC')
with open(testfile) as fp:
testclasses = [l.rstrip() for l in fp.readlines()]
jdk = vmLauncher.jdk()
vmArgs += mx.get_runtime_jvm_args(unittestDeps, cp_prefix=prefixCp+coreCp, jdk=jdk)
# suppress menubar and dock when running on Mac
vmArgs = prefixArgs + ['-Djava.awt.headless=true'] + vmArgs
if jdk.javaCompliance > '1.8':
# This is required to access jdk.internal.module.Modules for supporting
# the @AddExports annotation.
vmArgs = vmArgs + ['--add-exports=java.base/jdk.internal.module=ALL-UNNAMED']
# Execute Junit directly when one test is being run. This simplifies
# replaying the VM execution in a native debugger (e.g., gdb).
mainClassArgs = coreArgs + (testclasses if len(testclasses) == 1 else ['@' + mx._cygpathU2W(testfile)])
config = (vmArgs, mainClass, mainClassArgs)
for p in _config_participants:
config = p(config)
vmLauncher.launcher(*config)
def microbench(self, args):
"""run JMH microbenchmark projects"""
parser = ArgumentParser(prog='mx microbench', description=microbench.__doc__,
usage="%(prog)s [command options|VM options] [-- [JMH options]]")
parser.add_argument('--jar', help='Explicitly specify micro-benchmark location')
self.add_arguments(parser)
mx.warn("`mx microbench` is deprecated! Consider moving to `mx_benchmark.JMHRunnerBenchmarkSuite`")
known_args, args = parser.parse_known_args(args)
vmArgs, jmhArgs = mx.extract_VM_args(args, useDoubleDash=True)
vmArgs = self.parseVmArgs(vmArgs)
# look for -f in JMH arguments
forking = True
for i in range(len(jmhArgs)):
arg = jmhArgs[i]
if arg.startswith('-f'):
if arg == '-f' and (i+1) < len(jmhArgs):
arg += jmhArgs[i+1]
try:
if int(arg[2:]) == 0:
forking = False
except ValueError:
pass
if known_args.jar:
# use the specified jar
args = ['-jar', known_args.jar]
if not forking:
args += vmArgs
# we do not know the compliance level of the jar - assuming 1.8
self.javaCompliance = mx.JavaCompliance('1.8')
else:
# find all projects with a direct JMH dependency
projects_dict = mx_benchmark.JMHRunnerBenchmarkSuite.get_jmh_projects_dict()
jmhProjects = projects_dict['JMH'] if 'JMH' in projects_dict else []
# get java compliance - 1.8 is minimum since we build jmh-runner with java 8
self.javaCompliance = max([p.javaCompliance for p in jmhProjects] + [mx.JavaCompliance('1.8')])
cpArgs = mx.get_runtime_jvm_args([p.name for p in jmhProjects], jdk=mx.get_jdk(self.javaCompliance))
# execute JMH runner
if forking:
args, cpVmArgs = self.filterVmArgs(cpArgs)
vmArgs += cpVmArgs
else:
args = cpArgs + vmArgs
args += ['org.openjdk.jmh.Main']
if forking:
def quoteSpace(s):
if " " in s:
return '"' + s + '"'
return s
forkedVmArgs = map(quoteSpace, self.parseForkedVmArgs(vmArgs))
args += ['--jvmArgsPrepend', ' '.join(forkedVmArgs)]
self.run_java(args + jmhArgs)
def rbcheck(args):
'''Checks FastR builtins against GnuR
gnur-only: GnuR builtins not implemented in FastR (i.e. TODO list).
fastr-only: FastR builtins not implemented in GnuR
both-diff: implemented in both GnuR and FastR, but with difference
in signature (e.g. visibility)
both: implemented in both GnuR and FastR with matching signature
If the option --filter is not given, shows all groups.
Multiple groups can be combined: e.g. "--filter gnur-only,fastr-only"'''
vmArgs = mx.get_runtime_jvm_args('com.oracle.truffle.r.test')
args.append("--suite-path")
args.append(mx.primary_suite().dir)
vmArgs += ['com.oracle.truffle.r.test.tools.RBuiltinCheck']
mx.run_java(vmArgs + args)
def run(self, cwd, args):
_check_vm_args(self.name(), args)
truffle_options = [
# '-Dgraal.TruffleCompilationExceptionsAreFatal=true'
]
dists = ["GRAALPYTHON", "TRUFFLE_NFI", "GRAALPYTHON-LAUNCHER"]
# add configuration specified distributions
if self._distributions:
assert isinstance(
self._distributions,
list), "distributions must be either None or a list"
dists += self._distributions
extra_polyglot_args = self._extra_polyglot_args if isinstance(
self._extra_polyglot_args, list) else []
if mx.suite("tools", fatalIfMissing=False):
dists.append('CHROMEINSPECTOR')
if mx.suite("sulong", fatalIfMissing=False):
dists.append('SULONG')
if mx.suite("sulong-managed", fatalIfMissing=False):
dists.append('SULONG_MANAGED')
extra_polyglot_args += ["--experimental-options"]
else:
extra_polyglot_args += ["--experimental-options"]
vm_args = mx.get_runtime_jvm_args(dists,
cp_suffix=self._cp_suffix,
cp_prefix=self._cp_prefix)
if isinstance(self._extra_vm_args, list):
vm_args += self._extra_vm_args
vm_args += [
"-Dpython.home=%s" % join(SUITE.dir, "graalpython"),
"com.oracle.graal.python.shell.GraalPythonMain"
]
cmd = truffle_options + vm_args + extra_polyglot_args + args
host_vm = self.host_vm()
with environ(self._env):
if hasattr(host_vm, 'run_lang'):
return host_vm.run_lang('graalpython',
extra_polyglot_args + args, cwd)
else:
return host_vm.run(cwd, cmd)
def testgen(args):
'''generate the expected output for unit tests'''
# check we are in the home directory
if os.getcwd() != _fastr_suite.dir:
mx.abort('must run rtestgen from FastR home directory')
def need_version_check():
vardef = 'FASTR_TESTGEN_GNUR' in os.environ
varval = os.environ['FASTR_TESTGEN_GNUR'] if vardef else None
version_check = vardef and varval != 'internal'
if version_check:
rpath = join(varval, 'bin', 'R')
else:
rpath = None
return version_check, rpath
version_check, rpath = need_version_check()
if version_check:
# check the version of GnuR against FastR
try:
fastr_version = subprocess.check_output([
mx.get_jdk().java,
mx.get_runtime_jvm_args('com.oracle.truffle.r.runtime'),
'com.oracle.truffle.r.runtime.RVersionNumber'
])
gnur_version = subprocess.check_output([rpath, '--version'])
if not gnur_version.startswith(fastr_version):
mx.abort(
'R version is incompatible with FastR, please update to ' +
fastr_version)
except subprocess.CalledProcessError:
mx.abort('RVersionNumber.main failed')
tests = _all_generated_unit_tests()
# now just invoke unittst with the appropriate options
mx.log("generating expected output for packages: ")
for pkg in tests:
mx.log(" " + str(pkg))
os.environ["TZDIR"] = "/usr/share/zoneinfo/"
_unset_conflicting_envs()
mx_unittest.unittest([
'-Dfastr.test.gen.expected=' + _test_srcdir(),
'-Dfastr.test.gen.expected.quiet', '-Dfastr.test.project.output.dir=' +
mx.project('com.oracle.truffle.r.test').output_dir()
] + tests)
def create_parser(grammar_project, grammar_package, grammar_name, copyright_template, args=None, out=None):
"""create the DSL expression parser using antlr"""
grammar_dir = mx.project(grammar_project).source_dirs()[0] + "/" + grammar_package.replace(".", "/") + "/"
mx.run_java(mx.get_runtime_jvm_args(['ANTLR4_COMPLETE']) + ["org.antlr.v4.Tool", "-package", grammar_package, "-no-listener"] + args + [grammar_dir + grammar_name + ".g4"], out=out)
for filename in [grammar_dir + grammar_name + "Lexer.java", grammar_dir + grammar_name + "Parser.java"]:
with open(filename, 'r') as content_file:
content = content_file.read()
# remove first line
content = "\n".join(content.split("\n")[1:])
# modify SuppressWarnings to remove useless entries
content = PTRN_SUPPRESS_WARNINGS.sub('@SuppressWarnings("all")', content)
# remove useless casts
content = PTRN_LOCALCTXT_CAST.sub('_localctx', content)
content = PTRN_TOKEN_CAST.sub('_errHandler.recoverInline(this)', content)
# add copyright header
content = copyright_template.format(content)
with open(filename, 'w') as content_file:
content_file.write(content)
def native_junit(native_image, unittest_args, build_args=None, run_args=None):
build_args = build_args if not None else []
run_args = run_args if not None else []
junit_native_dir = join(svmbuild_dir(), platform_subdir(), 'junit')
mkpath(junit_native_dir)
junit_tmp_dir = tempfile.mkdtemp(dir=junit_native_dir)
try:
unittest_deps = []
def dummy_harness(test_deps, vm_launcher, vm_args):
unittest_deps.extend(test_deps)
unittest_file = join(junit_tmp_dir, 'svmjunit.tests')
_run_tests(unittest_args, dummy_harness, _VMLauncher('dummy_launcher', None, mx_compiler.jdk), ['@Test', '@Parameters'], unittest_file, None, None, None, None)
extra_image_args = mx.get_runtime_jvm_args(unittest_deps, jdk=mx_compiler.jdk)
native_image(build_args + extra_image_args + ['--tool:junit=' + unittest_file, '-H:Path=' + junit_tmp_dir])
unittest_image = join(junit_tmp_dir, 'svmjunit')
mx.run([unittest_image] + run_args)
finally:
remove_tree(junit_tmp_dir)
def create_parser(grammar_project, grammar_package, grammar_name, copyright_template, args=None, out=None):
"""create the DSL expression parser using antlr"""
grammar_dir = os.path.join(mx.project(grammar_project).source_dirs()[0], *grammar_package.split(".")) + os.path.sep
mx.run_java(mx.get_runtime_jvm_args(['ANTLR4_COMPLETE']) + ["org.antlr.v4.Tool", "-package", grammar_package, "-no-listener"] + args + [grammar_dir + grammar_name + ".g4"], out=out)
for filename in [grammar_dir + grammar_name + "Lexer.java", grammar_dir + grammar_name + "Parser.java"]:
with open(filename, 'r') as content_file:
content = content_file.read()
# remove first line
content = "\n".join(content.split("\n")[1:])
# modify SuppressWarnings to remove useless entries
content = PTRN_SUPPRESS_WARNINGS.sub('@SuppressWarnings("all")', content)
# remove useless casts
content = PTRN_LOCALCTXT_CAST.sub('_localctx', content)
content = PTRN_TOKEN_CAST.sub('_errHandler.recoverInline(this)', content)
# add copyright header
content = copyright_template.format(content)
with open(filename, 'w') as content_file:
content_file.write(content)
def _toolchain_helper(self, args=None, out=None):
parser = ArgumentParser(
prog='mx ' + self.mx_command,
description='launch toolchain commands',
epilog=
'Additional arguments are forwarded to the LLVM image command.',
add_help=False)
parser.add_argument('command',
help='toolchain command',
metavar='<command>',
choices=self._supported_exes())
parsed_args, tool_args = parser.parse_known_args(args)
main = self._tool_to_main(self.exe_map[parsed_args.command])
if "JACOCO" in os.environ:
mx_gate._jacoco = os.environ["JACOCO"]
return mx.run_java(mx.get_runtime_jvm_args([self.dist]) + [main] +
tool_args,
out=out)
def run(self, cwd, args):
_check_vm_args(self.name(), args)
truffle_options = [
# '-Dgraal.TruffleCompilationExceptionsAreFatal=true'
]
dists = ["GRAALPYTHON", "TRUFFLE_NFI", "GRAALPYTHON-LAUNCHER"]
# add configuration specified distributions
if self._distributions:
assert isinstance(
self._distributions,
list), "distributions must be either None or a list"
dists += self._distributions
extra_polyglot_args = self._extra_polyglot_args if isinstance(
self._extra_polyglot_args, list) else []
if mx.suite("sulong", fatalIfMissing=False):
dists.append('SULONG')
if mx.suite("sulong-managed", fatalIfMissing=False):
dists.append('SULONG_MANAGED')
extra_polyglot_args += [
"--experimental-options",
mx_subst.path_substitutions.substitute(
'--llvm.libraryPath=<path:SULONG_MANAGED_LIBS>')
]
else:
extra_polyglot_args += [
"--experimental-options",
mx_subst.path_substitutions.substitute(
'--llvm.libraryPath=<path:SULONG_LIBS>')
]
vm_args = mx.get_runtime_jvm_args(dists,
cp_suffix=self._cp_suffix,
cp_prefix=self._cp_prefix)
if isinstance(self._extra_vm_args, list):
vm_args += self._extra_vm_args
vm_args += [
"-Dpython.home=%s" % join(SUITE.dir, "graalpython"),
"com.oracle.graal.python.shell.GraalPythonMain"
]
cmd = truffle_options + vm_args + extra_polyglot_args + args
return self.host_vm().run(cwd, cmd)
def doImage(args,
projects,
imageVmArgs,
imageNormalArgs,
extraClassPath="",
timeout=None):
projects += [svmDistribution]
if imageVmArgs is None:
imageVmArgs = []
if imageNormalArgs is None:
imageNormalArgs = []
extraVmArgs, extraNormalArgs = _parse_standard_arguments(False, args)
server, port = processServerArguments(args)
compilerClasspath = _classpath([svmDistribution])
cpVmArgs = mx.get_runtime_jvm_args([librarySupportDistribution] + projects,
cp_suffix=extraClassPath,
jdk=mx_compiler.jdk)
(idx, classpath) = mx.find_classpath_arg(cpVmArgs)
imageVmArgs += cpVmArgs[:idx - 1] + cpVmArgs[idx + 1:]
classpathlist = list(set(classpath.split(os.pathsep)))
classpath = os.pathsep.join(classpathlist)
vmArgs, normalArgs = mx.extract_VM_args(args,
useDoubleDash=True,
defaultAllVMArgs=False)
imageGenArgs = extraNormalArgs + imageNormalArgs + normalArgs
if mx._opts.strip_jars:
imageGenArgs += ['-H:-VerifyNamingConventions']
run_executable(server,
port,
imageVmArgs + extraVmArgs + vmArgs,
'com.oracle.svm.hosted.NativeImageGeneratorRunner',
compilerClasspath,
classpath,
imageGenArgs,
timeout=timeout)
def do_run_r(args, command, extraVmArgs=None, jdk=None, **kwargs):
'''
This is the basic function that runs a FastR process, where args have already been parsed.
Args:
args: a list of command arguments
command: e.g. 'R', implicitly defines the entry class (can be None for AOT)
extraVmArgs: additional vm arguments
jdk: jdk (an mx.JDKConfig instance) to use
**kwargs other keyword args understood by run_java
nonZeroIsFatal: whether to terminate the execution run fails
out,err possible redirects to collect output
By default a non-zero return code will cause an mx.abort, unless nonZeroIsFatal=False
The assumption is that the VM is already built and available.
'''
env = kwargs['env'] if 'env' in kwargs else os.environ
setREnvironment(env)
if not jdk:
jdk = get_default_jdk()
dists = ['FASTR']
if mx.suite("sulong", fatalIfMissing=False):
dists.append('SULONG')
vmArgs = mx.get_runtime_jvm_args(dists, jdk=jdk)
vmArgs += set_graal_options()
vmArgs += _sulong_options()
args = _sulong_args() + args
if not "FASTR_NO_ASSERTS" in os.environ and (extraVmArgs is None
or not '-da' in extraVmArgs):
# unless explicitly disabled we enable assertion checking
vmArgs += ['-ea', '-esa']
if extraVmArgs:
vmArgs += extraVmArgs
vmArgs = _sanitize_vmArgs(jdk, vmArgs)
if command:
vmArgs.extend(_command_class_dict[command.lower()])
return mx.run_java(vmArgs + args, jdk=jdk, **kwargs)
def gate_svm_sl_tck(tasks):
with Task('SVM Truffle TCK', tasks, tags=[VmGateTasks.svm_sl_tck]) as t:
if t:
tools_suite = mx.suite('tools')
if not tools_suite:
mx.abort("Cannot resolve tools suite.")
native_image_context, svm = graalvm_svm()
with native_image_context(
svm.IMAGE_ASSERTION_FLAGS) as native_image:
svmbuild = mkdtemp()
try:
import mx_compiler
unittest_deps = []
unittest_file = join(svmbuild, 'truffletck.tests')
mx_unittest._run_tests(
[], lambda deps, vm_launcher, vm_args: unittest_deps.
extend(deps),
mx_unittest._VMLauncher('dummy_launcher', None,
mx_compiler.jdk),
['@Test', '@Parameters'], unittest_file, [],
[re.compile('com.oracle.truffle.tck.tests')], None,
mx.suite('truffle'))
if not exists(unittest_file):
mx.abort('TCK tests not found.')
unittest_deps.append(
mx.dependency('truffle:TRUFFLE_SL_TCK'))
unittest_deps.append(
mx.dependency('truffle:TRUFFLE_TCK_INSTRUMENTATION'))
vm_image_args = mx.get_runtime_jvm_args(
unittest_deps, jdk=mx_compiler.jdk)
options = [
'--macro:truffle',
'--tool:all',
'-H:Path={}'.format(svmbuild),
'-H:+TruffleCheckBlackListedMethods',
'-H:Class=org.junit.runner.JUnitCore',
]
tests_image = native_image(vm_image_args + options)
with open(unittest_file) as f:
test_classes = [l.rstrip() for l in f.readlines()]
mx.run([tests_image] + test_classes)
finally:
mx.rmtree(svmbuild)
def createCommandLineArgs(self, benchmarks, bmSuiteArgs):
vmArgs = self.vmArgs(bmSuiteArgs)
dists = ["GRAALPYTHON", "GRAALPYTHON-LAUNCHER"]
if mx.suite("tools", fatalIfMissing=False):
dists.extend(('CHROMEINSPECTOR', 'TRUFFLE_PROFILER'))
if mx.suite("sulong", fatalIfMissing=False):
dists.append('SULONG_NATIVE')
if mx.suite("sulong-managed", fatalIfMissing=False):
dists.append('SULONG_MANAGED')
vmArgs += [
"-Dorg.graalvm.language.python.home=%s" % join(SUITE.dir, "graalpython"),
]
vmArgs += mx.get_runtime_jvm_args(dists + ['com.oracle.graal.python.benchmarks'], jdk=mx.get_jdk())
jmh_entry = ["com.oracle.graal.python.benchmarks.parser.ParserBenchRunner"]
runArgs = self.runArgs(bmSuiteArgs)
bench_name = benchmarks[0]
bench_args = self._benchmarks[bench_name]
return vmArgs + jmh_entry + runArgs + [bench_name] + bench_args
def run(self, cwd, args):
_check_vm_args(self.name(), args)
truffle_options = [
# '-Dgraal.TruffleCompilationExceptionsAreFatal=true'
]
dists = ["GRAALPYTHON", "GRAALPYTHON-LAUNCHER"]
if mx.suite("sulong", fatalIfMissing=False):
dists.append('SULONG')
if mx.suite("sulong-managed", fatalIfMissing=False):
dists.append('SULONG_MANAGED')
vm_args = mx.get_runtime_jvm_args(dists) + [
"-Dpython.home=%s" % join(_graalpython_suite.dir, "graalpython"),
"com.oracle.graal.python.shell.GraalPythonMain"
]
cmd = truffle_options + vm_args + args
return self.host_vm().run(cwd, cmd)
def runLLVMUnittests(unittest_runner):
"""runs the interop unit tests with a different unittest runner (e.g. AOT-based)"""
libpath = mx_subst.path_substitutions.substitute(
'-Dpolyglot.llvm.libraryPath=<path:SULONG_TEST_NATIVE>')
libs = mx_subst.path_substitutions.substitute(
'-Dpolyglot.llvm.libraries=<lib:sulongtest>')
test_harness_dist = mx.distribution('SULONG_TEST')
java_run_props = [
x for x in mx.get_runtime_jvm_args(test_harness_dist)
if x.startswith('-D')
]
test_suite = 'SULONG_TEST_SUITES'
mx_testsuites.compileTestSuite(test_suite, extra_build_args=[])
run_args = [libpath, libs] + java_run_props
build_args = ['--language:llvm'] + java_run_props
unittest_runner(['com.oracle.truffle.llvm.test.interop', '--run-args'] +
run_args + ['--build-args'] + build_args)
def contents(self, result):
java = mx.get_jdk().java
classpath_deps = [
dep for dep in self.subject.buildDependencies
if isinstance(dep, mx.ClasspathDependency)
]
jvm_args = [
pipes.quote(arg) for arg in mx.get_runtime_jvm_args(classpath_deps)
]
jvm_args.append('-Dorg.graalvm.language.ruby.home=' + root)
main_class = 'org.truffleruby.launcher.RubyLauncher'
ruby_options = [
'--experimental-options',
'--building-core-cexts',
'--launcher=' + result,
'--disable-gems',
'--disable-rubyopt',
]
command = [java] + jvm_args + [main_class] + ruby_options + ['"$@"']
return "#!/usr/bin/env bash\n" + "exec " + " ".join(command) + "\n"
def do_run_r(args, command, extraVmArgs=None, jdk=None, **kwargs):
'''
This is the basic function that runs a FastR process, where args have already been parsed.
Args:
args: a list of command arguments
command: e.g. 'R', implicitly defines the entry class (can be None for AOT)
extraVmArgs: additional vm arguments
jdk: jdk (an mx.JDKConfig instance) to use
**kwargs other keyword args understood by run_java
nonZeroIsFatal: whether to terminate the execution run fails
out,err possible redirects to collect output
By default a non-zero return code will cause an mx.abort, unless nonZeroIsFatal=False
The assumption is that the VM is already built and available.
'''
env = kwargs['env'] if 'env' in kwargs else os.environ
setREnvironment(env)
if not jdk:
jdk = get_default_jdk()
dists = ['FASTR']
if _mx_sulong:
dists.append('SULONG')
vmArgs = mx.get_runtime_jvm_args(dists, jdk=jdk)
vmArgs += set_graal_options()
vmArgs += _sulong_options()
if extraVmArgs is None or not '-da' in extraVmArgs:
# unless explicitly disabled we enable assertion checking
vmArgs += ['-ea', '-esa']
if extraVmArgs:
vmArgs += extraVmArgs
vmArgs = _sanitize_vmArgs(jdk, vmArgs)
if command:
vmArgs.append(_command_class_dict[command.lower()])
return mx.run_java(vmArgs + args, jdk=jdk, **kwargs)
def contents(self, tool, exe):
# platform support
all_params = '"%*"' if mx.is_windows() else '"$@"'
_quote = _quote_windows if mx.is_windows() else pipes.quote
# build command line
java = mx.get_jdk().java
classpath_deps = [dep for dep in self.subject.buildDependencies if isinstance(dep, mx.ClasspathDependency)]
extra_props = ['-Dorg.graalvm.launcher.executablename="{}"'.format(exe)]
main_class = self.subject.suite.toolchain._tool_to_main(tool)
# add jvm args from dependencies
jvm_args = [_quote(arg) for arg in mx.get_runtime_jvm_args(classpath_deps)]
# add properties from the project
if hasattr(self.subject, "getJavaProperties"):
for key, value in sorted(self.subject.getJavaProperties().items()):
jvm_args.append("-D" + key + "=" + value)
command = [java] + jvm_args + extra_props + [main_class, all_params]
# create script
if mx.is_windows():
return "@echo off\n" + " ".join(command) + "\n"
else:
return "#!/usr/bin/env bash\n" + "exec " + " ".join(command) + "\n"
def run(self, cwd, args):
jar = mx.get_env(ENV_JYTHON_JAR)
if jar:
_check_vm_args(self.name(), args)
host_vm = self.host_vm()
vm_args = mx.get_runtime_jvm_args([])
vm_args += ["-jar", jar]
for a in args[:]:
if a.startswith("-D") or a.startswith("-XX"):
vm_args.insert(0, a)
args.remove(a)
args = self._override_iterations_args(args)
cmd = vm_args + args
if not self._env:
self._env = dict()
with environ(self._env):
return host_vm.run(cwd, cmd)
else:
return AbstractPythonIterationsControlVm.run(self, cwd, args)
def _native_junit(native_image, unittest_args, build_args=None, run_args=None, blacklist=None, whitelist=None):
unittest_args = unittest_args
build_args = build_args or []
run_args = run_args or ['--verbose']
junit_native_dir = join(svmbuild_dir(), platform_name(), 'junit')
mkpath(junit_native_dir)
junit_tmp_dir = tempfile.mkdtemp(dir=junit_native_dir)
try:
unittest_deps = []
def dummy_harness(test_deps, vm_launcher, vm_args):
unittest_deps.extend(test_deps)
unittest_file = join(junit_tmp_dir, 'svmjunit.tests')
_run_tests(unittest_args, dummy_harness, _VMLauncher('dummy_launcher', None, mx_compiler.jdk), ['@Test', '@Parameters'], unittest_file, blacklist, whitelist, None, None)
if not exists(unittest_file):
mx.abort('No matching unit tests found. Skip image build and execution.')
with open(unittest_file, 'r') as f:
mx.log('Building junit image for matching: ' + ' '.join(l.rstrip() for l in f))
extra_image_args = mx.get_runtime_jvm_args(unittest_deps, jdk=mx_compiler.jdk)
unittest_image = native_image(build_args + extra_image_args + ['--tool:junit=' + unittest_file, '-H:Path=' + junit_tmp_dir])
mx.run([unittest_image] + run_args)
finally:
remove_tree(junit_tmp_dir)
def native_junit(native_image, unittest_args=None, build_args=None, run_args=None):
unittest_args = unittest_args or ['com.oracle.svm.test']
build_args = build_args or []
run_args = run_args or ['--verbose']
junit_native_dir = join(svmbuild_dir(), platform_name(), 'junit')
mkpath(junit_native_dir)
junit_tmp_dir = tempfile.mkdtemp(dir=junit_native_dir)
try:
unittest_deps = []
def dummy_harness(test_deps, vm_launcher, vm_args):
unittest_deps.extend(test_deps)
unittest_file = join(junit_tmp_dir, 'svmjunit.tests')
_run_tests(unittest_args, dummy_harness, _VMLauncher('dummy_launcher', None, mx_compiler.jdk), ['@Test', '@Parameters'], unittest_file, None, None, None, None)
if not exists(unittest_file):
mx.abort('No matching unit tests found. Skip image build and execution.')
with open(unittest_file, 'r') as f:
mx.log('Building junit image for matching: ' + ' '.join(l.rstrip() for l in f))
extra_image_args = mx.get_runtime_jvm_args(unittest_deps, jdk=mx_compiler.jdk)
unittest_image = native_image(build_args + extra_image_args + ['--tool:junit=' + unittest_file, '-H:Path=' + junit_tmp_dir])
mx.run([unittest_image] + run_args)
finally:
remove_tree(junit_tmp_dir)
def harness(unittestDeps, vmLauncher, vmArgs):
prefixArgs = ['-esa', '-ea']
if '-JUnitGCAfterTest' in junit_args:
prefixArgs.append('-XX:-DisableExplicitGC')
jdk = vmLauncher.jdk()
vmArgs += mx.get_runtime_jvm_args(unittestDeps,
cp_prefix=prefixCp + coreCp,
jdk=jdk)
# suppress menubar and dock when running on Mac
vmArgs = prefixArgs + ['-Djava.awt.headless=true'] + vmArgs
if jdk.javaCompliance > '1.8':
# This is required to access jdk.internal.module.Modules for supporting
# the @AddExports annotation.
vmArgs = vmArgs + [
'--add-exports=java.base/jdk.internal.module=ALL-UNNAMED'
]
with open(testfile) as fp:
testclass = None
for l in fp:
if testclass:
testclass = None
break
testclass = l.rstrip()
if testclass:
# Execute Junit directly when one test is being run. This simplifies
# replaying the VM execution in a native debugger (e.g., gdb).
mainClassArgs = junit_args + [testclass]
else:
mainClassArgs = junit_args + ['@' + mx._cygpathU2W(testfile)]
config = (vmArgs, mainClass, mainClassArgs)
for p in _config_participants:
config = p(config)
vmLauncher.launcher(*config)
def contents(self, tool, exe):
# platform support
all_params = '"%*"' if mx.is_windows() else '"$@"'
_quote = _quote_windows if mx.is_windows() else pipes.quote
# build command line
java = mx.get_jdk().java
classpath_deps = [
dep for dep in self.subject.buildDependencies
if isinstance(dep, mx.ClasspathDependency)
]
extra_props = [
'-Dorg.graalvm.launcher.executablename="{}"'.format(exe)
]
main_class = self.subject.suite.toolchain._tool_to_main(tool)
jvm_args = [
_quote(arg) for arg in mx.get_runtime_jvm_args(classpath_deps)
]
command = [java] + jvm_args + extra_props + [main_class, all_params]
# create script
if mx.is_windows():
return "@echo off\n" + " ".join(command) + "\n"
else:
return "#!/usr/bin/env bash\n" + "exec " + " ".join(command) + "\n"
def store_jvm_args(self):
distributions = [dep.name for dep in self.subject.buildDependencies]
jvm_args = mx.get_runtime_jvm_args(distributions)
properties = []
while jvm_args:
arg = jvm_args.pop(0)
if arg == '-cp':
classpath = self.relativize(jvm_args.pop(0)).split(':')
else:
properties.append(self.relativize(arg))
sulong_libs = mx_subst.path_substitutions.substitute('-Dpolyglot.llvm.libraryPath=<path:SULONG_LIBS>')
properties.append(self.relativize(sulong_libs))
boot_dists = ['GRAAL_SDK', 'TRUFFLE_API', 'LAUNCHER_COMMON']
bootclasspath = self.relativize(mx.classpath(boot_dists)).split(':')
for jar in bootclasspath:
classpath.remove(jar)
with open(self.jvm_args_file, 'w') as f:
f.write('bootclasspath=\\\n' + ':\\\n'.join(bootclasspath) + '\n\n')
f.write('classpath=\\\n' + ':\\\n'.join(classpath) + '\n\n')
f.write('properties=(\n"' + '"\n"'.join(properties) + '"\n)\n')
def harness(unittestDeps, vmLauncher, vmArgs):
prefixArgs = ['-esa', '-ea']
if gc_after_test:
prefixArgs.append('-XX:-DisableExplicitGC')
with open(testfile) as fp:
testclasses = [l.rstrip() for l in fp.readlines()]
vmArgs += mx.get_runtime_jvm_args(unittestDeps,
cp_prefix=prefixCp + coreCp,
jdk=vmLauncher.jdk())
# suppress menubar and dock when running on Mac
vmArgs = prefixArgs + ['-Djava.awt.headless=true'] + vmArgs
# Execute Junit directly when one test is being run. This simplifies
# replaying the VM execution in a native debugger (e.g., gdb).
mainClassArgs = coreArgs + (testclasses if len(testclasses) == 1 else
['@' + mx._cygpathU2W(testfile)])
config = (vmArgs, mainClass, mainClassArgs)
for p in _config_participants:
config = p(config)
vmLauncher.launcher(*config)
def ruby_check_heap_dump(input_args, out=None):
print("mx ruby_check_heap_dump " + " ".join(input_args))
args = input_args
args.insert(0, "--experimental-options")
dists = ['TRUFFLERUBY', 'TRUFFLE_NFI', 'SULONG_NATIVE', 'TRUFFLERUBY-TEST']
vm_args, truffleruby_args = mx.extract_VM_args(args,
useDoubleDash=True,
defaultAllVMArgs=False)
vm_args += mx.get_runtime_jvm_args(dists)
# vm_args.append("-agentlib:jdwp=transport=dt_socket,server=y,address=8000,suspend=y")
vm_args.append("org.truffleruby.LeakTest")
out = mx.OutputCapture() if out is None else out
retval = mx.run_java(vm_args + truffleruby_args,
jdk=jdk,
nonZeroIsFatal=False,
out=out)
if retval == 0:
print("PASSED")
print(out.data)
elif os.environ.get("CI") and "--keep-dump" not in input_args:
# rerun once with heap dumping enabled
out = mx.OutputCapture()
ruby_check_heap_dump(["--keep-dump"] + input_args, out=out)
path = out.data.strip().partition("Dump file:")[2].strip()
if path:
save_path = os.path.join(root, "dumps", "leak_test")
try:
os.makedirs(save_path)
except OSError:
pass
dest = shutil.copy(path, save_path) # pylint: disable=assignment-from-no-return
print("Heapdump file kept in " + dest)
raise Exception("heap dump check failed")
else:
print("FAILED")
print(out.data)
raise Exception("heap dump check failed")
def testgen(args):
'''generate the expected output for unit tests, and All/Failing test classes'''
parser = ArgumentParser(prog='r testgen')
parser.add_argument('--tests', action='store', default=_all_unit_tests(), help='pattern to match test classes')
args = parser.parse_args(args)
# check we are in the home directory
if os.getcwd() != _fastr_suite.dir:
mx.abort('must run rtestgen from FastR home directory')
def need_version_check():
vardef = os.environ.has_key('FASTR_TESTGEN_GNUR')
varval = os.environ['FASTR_TESTGEN_GNUR'] if vardef else None
version_check = vardef and varval != 'internal'
if version_check:
rpath = join(varval, 'bin', 'R')
else:
rpath = None
return version_check, rpath
version_check, rpath = need_version_check()
if version_check:
# check the version of GnuR against FastR
try:
fastr_version = subprocess.check_output([mx.get_jdk().java, mx.get_runtime_jvm_args('com.oracle.truffle.r.runtime'), 'com.oracle.truffle.r.runtime.RVersionNumber'])
gnur_version = subprocess.check_output([rpath, '--version'])
if not gnur_version.startswith(fastr_version):
mx.abort('R version is incompatible with FastR, please update to ' + fastr_version)
except subprocess.CalledProcessError:
mx.abort('RVersionNumber.main failed')
# now just invoke junit with the appropriate options
mx.log("generating expected output for packages: ")
for pkg in args.tests.split(','):
mx.log(" " + str(pkg))
os.environ["TZDIR"] = "/usr/share/zoneinfo/"
_unset_conflicting_envs()
junit(['--tests', args.tests, '--gen-expected-output', '--gen-expected-quiet'])
def rbdiag(args):
'''Diagnoses FastR builtins
-v Verbose output including the list of unimplemented specializations
-n Ignore RNull as an argument type
-m Ignore RMissing as an argument type
--mnonly Uses the RMissing and RNull values as the only samples for the chimney-sweeping
--noSelfTest Does not perform the pipeline self-test using the generated samples as the intro to each chimney-sweeping. It has no effect when --mnonly is specified as the self-test is never performed in that case.
--sweep Performs the 'chimney-sweeping'. The sample combination selection method is determined automatically.
--sweep=lite Performs the 'chimney-sweeping'. The diagonal sample selection method is used.
--sweep=total Performs the 'chimney-sweeping'. The total sample selection method is used.
--matchLevel=same Outputs produced by FastR and GnuR must be same (default)
--matchLevel=error Outputs are considered matching if none or both outputs contain an error
--maxSweeps=N Sets the maximum number of sweeps
--outMaxLev=N Sets the maximum output detail level for report messages. Use 0 for the basic messages only.
If no builtin is specified, all registered builtins are diagnosed.
An external builtin is specified by the fully qualified name of its node class.
Examples:
mx rbdiag
mx rbdiag colSums colMeans -v
mx rbdiag scan -m -n
mx rbdiag colSums --sweep
mx rbdiag com.oracle.truffle.r.library.stats.Rnorm
'''
vmArgs = mx.get_runtime_jvm_args('com.oracle.truffle.r.nodes.test')
setREnvironment()
os.environ["FASTR_TESTGEN_GNUR"] = "internal"
# this should work for Linux and Mac:
os.environ["TZDIR"] = "/usr/share/zoneinfo/"
vmArgs += ['com.oracle.truffle.r.nodes.test.RBuiltinDiagnostics']
mx.run_java(vmArgs + args)
def rbdiag(args):
'''Diagnoses FastR builtins
-v Verbose output including the list of unimplemented specializations
-n Ignore RNull as an argument type
-m Ignore RMissing as an argument type
--mnonly Uses the RMissing and RNull values as the only samples for the chimney-sweeping
--noSelfTest Does not perform the pipeline self-test using the generated samples as the intro to each chimney-sweeping. It has no effect when --mnonly is specified as the self-test is never performed in that case.
--sweep Performs the 'chimney-sweeping'. The sample combination selection method is determined automatically.
--sweep=lite Performs the 'chimney-sweeping'. The diagonal sample selection method is used.
--sweep=total Performs the 'chimney-sweeping'. The total sample selection method is used.
--matchLevel=same Outputs produced by FastR and GnuR must be same (default)
--matchLevel=error Outputs are considered matching if none or both outputs contain an error
--maxSweeps=N Sets the maximum number of sweeps
--outMaxLev=N Sets the maximum output detail level for report messages. Use 0 for the basic messages only.
If no builtin is specified, all registered builtins are diagnosed.
An external builtin is specified by the fully qualified name of its node class.
Examples:
mx rbdiag
mx rbdiag colSums colMeans -v
mx rbdiag scan -m -n
mx rbdiag colSums --sweep
mx rbdiag com.oracle.truffle.r.library.stats.Rnorm
'''
vmArgs = mx.get_runtime_jvm_args('com.oracle.truffle.r.nodes.test')
setREnvironment()
os.environ["FASTR_TESTGEN_GNUR"] = "internal"
# this should work for Linux and Mac:
os.environ["TZDIR"] = "/usr/share/zoneinfo/"
vmArgs += ['com.oracle.truffle.r.nodes.test.RBuiltinDiagnostics']
mx.run_java(vmArgs + args)
def llirtestgen(args=None, out=None):
return mx.run_java(mx.get_runtime_jvm_args(["LLIR_TEST_GEN"]) + ["com.oracle.truffle.llvm.tests.llirtestgen.LLIRTestGen"] + args, out=out)
def getClasspathOptions():
"""gets the classpath of the Sulong distributions"""
return mx.get_runtime_jvm_args('SULONG')
def junit(args):
'''run ZipPy Junit tests'''
parser = ArgumentParser(prog='mx junit')
parser.add_argument(
'--tests',
action='store',
help=
'patterns to match test classes (specify multiple patterns using \',\')'
)
parser.add_argument('--J',
dest='vm_args',
action='append',
help='Java VM arguments (e.g. --J @-dsa)',
metavar='@<args>')
parser.add_argument('--jdk',
action='store',
help='JDK to use for the "java" command')
args = parser.parse_args(args)
vmArgs = ['-ea', '-esa']
# enable when necessary
# vmArgs += ['-Xss12m']
if args.vm_args:
vmArgs = vmArgs + mx_fastr.split_j_args(args.vm_args)
testfile = os.environ.get('MX_TESTFILE', None)
if testfile is None:
(_, testfile) = tempfile.mkstemp(".testclasses", "mx")
os.close(_)
if args.jdk:
jdk = mx.get_jdk(tag=args.jdk)
if not jdk:
mx.abort("JDK '" + args.jdk + "' not found!")
else:
tag = 'jvmci' if _mx_graal else None
jdk = mx.get_jdk(tag=tag)
candidates = []
for p in mx.projects(opt_limit_to_suite=True, limit_to_primary=True):
if not p.isJavaProject() or jdk.javaCompliance < p.javaCompliance:
continue
candidates += mx_unittest._find_classes_with_annotations(
p, None, ['@Test']).keys()
tests = [] if args.tests is None else [
name for name in args.tests.split(',')
]
classes = []
if len(tests) == 0:
classes = candidates
else:
for test in tests:
exists = False
for candidate in candidates:
if test in candidate:
exists = True
classes.append(candidate)
if not exists:
mx.warn('no tests matched by substring "' + test + '"')
vmArgs += mx.get_runtime_jvm_args(['ZIPPY', 'ZIPPY_UNIT_TESTS'], jdk=jdk)
if len(classes) != 0:
if len(classes) == 1:
testClassArgs = ['--testclass', classes[0]]
else:
with open(testfile, 'w') as f:
for c in classes:
f.write(c + '\n')
testClassArgs = ['--testsfile', testfile]
junitArgs = ['edu.uci.python.test.ZipPyJUnitRunner'] + testClassArgs
rc = mx.run_java(vmArgs + junitArgs, nonZeroIsFatal=False, jdk=jdk)
return rc
else:
return 0
def do_run_python(args, extra_vm_args=None, env=None, jdk=None, **kwargs):
if not env:
env = os.environ
check_vm_env = env.get('GRAALPYTHON_MUST_USE_GRAAL', False)
if check_vm_env:
if check_vm_env == '1':
check_vm(must_be_jvmci=True)
elif check_vm_env == '0':
check_vm()
dists = ['GRAALPYTHON']
vm_args, graalpython_args = mx.extract_VM_args(args,
useDoubleDash=True,
defaultAllVMArgs=False)
internal_graalpython_args, graalpython_args, additional_dists = _extract_graalpython_internal_options(
graalpython_args)
dists += additional_dists
if '--python.WithJavaStacktrace' not in graalpython_args:
graalpython_args.insert(0, '--python.WithJavaStacktrace')
if _sulong:
dists.append('SULONG')
if mx.suite("sulong-managed", fatalIfMissing=False):
dists.append('SULONG_MANAGED')
vm_args.append(
mx_subst.path_substitutions.substitute(
'-Dpolyglot.llvm.libraryPath=<path:SULONG_LIBS>:<path:SULONG_MANAGED_LIBS>'
))
else:
vm_args.append(
mx_subst.path_substitutions.substitute(
'-Dpolyglot.llvm.libraryPath=<path:SULONG_LIBS>'))
# Try eagerly to include tools on Tim's computer
if not mx.suite("/tools", fatalIfMissing=False):
def _is_user(user, home=None):
if home:
return os.environ.get("USER") == user and os.environ.get(home)
return os.environ.get("USER") == user
if _is_user("tim",
"MAGLEV_HOME") or _is_user("cbasca") or _is_user("fa"):
_suite.import_suite("tools",
version=None,
urlinfos=None,
in_subdir=True)
dists.append('CHROMEINSPECTOR')
if _sulong:
vm_args.append("-Dpolyglot.llvm.enableLVI=true")
vm_args += mx.get_runtime_jvm_args(dists, jdk=jdk)
vm_args += internal_graalpython_args
if not jdk:
jdk = get_jdk()
# default: assertion checking is enabled
if extra_vm_args is None or '-da' not in extra_vm_args:
vm_args += ['-ea', '-esa']
if extra_vm_args:
vm_args += extra_vm_args
vm_args.append("com.oracle.graal.python.shell.GraalPythonMain")
return mx.run_java(vm_args + graalpython_args, jdk=jdk, **kwargs)
def run_grid_server(args, **kwargs):
vmArgs = mx.get_runtime_jvm_args(['GRID_DEVICE_REMOTE_SERVER'], jdk=get_default_jdk())
vmArgs.append('com.oracle.truffle.r.library.fastrGrid.device.remote.server.RemoteDeviceServer')
return mx.run_java(vmArgs + args, jdk=get_default_jdk(), **kwargs)
def getClasspathOptions():
"""gets the classpath of the Sulong distributions"""
return mx.get_runtime_jvm_args(['SULONG', 'SULONG_LAUNCHER'])
def getClasspathOptions(extra_dists=None):
"""gets the classpath of the Sulong distributions"""
return mx.get_runtime_jvm_args(
['SULONG', 'SULONG_LAUNCHER', 'TRUFFLE_NFI'] + (extra_dists or []))
def extract_bitcode(args=None, out=None):
"""Extract embedded LLVM bitcode from object files"""
return mx.run_java(
mx.get_runtime_jvm_args(["com.oracle.truffle.llvm.tools"]) +
["com.oracle.truffle.llvm.tools.ExtractBitcode"] + args,
out=out)
def extract_bitcode(args=None, out=None):
return mx.run_java(
mx.get_runtime_jvm_args(["com.oracle.truffle.llvm.tools"]) +
["com.oracle.truffle.llvm.tools.ExtractBitcode"] + args,
out=out)
def junit(args, harness, parser=None, jdk_default=None):
"""run Junit tests"""
suppliedParser = parser is not None
parser = parser if suppliedParser else ArgumentParser(prog='mx junit')
parser.add_argument('--tests', action='store', help='pattern to match test classes')
parser.add_argument('--J', dest='vm_args', action='append', help='target VM arguments (e.g. --J @-dsa)', metavar='@<args>')
parser.add_argument('--jdk', action='store', help='jdk to use')
if suppliedParser:
parser.add_argument('remainder', nargs=REMAINDER, metavar='...')
args = parser.parse_args(args)
vmArgs = ['-ea', '-esa']
if args.vm_args:
vmArgs = vmArgs + mx_fastr.split_j_args(args.vm_args)
testfile = os.environ.get('MX_TESTFILE', None)
if testfile is None:
(_, testfile) = tempfile.mkstemp(".testclasses", "mx")
os.close(_)
candidates = []
if args.jdk:
jdk = mx.get_jdk(tag=args.jdk)
if not jdk:
mx.abort("jdk '" + args.jdk + "' not found")
else:
if not jdk_default:
jdk = mx.get_jdk()
else:
jdk = jdk_default
for p in mx.projects(opt_limit_to_suite=True):
if not p.isJavaProject() or jdk.javaCompliance < p.javaCompliance:
continue
candidates += _find_classes_with_annotations(p, None, ['@Test']).keys()
tests = [] if args.tests is None else [name for name in args.tests.split(',')]
classes = []
if len(tests) == 0:
classes = candidates
else:
for t in tests:
found = False
for c in candidates:
if t in c:
found = True
classes.append(c)
if not found:
mx.warn('no tests matched by substring "' + t + '"')
vmArgs += mx.get_runtime_jvm_args([pcp.name for pcp in mx.projects(opt_limit_to_suite=True) if pcp.isJavaProject() and pcp.javaCompliance <= jdk.javaCompliance], jdk=jdk)
if len(classes) != 0:
if len(classes) == 1:
testClassArgs = ['--testclass', classes[0]]
else:
with open(testfile, 'w') as f:
for c in classes:
f.write(c + '\n')
testClassArgs = ['--testsfile', testfile]
junitArgs = ['com.oracle.truffle.r.test.FastRJUnitWrapper'] + testClassArgs
rc = harness(args, vmArgs, jdk, junitArgs)
return rc
else:
return 0
