def wikidoc(args):
"""generate Confluence Wiki format for package-info.java files"""
# Ensure the wiki doclet is up to date
mx.build(['--projects', 'com.oracle.max.tools'])
# the WikiDoclet cannot see the -classpath argument passed to javadoc so we pass the
# full list of projects as an explicit argument, thereby enabling it to map classes
# to projects, which is needed to generate Wiki links to the source code.
# There is no virtue in running the doclet on dependent projects as there are
# no generated links between Wiki pages
toolsDir = mx.project('com.oracle.max.tools').output_dir()
baseDir = mx.project('com.oracle.max.base').output_dir()
dp = os.pathsep.join([toolsDir, baseDir])
project_list = ','.join(p.name for p in mx.sorted_deps())
for a in [
'-docletpath', dp, '-doclet',
'com.oracle.max.tools.javadoc.wiki.WikiDoclet', '-projects',
project_list
]:
args.append('--arg')
args.append('@' + a)
mx.javadoc(args,
parser=ArgumentParser('mx wikidoc'),
docDir='wikidoc',
includeDeps=False,
stdDoclet=False)
def validate_parser(grammar_project,
grammar_path,
create_command,
args=None,
out=None):
def read_file(path):
with open(path, "r") as f:
return f.readlines()
parser_path = grammar_path.replace(".g4", "Parser.java")
lexer_path = grammar_path.replace(".g4", "Lexer.java")
parser = mx.project(grammar_project).source_dirs()[0] + "/" + parser_path
lexer = mx.project(grammar_project).source_dirs()[0] + "/" + lexer_path
parser_before = read_file(parser)
lexer_before = read_file(lexer)
create_command([], out)
parser_after = read_file(parser)
lexer_after = read_file(lexer)
if (parser_before != parser_after or lexer_before != lexer_after):
with open(parser, "w") as f:
f.writelines(parser_before)
with open(lexer, "w") as f:
f.writelines(lexer_before)
mx.abort(
"Parser generated from " + grammar_path +
" does not match content of " + parser_path + " or " + lexer_path +
"." +
" Make sure the grammar files are up to date with the generated code. You can regenerate the generated code using mx."
)
def _espresso_gate_runner(args, tasks):
# Jackpot configuration is inherited from Truffle.
with Task('Jackpot', tasks, tags=[EspressoTags.jackpot]) as t:
if t:
jackpot(['--fail-on-warnings'], suite=None, nonZeroIsFatal=True)
with Task('Espresso: GraalVM dist names', tasks, tags=['names']) as t:
if t:
mx_sdk_vm.verify_graalvm_configs(suites=['espresso'])
mokapot_header_gate_name = 'Verify consistency of mokapot headers'
with Task(mokapot_header_gate_name, tasks, tags=[EspressoTags.verify]) as t:
if t:
import mx_sdk_vm_impl
run_instructions = "$ mx --dynamicimports=/substratevm --native-images=lib:espresso gate --all-suites --task '{}'".format(mokapot_header_gate_name)
if mx_sdk_vm_impl._skip_libraries(espresso_library_config):
mx.abort("""\
The registration of the Espresso library ('lib:espresso') is skipped. Please run this gate as follows:
{}""".format(run_instructions))
errors = False
mokapot_dir = join(mx.project('com.oracle.truffle.espresso.mokapot').dir, 'include')
libespresso_dir = mx.project(mx_sdk_vm_impl.GraalVmNativeImage.project_name(espresso_library_config)).get_output_root()
for header in ['libespresso_dynamic.h', 'graal_isolate_dynamic.h']:
committed_header = join(mokapot_dir, header)
if not mx.exists(committed_header):
mx.abort("Cannot locate '{}'. Was the file moved or renamed?".format(committed_header))
generated_header = join(libespresso_dir, header)
if not mx.exists(generated_header):
mx.abort("Cannot locate '{}'. Did you forget to build? Example:\n'mx --dynamicimports=/substratevm --native-images=lib:espresso build'".format(generated_header))
committed_header_copyright = []
with open(committed_header, 'r') as committed_header_file:
for line in committed_header_file.readlines():
if line == '/*\n' or line.startswith(' *') or line == '*/\n':
committed_header_copyright.append(line)
else:
break
with open(generated_header, 'r') as generated_header_file:
generated_header_lines = []
for line in generated_header_file.readlines():
# Ignore definitions that are not needed for Espresso
if not line.startswith("typedef") or "(*Espresso_" in line or "__graal" in line or "(*graal_" in line:
generated_header_lines.append(line)
else:
newline = generated_header_lines.pop() # Remove newline before ignored declaration
assert newline == "\n"
errors = errors or mx.update_file(committed_header, ''.join(committed_header_copyright + generated_header_lines), showDiff=True)
if errors:
mx.abort("""\
One or more header files in the include dir of the mokapot project ('{committed}/') do not match those generated by Native Image ('{generated}/').
To fix the issue, run this gate locally:
{instructions}
And adapt the code to the modified headers in '{committed}'.
""".format(committed=os.path.relpath(mokapot_dir, _suite.vc_dir), generated=os.path.relpath(libespresso_dir, _suite.vc_dir), instructions=run_instructions))
def rbcheck(args):
'''check FastR builtins against GnuR'''
parser = ArgumentParser(prog='mx rbcheck')
parser.add_argument('--check-internal', action='store_const', const='--check-internal', help='check .Internal functions')
parser.add_argument('--unknown-to-gnur', action='store_const', const='--unknown-to-gnur', help='list builtins not in GnuR FUNCTAB')
parser.add_argument('--todo', action='store_const', const='--todo', help='show unimplemented')
parser.add_argument('--no-eval-args', action='store_const', const='--no-eval-args', help='list functions that do not evaluate their args')
parser.add_argument('--visibility', action='store_const', const='--visibility', help='list visibility specification')
parser.add_argument('--printGnuRFunctions', action='store', help='ask GnuR to "print" value of functions')
parser.add_argument('--packageBase', action='store', help='directory to be recursively scanned for R sources (used to get frequencies for builtins)')
parser.add_argument('--interactive', action='store_const', const='--interactive', help='interactive querying of the word frequencies')
args = parser.parse_args(args)
class_map = mx.project('com.oracle.truffle.r.nodes').find_classes_with_matching_source_line(None, lambda line: "@RBuiltin" in line, True)
classes = []
for className, path in class_map.iteritems():
classNameX = className.split("$")[0] if '$' in className else className
if not classNameX.endswith('Factory'):
classes.append([className, path])
class_map = mx.project('com.oracle.truffle.r.nodes.builtin').find_classes_with_matching_source_line(None, lambda line: "@RBuiltin" in line, True)
for className, path in class_map.iteritems():
classNameX = className.split("$")[0] if '$' in className else className
if not classNameX.endswith('Factory'):
classes.append([className, path])
(_, testfile) = tempfile.mkstemp(".classes", "mx")
os.close(_)
with open(testfile, 'w') as f:
for c in classes:
f.write(c[0] + ',' + c[1][0] + '\n')
analyzeArgs = []
if args.check_internal:
analyzeArgs.append(args.check_internal)
if args.unknown_to_gnur:
analyzeArgs.append(args.unknown_to_gnur)
if args.todo:
analyzeArgs.append(args.todo)
if args.no_eval_args:
analyzeArgs.append(args.no_eval_args)
if args.visibility:
analyzeArgs.append(args.visibility)
if args.interactive:
analyzeArgs.append(args.interactive)
if args.printGnuRFunctions:
analyzeArgs.append('--printGnuRFunctions')
analyzeArgs.append(args.printGnuRFunctions)
if args.packageBase:
analyzeArgs.append('--packageBase')
analyzeArgs.append(args.packageBase)
analyzeArgs.append(testfile)
cp = mx.classpath('com.oracle.truffle.r.test')
mx.run_java(['-cp', cp, 'com.oracle.truffle.r.test.tools.AnalyzeRBuiltin'] + analyzeArgs)
def jttgen(args):
"""(re)generate harness and run scheme for the JavaTester tests
Run the JavaTester to update the JavaTesterRunScheme.java and JavaTesterTests.java
files in the com.sun.max.vm.jtrun.all package."""
testDirs = [join(mx.project('com.oracle.max.vm.tests').dir, 'src'), join(mx.project('com.oracle.max.tests').dir, 'src')]
tests = []
for testDir in testDirs:
for name in os.listdir(join(testDir, 'jtt')):
if name != 'hotspot' and name != 'fail':
tests.append(join(testDir, 'jtt', name))
return mx.run_java(['-cp', mx.classpath('com.oracle.max.vm.tests'), 'test.com.sun.max.vm.compiler.JavaTester',
'-scenario=target', '-run-scheme-package=all', '-native-tests'] + tests)
def jttgen(args):
"""(re)generate harness and run scheme for the JavaTester tests
Run the JavaTester to update the JavaTesterRunScheme.java and JavaTesterTests.java
files in the com.sun.max.vm.jtrun.all package."""
testDirs = [join(mx.project('com.oracle.max.vm.tests').dir, 'src'), join(mx.project('com.oracle.max.tests').dir, 'src')]
tests = []
for testDir in testDirs:
for name in os.listdir(join(testDir, 'jtt')):
if name != 'hotspot' and name != 'fail':
tests.append(join(testDir, 'jtt', name))
return mx.run_java(['-cp', mx.classpath('com.oracle.max.vm.tests'), 'test.com.sun.max.vm.compiler.JavaTester',
'-scenario=target', '-run-scheme-package=all', '-native-tests'] + tests)
def checkheaders(args):
"""check Java source headers against any required pattern"""
failures = {}
for p in mx.projects():
if not p.isJavaProject():
continue
csConfig = join(mx.project(p.checkstyleProj).dir, '.checkstyle_checks.xml')
if not exists(csConfig):
mx.log('Cannot check headers for ' + p.name + ' - ' + csConfig + ' does not exist')
continue
dom = xml.dom.minidom.parse(csConfig)
for module in dom.getElementsByTagName('module'):
if module.getAttribute('name') == 'RegexpHeader':
for prop in module.getElementsByTagName('property'):
if prop.getAttribute('name') == 'header':
value = prop.getAttribute('value')
matcher = re.compile(value, re.MULTILINE)
for sourceDir in p.source_dirs():
for root, _, files in os.walk(sourceDir):
for name in files:
if name.endswith('.java') and name != 'package-info.java':
f = join(root, name)
with open(f) as fp:
content = fp.read()
if not matcher.match(content):
failures[f] = csConfig
for n, v in failures.iteritems():
mx.log('{0}: header does not match RegexpHeader defined in {1}'.format(n, v))
return len(failures)
def checkheaders(args):
"""check Java source headers against any required pattern"""
failures = {}
for p in mx.projects():
if not p.isJavaProject():
continue
csConfig = join(mx.project(p.checkstyleProj).dir, ".checkstyle_checks.xml")
if not exists(csConfig):
mx.log("Cannot check headers for " + p.name + " - " + csConfig + " does not exist")
continue
dom = xml.dom.minidom.parse(csConfig)
for module in dom.getElementsByTagName("module"):
if module.getAttribute("name") == "RegexpHeader":
for prop in module.getElementsByTagName("property"):
if prop.getAttribute("name") == "header":
value = prop.getAttribute("value")
matcher = re.compile(value, re.MULTILINE)
for sourceDir in p.source_dirs():
for root, _, files in os.walk(sourceDir):
for name in files:
if name.endswith(".java") and name != "package-info.java":
f = join(root, name)
with open(f) as fp:
content = fp.read()
if not matcher.match(content):
failures[f] = csConfig
for n, v in failures.iteritems():
mx.log("{0}: header does not match RegexpHeader defined in {1}".format(n, v))
return len(failures)
def jaotc_test(args):
"""run (acceptance) tests for the AOT compiler (jaotc)"""
all_tests = ['HelloWorld', 'java.base', 'javac']
parser = ArgumentParser(prog='mx jaotc-test')
parser.add_argument("--list", default=None, action="store_true", help="Print the list of available jaotc tests.")
parser.add_argument('tests', help='tests to run (omit to run all tests)', nargs=ZERO_OR_MORE)
args = parser.parse_args(args)
if args.list:
print "The following jaotc tests are available:\n"
for name in all_tests:
print " " + name
return
tests = args.tests or all_tests
for test in tests:
mx.log('Testing `{}`'.format(test))
if test == 'HelloWorld':
test_class(
classpath=mx.classpath('JAOTC_TEST'),
main_class='jdk.tools.jaotc.test.HelloWorld'
)
elif test == 'javac':
test_javac('jdk.tools.jaotc')
elif test == 'java.base':
test_modules(
classpath=mx.project('jdk.tools.jaotc.test').output_dir(),
main_class='jdk.tools.jaotc.test.HelloWorld',
modules=['java.base']
)
else:
mx.abort('Unknown jaotc test: {}'.format(test))
def _alldeps(args):
parser = ArgumentParser(prog='mx mxt-alldeps')
parser.add_argument('--kind',
action='store',
help='project, dist or library',
default='project')
parser.add_argument('--name',
action='store',
help='name of entity',
required=True)
parser.add_argument('--includeLibs',
action='store_true',
help='includeLibs')
parser.add_argument('--ignoreSelf', action='store_true', help='ignoreSelf')
args = parser.parse_args(args)
entity = None
if args.kind == 'project':
entity = mx.project(args.name)
elif args.kind == 'library':
entity = mx.library(args.name)
elif args.kind == 'dist':
entity = mx.distribution(args.name)
else:
mx.abort('illegal kind: ' + args.kind)
in_deps = []
deps = entity.all_deps(in_deps,
includeLibs=args.includeLibs,
includeSelf=not args.ignoreSelf)
print('alldeps:')
for d in deps:
print(d.__class__.__name__, ":", d.name)
def create_parser(grammar_project,
grammar_package,
grammar_name,
copyright_template,
args=None,
out=None):
"""create the DSL expression parser using antlr"""
grammar_dir = os.path.join(
mx.project(grammar_project).source_dirs()[0],
*grammar_package.split(".")) + os.path.sep
mx.run_java(
mx.get_runtime_jvm_args(['ANTLR4_COMPLETE']) +
["org.antlr.v4.Tool", "-package", grammar_package, "-no-listener"] +
args + [grammar_dir + grammar_name + ".g4"],
out=out)
for filename in [
grammar_dir + grammar_name + "Lexer.java",
grammar_dir + grammar_name + "Parser.java"
]:
with open(filename, 'r') as content_file:
content = content_file.read()
# remove first line
content = "\n".join(content.split("\n")[1:])
# modify SuppressWarnings to remove useless entries
content = PTRN_SUPPRESS_WARNINGS.sub('@SuppressWarnings("all")',
content)
# remove useless casts
content = PTRN_LOCALCTXT_CAST.sub('_localctx', content)
content = PTRN_TOKEN_CAST.sub('_errHandler.recoverInline(this)',
content)
# add copyright header
content = copyright_template.format(content)
with open(filename, 'w') as content_file:
content_file.write(content)
def getSearchPathOption(lib_args=None):
if lib_args is None:
lib_args = ['-lgmp', '-lgfortran']
lib_names = []
lib_aliases = {
'-lc': ['libc.so.6', 'libc.dylib'],
'-lstdc++': ['libstdc++.so.6', 'libstdc++.6.dylib'],
'-lgmp': ['libgmp.so.10', 'libgmp.10.dylib'],
'-lgfortran': ['libgfortran.so.3', 'libgfortran.3.dylib'],
'-lpcre': ['libpcre.so.3', 'libpcre.dylib']
}
osStr = mx.get_os()
index = {'linux': 0, 'darwin': 1}[mx.get_os()]
if index is None:
mx.log_error("{0} not supported!".format(osStr))
for lib_arg in ['-lc', '-lstdc++'] + lib_args:
if lib_arg in lib_aliases:
lib_arg = lib_aliases[lib_arg][index]
else:
lib_arg = lib_arg[2:]
lib_names.append(lib_arg)
libpath = join(
mx.project('com.oracle.truffle.llvm.test.native').getOutput(), 'bin')
for path, _, files in os.walk(libpath):
for f in files:
if f.endswith('.so'):
lib_names.append(join(path, f))
return '-Dsulong.DynamicNativeLibraryPath=' + ':'.join(lib_names)
def _graal_nodejs_post_gate_runner(args, tasks):
_setEnvVar('NODE_INTERNAL_ERROR_CHECK', 'true')
with Task('UnitTests', tasks, tags=[GraalNodeJsTags.allTests, GraalNodeJsTags.unitTests]) as t:
if t:
commonArgs = ['-ea']
unitTestDir = join('test', 'graal')
mx.run(['rm', '-rf', 'node_modules', 'build'], cwd=unitTestDir)
npm(['--scripts-prepend-node-path=auto', 'install', '--nodedir=' + _suite.dir] + commonArgs, cwd=unitTestDir)
npm(['--scripts-prepend-node-path=auto', 'test'] + commonArgs, cwd=unitTestDir)
npm(['--scripts-prepend-node-path=auto', 'test', '-Dtruffle.js.NashornJavaInterop=true'] + commonArgs, cwd=unitTestDir)
with Task('TestNpm', tasks, tags=[GraalNodeJsTags.allTests]) as t:
if t:
tmpdir = tempfile.mkdtemp()
try:
npm(['--scripts-prepend-node-path=auto', 'install', '--nodedir=' + _suite.dir, 'microtime'], cwd=tmpdir)
node(['-e', 'print(require("microtime").now());'], cwd=tmpdir)
finally:
shutil.rmtree(tmpdir)
with Task('JniProfilerTests', tasks, tags=[GraalNodeJsTags.allTests, GraalNodeJsTags.jniProfilerTests]) as t:
if t:
commonArgs = ['-ea']
unitTestDir = join(mx.project('com.oracle.truffle.trufflenode.jniboundaryprofiler').dir, 'tests')
mx.run(['rm', '-rf', 'node_modules', 'build'], cwd=unitTestDir)
npm(['--scripts-prepend-node-path=auto', 'install', '--nodedir=' + _suite.dir] + commonArgs, cwd=unitTestDir)
node(['-profile-native-boundary', '-Dtruffle.js.NashornJavaInterop=true', 'test.js'] + commonArgs, cwd=unitTestDir)
def jaotc_test(args):
"""run (acceptance) tests for the AOT compiler (jaotc)"""
_check_jaotc_support()
all_tests = ['HelloWorld', 'java.base', 'javac']
parser = ArgumentParser(prog='mx jaotc-test')
parser.add_argument("--list",
default=None,
action="store_true",
help="Print the list of available jaotc tests.")
parser.add_argument('tests',
help='tests to run (omit to run all tests)',
nargs=ZERO_OR_MORE)
args = parser.parse_args(args)
if args.list:
print("The following jaotc tests are available:\n")
for name in all_tests:
print(" " + name)
return
tests = args.tests or all_tests
for test in tests:
mx.log('Testing `{}`'.format(test))
if test == 'HelloWorld':
test_class(classpath=mx.classpath('JAOTC_TEST'),
main_class='jdk.tools.jaotc.test.HelloWorld')
elif test == 'javac':
test_javac('jdk.tools.jaotc')
elif test == 'java.base':
test_modules(
classpath=mx.project('jdk.tools.jaotc.test').output_dir(),
main_class='jdk.tools.jaotc.test.HelloWorld',
modules=['java.base'])
else:
mx.abort('Unknown jaotc test: {}'.format(test))
def checkheaders(args):
"""check Java source headers against any required pattern"""
failures = {}
for p in mx.projects():
if not p.isJavaProject():
continue
csConfig = join(mx.project(p.checkstyleProj).dir, '.checkstyle_checks.xml')
if not exists(csConfig):
mx.log('Cannot check headers for ' + p.name + ' - ' + csConfig + ' does not exist')
continue
dom = xml.dom.minidom.parse(csConfig)
for module in dom.getElementsByTagName('module'):
if module.getAttribute('name') == 'RegexpHeader':
for prop in module.getElementsByTagName('property'):
if prop.getAttribute('name') == 'header':
value = prop.getAttribute('value')
matcher = re.compile(value, re.MULTILINE)
for sourceDir in p.source_dirs():
for root, _, files in os.walk(sourceDir):
for name in files:
if name.endswith('.java') and name != 'package-info.java':
f = join(root, name)
with open(f) as fp:
content = fp.read()
if not matcher.match(content):
failures[f] = csConfig
for n, v in failures.iteritems():
mx.log('{0}: header does not match RegexpHeader defined in {1}'.format(n, v))
return len(failures)
def mx_post_parse_cmd_line(opts):
mx_fastr_dists.mx_post_parse_cmd_line(opts)
if mx.suite("sulong", fatalIfMissing=False) and not _fastr_suite.isBinarySuite():
# native.recommended runs FastR, it already has a build dependency to the FASTR distribution
# if we are running with sulong we also need the SULONG distribution
rec = mx.project('com.oracle.truffle.r.native.recommended')
rec.buildDependencies += [mx.distribution('SULONG')]
def _graal_nodejs_post_gate_runner(args, tasks):
_setEnvVar('NODE_INTERNAL_ERROR_CHECK', 'true')
with Task('UnitTests', tasks, tags=[GraalNodeJsTags.allTests, GraalNodeJsTags.unitTests]) as t:
if t:
_setEnvVar('NODE_JVM_CLASSPATH', mx.distribution('graal-js:TRUFFLE_JS_TESTS').path)
commonArgs = ['-ea', '-esa']
unitTestDir = join('test', 'graal')
mx.run(['rm', '-rf', 'node_modules', 'build'], cwd=unitTestDir)
npm(['--scripts-prepend-node-path=auto', 'install', '--nodedir=' + _suite.dir] + commonArgs, cwd=unitTestDir)
npm(['--scripts-prepend-node-path=auto', 'test'] + commonArgs, cwd=unitTestDir)
with Task('TestNpm', tasks, tags=[GraalNodeJsTags.allTests]) as t:
if t:
tmpdir = tempfile.mkdtemp()
try:
npm(['init', '-y'], cwd=tmpdir)
npm(['--scripts-prepend-node-path=auto', 'install', '--nodedir=' + _suite.dir, '--build-from-source', 'microtime'], cwd=tmpdir)
node(['-e', 'console.log(require("microtime").now());'], cwd=tmpdir)
finally:
mx.rmtree(tmpdir)
with Task('JniProfilerTests', tasks, tags=[GraalNodeJsTags.allTests, GraalNodeJsTags.jniProfilerTests]) as t:
if t:
commonArgs = ['-ea', '-esa']
unitTestDir = join(mx.project('com.oracle.truffle.trufflenode.jniboundaryprofiler').dir, 'tests')
mx.run(['rm', '-rf', 'node_modules', 'build'], cwd=unitTestDir)
npm(['--scripts-prepend-node-path=auto', 'install', '--nodedir=' + _suite.dir] + commonArgs, cwd=unitTestDir)
node(['-profile-native-boundary', 'test.js'] + commonArgs, cwd=unitTestDir)
def gnu_rtests(args, env=None):
'''
run tests of the internally built GNU R under tests subdirectory
'''
os.chdir(_fastr_suite.dir) # Packages install fails otherwise
# mx_fastr_pkgs.installpkgs(['--pkg-pattern', '^MASS$']) # required by tests/Examples/base-Ex.R
np = mx.project('com.oracle.truffle.r.native')
tst = join(np.dir, 'gnur', 'tests')
tstsrc = join(tst, 'src')
tstlog = join(tst, 'log')
shutil.rmtree(tstlog, True)
os.mkdir(tstlog)
diffname = join(tstlog, 'all.diff')
diff = open(diffname, 'a')
try:
for subd in ['Examples', '']:
logd = join(tstlog, subd)
if subd != '':
os.mkdir(logd)
os.chdir(logd)
srcd = join(tstsrc, subd)
for f in sorted(os.listdir(srcd)):
if f.endswith('.R'):
print 'Running {} explicitly by FastR CMD BATCH ...'.format(
f)
mx.run(
[r_path(), '--vanilla', 'CMD', 'BATCH',
join(srcd, f)] + args,
nonZeroIsFatal=False,
env=env,
timeout=90)
outf = f + 'out'
if os.path.isfile(outf):
outff = outf + '.fastr'
os.rename(outf, outff)
print 'Running {} explicitly by GnuR CMD BATCH ...'.format(
f)
mx.run([
join(_gnur_path(), 'R'), '--vanilla', 'CMD',
'BATCH',
join(srcd, f)
] + args,
nonZeroIsFatal=False,
env=env,
timeout=90)
if os.path.isfile(outf):
outfg = outf + '.gnur'
os.rename(outf, outfg)
diff.write('\nRdiff {} {}:\n'.format(outfg, outff))
diff.flush()
subprocess.Popen(
[r_path(), 'CMD', 'Rdiff', outfg, outff],
stdout=diff,
stderr=diff,
shell=False)
diff.flush()
diff.close()
print 'FastR to GnuR diff was written to {}'.format(diffname)
finally:
shutil.rmtree(join(_fastr_suite.dir, 'deparse'), True)
def _unittest(args, annotations, prefixCp="", blacklist=None, whitelist=None, verbose=False, fail_fast=False, enable_timing=False, regex=None, color=False, eager_stacktrace=False, gc_after_test=False, suite=None):
testfile = os.environ.get('MX_TESTFILE', None)
if testfile is None:
(_, testfile) = tempfile.mkstemp(".testclasses", "mxtool")
os.close(_)
mainClass = 'com.oracle.mxtool.junit.MxJUnitWrapper'
if not exists(join(mx.project('com.oracle.mxtool.junit').output_dir(), mainClass.replace('.', os.sep) + '.class')):
mx.build(['--only', 'com.oracle.mxtool.junit'])
coreCp = mx.classpath(['com.oracle.mxtool.junit'])
coreArgs = []
if verbose:
coreArgs.append('-JUnitVerbose')
if fail_fast:
coreArgs.append('-JUnitFailFast')
if enable_timing:
coreArgs.append('-JUnitEnableTiming')
if color:
coreArgs.append('-JUnitColor')
if eager_stacktrace:
coreArgs.append('-JUnitEagerStackTrace')
if gc_after_test:
coreArgs.append('-JUnitGCAfterTest')
def harness(unittestCp, vmLauncher, vmArgs):
prefixArgs = ['-esa', '-ea']
if gc_after_test:
prefixArgs.append('-XX:-DisableExplicitGC')
with open(testfile) as fp:
testclasses = [l.rstrip() for l in fp.readlines()]
cp = prefixCp + coreCp + os.pathsep + unittestCp
# suppress menubar and dock when running on Mac
vmArgs = prefixArgs + ['-Djava.awt.headless=true'] + vmArgs + ['-cp', mx._separatedCygpathU2W(cp)]
# Execute Junit directly when one test is being run. This simplifies
# replaying the VM execution in a native debugger (e.g., gdb).
mainClassArgs = coreArgs + (testclasses if len(testclasses) == 1 else ['@' + mx._cygpathU2W(testfile)])
config = (vmArgs, mainClass, mainClassArgs)
for p in _config_participants:
config = p(config)
vmLauncher.launcher(*config)
vmLauncher = _vm_launcher
if vmLauncher is None:
jdk = mx.get_jdk()
def _run_vm(vmArgs, mainClass, mainClassArgs):
mx.run_java(vmArgs + [mainClass] + mainClassArgs, jdk=jdk)
vmLauncher = _VMLauncher('default VM launcher', _run_vm, jdk)
try:
_run_tests(args, harness, vmLauncher, annotations, testfile, blacklist, whitelist, regex, mx.suite(suite) if suite else None)
finally:
if os.environ.get('MX_TESTFILE') is None:
os.remove(testfile)
def mx_post_parse_cmd_line(opts):
mx_fastr_dists.mx_post_parse_cmd_line(opts)
mx_subst.results_substitutions.register_no_arg('graalvm_version', mx.suite('sdk').release_version())
if mx.suite("sulong", fatalIfMissing=False) and not _fastr_suite.isBinarySuite():
# native.recommended runs FastR, it already has a build dependency to the FASTR distribution
# if we are running with sulong we also need the SULONG_NATIVE distribution
rec = mx.project('com.oracle.truffle.r.native.recommended')
rec.buildDependencies += [mx.distribution('SULONG_NATIVE')]
def _ap(args):
parser = ArgumentParser(prog='mx mxt-proj-ap-path')
parser.add_argument('--project', action='store', help='name of entity')
args = parser.parse_args(args)
project = mx.project(args.project)
result = project.annotation_processors_path()
print 'annotation_processors_path for: ', args.project
comps = result.split(':')
for comp in comps:
print comp
def _ap(args):
parser = ArgumentParser(prog='mx mxt-proj-ap-path')
parser.add_argument('--project', action='store', help='name of entity')
args = parser.parse_args(args)
project = mx.project(args.project)
result = project.annotation_processors_path()
print('annotation_processors_path for: ', args.project)
comps = result.split(':')
for comp in comps:
print(comp)
def getBitcodeLibrariesOption(*args):
libraries = []
if 'SULONG_NO_LIBRARY' not in os.environ:
libpath = join(
mx.project('com.oracle.truffle.llvm.libraries').getOutput(), 'bin')
for path, _, files in os.walk(libpath):
for f in files:
if f.endswith('.bc'):
libraries.append(join(path, f))
return ['-Dsulong.DynamicBitcodeLibraries=' +
':'.join(libraries)] if libraries else []
def wikidoc(args):
"""generate Confluence Wiki format for package-info.java files"""
# Ensure the wiki doclet is up to date
mx.build(['--projects', 'com.oracle.max.tools'])
# the WikiDoclet cannot see the -classpath argument passed to javadoc so we pass the
# full list of projects as an explicit argument, thereby enabling it to map classes
# to projects, which is needed to generate Wiki links to the source code.
# There is no virtue in running the doclet on dependent projects as there are
# no generated links between Wiki pages
toolsDir = mx.project('com.oracle.max.tools').output_dir()
baseDir = mx.project('com.oracle.max.base').output_dir()
dp = os.pathsep.join([toolsDir, baseDir])
project_list = ','.join(p.name for p in mx.sorted_deps())
for a in ['-docletpath', dp, '-doclet', 'com.oracle.max.tools.javadoc.wiki.WikiDoclet', '-projects', project_list]:
args.append('--arg')
args.append('@' + a)
mx.javadoc(args, parser=ArgumentParser('mx wikidoc'), docDir='wikidoc', includeDeps=False, stdDoclet=False)
def testgen(args):
'''generate the expected output for unit tests'''
# check we are in the home directory
if os.getcwd() != _fastr_suite.dir:
mx.abort('must run rtestgen from FastR home directory')
def _test_srcdir():
tp = 'com.oracle.truffle.r.test'
return join(mx.project(tp).dir, 'src', tp.replace('.', sep))
def need_version_check():
vardef = os.environ.has_key('FASTR_TESTGEN_GNUR')
varval = os.environ['FASTR_TESTGEN_GNUR'] if vardef else None
version_check = vardef and varval != 'internal'
if version_check:
rpath = join(varval, 'bin', 'R')
else:
rpath = None
return version_check, rpath
version_check, rpath = need_version_check()
if version_check:
# check the version of GnuR against FastR
try:
fastr_version = subprocess.check_output([
mx.get_jdk().java,
mx.get_runtime_jvm_args('com.oracle.truffle.r.runtime'),
'com.oracle.truffle.r.runtime.RVersionNumber'
])
gnur_version = subprocess.check_output([rpath, '--version'])
if not gnur_version.startswith(fastr_version):
mx.abort(
'R version is incompatible with FastR, please update to ' +
fastr_version)
except subprocess.CalledProcessError:
mx.abort('RVersionNumber.main failed')
tests = _all_generated_unit_tests()
# now just invoke unittst with the appropriate options
mx.log("generating expected output for packages: ")
for pkg in tests:
mx.log(" " + str(pkg))
os.environ["TZDIR"] = "/usr/share/zoneinfo/"
_unset_conflicting_envs()
mx_unittest.unittest([
'-Dfastr.test.gen.expected=' + _test_srcdir(),
'-Dfastr.test.gen.expected.quiet', '-Dfastr.test.project.output.dir=' +
mx.project('com.oracle.truffle.r.test').output_dir()
] + tests)
def runAndReturnStdOut(self, benchmarks: List[str], bmSuiteArgs: List[str]) -> Tuple[int, OutputCapture, Dict[str, str]]:
if len(benchmarks) > 1:
mx.abort("Only one benchmark can be run at a time")
bench_name = benchmarks[0]
bench_path = Path(mx.project("altrep.benchmarks").dir).joinpath(bench_name + ".r")
bench_runner_source = generate_bench_runner_source(self._bench_args, bench_path)
with tempfile.NamedTemporaryFile() as bench_runner_script:
bench_runner_script.write(bytes(bench_runner_source, "UTF-8"))
bench_runner_script.seek(0)
bench_runner_args = [
"--slave",
"--vanilla",
"-f", bench_runner_script.name
]
retcode = -1
output_capture = AltrepBenchmarkSuite.OutputCapture()
if self._bench_args.driver == AltrepBenchmarkSuite.Driver.FASTR:
extra_vm_args = []
extra_vm_args += ["-Dgraal.TraceTruffleCompilation=true"]
extra_vm_args += ["-Dgraal.CompilationFailureAction=ExitVM"]
extra_vm_args += ["-da"]
extra_vm_args += ["-dsa"]
#os.environ["MX_R_GLOBAL_ARGS"] = " ".join(extra_vm_args)
self.vm_args = extra_vm_args
fastr_args = ["--R.PrintErrorStacktracesToFile=true"]
self.run_args = fastr_args + bench_runner_args
if mx.suite("compiler", fatalIfMissing=False):
os.environ["DEFAULT_DYNAMIC_IMPORTS"] = "graal/compiler"
retcode = _fastr_suite.extensions.do_run_r(self.run_args, "R", nonZeroIsFatal=False,
extraVmArgs=extra_vm_args, out=output_capture, err=output_capture)
else:
# Run in GNU-R
self.vm_args = []
self.run_args = bench_runner_args
gnur_bin = str(self._gnur_dir.joinpath("bin", "R").absolute())
retcode = mx.run([gnur_bin] + self.run_args, out=output_capture, err=output_capture)
return retcode, output_capture, None
def test_javac(project_name, opts_set):
"""(jaotc-)Compiles the `jdk.compiler` module and compiles (mx) project_name using `javac` (+ AOT module)."""
# jaotc uses ':' as separator.
modules = ':'.join(['jdk.compiler'])
for common_opts in opts_set:
out_dir = tempfile.mkdtemp()
try:
mx.log('(jaotc) Compiling module(s) {} with {}'.format(
modules, ' '.join(common_opts)))
with mktemp_libfile() as lib_module:
lib_module.file.close()
run_jaotc(['-J' + opt for opt in common_opts] + [
'--exit-on-error', '--info', '--module', modules,
'--output', lib_module.name
])
aot_opts = [
'-XX:+UnlockDiagnosticVMOptions',
'-XX:+UseAOTStrictLoading',
'-XX:AOTLibrary=' + lib_module.name
]
project = mx.project(project_name)
java_files = collect_java_sources(project.source_dirs())
javac_args = mx.JavacCompiler(jdk).prepare(
sourceFiles=java_files,
project=project,
outputDir=out_dir,
classPath=mx.classpath(project,
includeSelf=False,
jdk=jdk,
ignoreStripped=True),
sourceGenDir=project.source_gen_dir(),
jnigenDir=project.jni_gen_dir(),
processorPath=project.annotation_processors_path(jdk),
disableApiRestrictions=True,
warningsAsErrors=False,
showTasks=False,
postCompileActions=[],
forceDeprecationAsWarning=False)
mx_compiler.run_vm(common_opts + aot_opts +
['com.sun.tools.javac.Main'] + javac_args)
finally:
shutil.rmtree(out_dir)
def create_parser(grammar_project, grammar_package, grammar_name, copyright_template, args=None, out=None):
"""create the DSL expression parser using antlr"""
grammar_dir = mx.project(grammar_project).source_dirs()[0] + "/" + grammar_package.replace(".", "/") + "/"
mx.run_java(mx.get_runtime_jvm_args(['ANTLR4_COMPLETE']) + ["org.antlr.v4.Tool", "-package", grammar_package, "-no-listener"] + args + [grammar_dir + grammar_name + ".g4"], out=out)
for filename in [grammar_dir + grammar_name + "Lexer.java", grammar_dir + grammar_name + "Parser.java"]:
with open(filename, 'r') as content_file:
content = content_file.read()
# remove first line
content = "\n".join(content.split("\n")[1:])
# modify SuppressWarnings to remove useless entries
content = PTRN_SUPPRESS_WARNINGS.sub('@SuppressWarnings("all")', content)
# remove useless casts
content = PTRN_LOCALCTXT_CAST.sub('_localctx', content)
content = PTRN_TOKEN_CAST.sub('_errHandler.recoverInline(this)', content)
# add copyright header
content = copyright_template.format(content)
with open(filename, 'w') as content_file:
content_file.write(content)
def nativebuild(args):
'''
force the build of part or all of the native project
'''
parser = ArgumentParser(prog='nativebuild')
parser.add_argument('--all', action='store_true', help='clean and build everything, else just ffi')
args = parser.parse_args(args)
nativedir = mx.project('com.oracle.truffle.r.native').dir
if args.all:
return subprocess.call(['make clean && make'], shell=True, cwd=nativedir)
else:
ffidir = join(nativedir, 'fficall')
jni_done = join(ffidir, 'jni.done')
jniboot_done = join(ffidir, 'jniboot.done')
if os.path.exists(jni_done):
os.remove(jni_done)
if os.path.exists(jniboot_done):
os.remove(jniboot_done)
return mx.build(['--no-java'])
def nativebuild(args):
'''
force the build of part or all of the native project
'''
parser = ArgumentParser(prog='nativebuild')
parser.add_argument('--all', action='store_true', help='clean and build everything, else just ffi')
args = parser.parse_args(args)
nativedir = mx.project('com.oracle.truffle.r.native').dir
if args.all:
return subprocess.call(['make clean && make'], shell=True, cwd=nativedir)
else:
ffidir = join(nativedir, 'fficall')
jni_done = join(ffidir, 'jni.done')
jniboot_done = join(ffidir, 'jniboot.done')
if os.path.exists(jni_done):
os.remove(jni_done)
if os.path.exists(jniboot_done):
os.remove(jniboot_done)
return mx.build(['--no-java'])
def _graal_nodejs_post_gate_runner(args, tasks):
_setEnvVar('NODE_INTERNAL_ERROR_CHECK', 'true')
with Task('UnitTests', tasks, tags=[GraalNodeJsTags.allTests, GraalNodeJsTags.unitTests]) as t:
if t:
_setEnvVar('NODE_JVM_CLASSPATH', mx.distribution('graal-js:TRUFFLE_JS_TESTS').path)
commonArgs = ['-ea', '-esa']
unitTestDir = join('test', 'graal')
for dir_name in 'node_modules', 'build':
p = join(unitTestDir, dir_name)
if exists(p):
mx.rmtree(p)
npm(['--scripts-prepend-node-path=auto', 'install', '--nodedir=' + _suite.dir] + commonArgs, cwd=unitTestDir)
npm(['--scripts-prepend-node-path=auto', 'test'] + commonArgs, cwd=unitTestDir)
with Task('TestNpm', tasks, tags=[GraalNodeJsTags.allTests, GraalNodeJsTags.windows]) as t:
if t:
tmpdir = tempfile.mkdtemp()
try:
npm(['init', '-y'], cwd=tmpdir)
npm(['install', '--scripts-prepend-node-path=true', 'microtime'], cwd=tmpdir)
node(['-e', 'console.log(require("microtime").now());'], cwd=tmpdir)
finally:
mx.rmtree(tmpdir, ignore_errors=True)
with Task('TestNpx', tasks, tags=[GraalNodeJsTags.allTests, GraalNodeJsTags.windows]) as t:
if t:
npx(['cowsay', 'GraalVM rules!'])
with Task('JniProfilerTests', tasks, tags=[GraalNodeJsTags.allTests, GraalNodeJsTags.jniProfilerTests]) as t:
if t:
commonArgs = ['-ea', '-esa']
unitTestDir = join(mx.project('com.oracle.truffle.trufflenode.jniboundaryprofiler').dir, 'tests')
for dir_name in 'node_modules', 'build':
p = join(unitTestDir, dir_name)
if exists(p):
mx.rmtree(p)
npm(['--scripts-prepend-node-path=auto', 'install', '--nodedir=' + _suite.dir] + commonArgs, cwd=unitTestDir)
node(['-profile-native-boundary', 'test.js'] + commonArgs, cwd=unitTestDir)
with Task('TestNodeInstrument', tasks, tags=[GraalNodeJsTags.allTests, GraalNodeJsTags.windows]) as t:
if t:
testnodeInstrument([])
def _alldeps(args):
parser = ArgumentParser(prog='mx mxt-alldeps')
parser.add_argument('--kind', action='store', help='project, dist or library', default='project')
parser.add_argument('--name', action='store', help='name of entity', required=True)
parser.add_argument('--includeLibs', action='store_true', help='includeLibs')
parser.add_argument('--ignoreSelf', action='store_true', help='ignoreSelf')
args = parser.parse_args(args)
entity = None
if args.kind == 'project':
entity = mx.project(args.name)
elif args.kind == 'library':
entity = mx.library(args.name)
elif args.kind == 'dist':
entity = mx.distribution(args.name)
else:
mx.abort('illegal kind: ' + args.kind)
in_deps = []
deps = entity.all_deps(in_deps, includeLibs=args.includeLibs, includeSelf=not args.ignoreSelf)
print 'alldeps:'
for d in deps:
print d.__class__.__name__, ":", d.name
def test_javac(project_name):
"""(jaotc-)Compiles the `jdk.compiler` module and compiles (mx) project_name using `javac` (+ AOT module)."""
# jaotc uses ':' as separator.
modules = ':'.join(['jdk.compiler'])
for common_opts in common_opts_variants:
out_dir = tempfile.mkdtemp()
try:
mx.log('(jaotc) Compiling module(s) {} with {}'.format(modules, ' '.join(common_opts)))
with mktemp_libfile() as lib_module:
run_jaotc(['-J' + opt for opt in common_opts] +
['--info', '--module', modules, '--output', lib_module.name])
aot_opts = [
'-XX:+UnlockDiagnosticVMOptions',
'-XX:+UseAOTStrictLoading',
'-XX:AOTLibrary=' + lib_module.name
]
project = mx.project(project_name)
java_files = collect_java_sources(project.source_dirs())
javac_args = mx.JavacCompiler(jdk).prepare(
sourceFiles=java_files,
project=project,
outputDir=out_dir,
classPath=mx.classpath(project, includeSelf=False, jdk=jdk, ignoreStripped=True),
sourceGenDir=project.source_gen_dir(),
jnigenDir=project.jni_gen_dir(),
processorPath=project.annotation_processors_path(jdk),
disableApiRestrictions=True,
warningsAsErrors=False,
showTasks=False,
postCompileActions=[],
forceDeprecationAsWarning=False)
mx_compiler.run_vm(common_opts + aot_opts + ['com.sun.tools.javac.Main'] + javac_args)
finally:
shutil.rmtree(out_dir)
def _unittest(args,
annotations,
prefixCp="",
blacklist=None,
whitelist=None,
verbose=False,
very_verbose=False,
fail_fast=False,
enable_timing=False,
regex=None,
color=False,
eager_stacktrace=False,
gc_after_test=False,
suite=None):
testfile = os.environ.get('MX_TESTFILE', None)
if testfile is None:
(_, testfile) = tempfile.mkstemp(".testclasses", "mxtool")
os.close(_)
mainClass = 'com.oracle.mxtool.junit.MxJUnitWrapper'
if not exists(
join(
mx.project('com.oracle.mxtool.junit').output_dir(),
mainClass.replace('.', os.sep) + '.class')):
mx.build(['--only', 'com.oracle.mxtool.junit'])
coreCp = mx.classpath(['com.oracle.mxtool.junit'])
coreArgs = []
if very_verbose:
coreArgs.append('-JUnitVeryVerbose')
elif verbose:
coreArgs.append('-JUnitVerbose')
if fail_fast:
coreArgs.append('-JUnitFailFast')
if enable_timing:
coreArgs.append('-JUnitEnableTiming')
if color:
coreArgs.append('-JUnitColor')
if eager_stacktrace:
coreArgs.append('-JUnitEagerStackTrace')
if gc_after_test:
coreArgs.append('-JUnitGCAfterTest')
def harness(unittestCp, vmLauncher, vmArgs):
prefixArgs = ['-esa', '-ea']
if gc_after_test:
prefixArgs.append('-XX:-DisableExplicitGC')
with open(testfile) as fp:
testclasses = [l.rstrip() for l in fp.readlines()]
cp = prefixCp + coreCp + os.pathsep + unittestCp
# suppress menubar and dock when running on Mac
vmArgs = prefixArgs + ['-Djava.awt.headless=true'] + vmArgs + [
'-cp', mx._separatedCygpathU2W(cp)
]
# Execute Junit directly when one test is being run. This simplifies
# replaying the VM execution in a native debugger (e.g., gdb).
mainClassArgs = coreArgs + (testclasses if len(testclasses) == 1 else
['@' + mx._cygpathU2W(testfile)])
config = (vmArgs, mainClass, mainClassArgs)
for p in _config_participants:
config = p(config)
vmLauncher.launcher(*config)
vmLauncher = _vm_launcher
if vmLauncher is None:
jdk = mx.get_jdk()
def _run_vm(vmArgs, mainClass, mainClassArgs):
mx.run_java(vmArgs + [mainClass] + mainClassArgs, jdk=jdk)
vmLauncher = _VMLauncher('default VM launcher', _run_vm, jdk)
try:
_run_tests(args, harness, vmLauncher, annotations, testfile, blacklist,
whitelist, regex,
mx.suite(suite) if suite else None)
finally:
if os.environ.get('MX_TESTFILE') is None:
os.remove(testfile)
def _junit_r_harness(args, vmArgs, jdk, junitArgs):
# always pass the directory where the expected output file should reside
runlistener_arg = 'expected=' + _test_srcdir()
# there should not be any unparsed arguments at this stage
if args.remainder:
mx.abort('unexpected arguments: ' + str(args.remainder).strip('[]') + '; did you forget --tests')
def add_arg_separator():
# can't update in Python 2.7
arg = runlistener_arg
if len(arg) > 0:
arg += ','
return arg
if args.gen_fastr_output:
runlistener_arg = add_arg_separator()
runlistener_arg += 'gen-fastr=' + args.gen_fastr_output
if args.check_expected_output:
args.gen_expected_output = True
runlistener_arg = add_arg_separator()
runlistener_arg += 'check-expected'
if args.gen_expected_output:
runlistener_arg = add_arg_separator()
runlistener_arg += 'gen-expected'
if args.keep_trailing_whitespace:
runlistener_arg = add_arg_separator()
runlistener_arg += 'keep-trailing-whitespace'
if args.gen_expected_quiet:
runlistener_arg = add_arg_separator()
runlistener_arg += 'gen-expected-quiet'
if args.gen_diff_output:
runlistener_arg = add_arg_separator()
runlistener_arg += 'gen-diff=' + args.gen_diff_output
if args.trace_tests:
runlistener_arg = add_arg_separator()
runlistener_arg += 'trace-tests'
# if args.test_methods:
# runlistener_arg = add_arg_separator()
# runlistener_arg = 'test-methods=' + args.test_methods
runlistener_arg = add_arg_separator()
runlistener_arg += 'test-project-output-dir=' + mx.project('com.oracle.truffle.r.test').output_dir()
# use a custom junit.RunListener
runlistener = 'com.oracle.truffle.r.test.TestBase$RunListener'
if len(runlistener_arg) > 0:
runlistener += ':' + runlistener_arg
junitArgs += ['--runlistener', runlistener]
# on some systems a large Java stack seems necessary
vmArgs += ['-Xss12m']
# no point in printing errors to file when running tests (that contain errors on purpose)
vmArgs += ['-DR:-PrintErrorStacktracesToFile']
setREnvironment()
return mx.run_java(vmArgs + junitArgs, nonZeroIsFatal=False, jdk=jdk)
def _test_srcdir():
tp = 'edu.uci.python.test'
return join(mx.project(tp).dir, 'src', tp.replace('.', sep))
def substitution_path():
path = mx.project('com.oracle.svm.bench').classpath_repr()
if not mx.exists(path):
mx.abort('Path to substitutions for scala dacapo not present: ' +
path + '. Did you build all of substratevm?')
return path
def _unittest(
args,
annotations,
prefixCp="",
blacklist=None,
whitelist=None,
verbose=False,
fail_fast=False,
enable_timing=False,
regex=None,
color=False,
eager_stacktrace=False,
gc_after_test=False,
suite=None,
):
testfile = os.environ.get("MX_TESTFILE", None)
if testfile is None:
(_, testfile) = tempfile.mkstemp(".testclasses", "mxtool")
os.close(_)
mainClass = "com.oracle.mxtool.junit.MxJUnitWrapper"
if not exists(join(mx.project("com.oracle.mxtool.junit").output_dir(), mainClass.replace(".", os.sep) + ".class")):
mx.build(["--only", "com.oracle.mxtool.junit"])
coreCp = mx.classpath(["com.oracle.mxtool.junit"])
coreArgs = []
if verbose:
coreArgs.append("-JUnitVerbose")
if fail_fast:
coreArgs.append("-JUnitFailFast")
if enable_timing:
coreArgs.append("-JUnitEnableTiming")
if color:
coreArgs.append("-JUnitColor")
if eager_stacktrace:
coreArgs.append("-JUnitEagerStackTrace")
if gc_after_test:
coreArgs.append("-JUnitGCAfterTest")
def harness(projectsCp, vmLauncher, vmArgs):
prefixArgs = ["-esa", "-ea"]
if gc_after_test:
prefixArgs.append("-XX:-DisableExplicitGC")
with open(testfile) as fp:
testclasses = [l.rstrip() for l in fp.readlines()]
cp = prefixCp + coreCp + os.pathsep + projectsCp
# suppress menubar and dock when running on Mac
vmArgs = prefixArgs + ["-Djava.awt.headless=true"] + vmArgs + ["-cp", mx._separatedCygpathU2W(cp)]
# Execute Junit directly when one test is being run. This simplifies
# replaying the VM execution in a native debugger (e.g., gdb).
mainClassArgs = coreArgs + (testclasses if len(testclasses) == 1 else ["@" + mx._cygpathU2W(testfile)])
config = (vmArgs, mainClass, mainClassArgs)
for p in _config_participants:
config = p(config)
_, launcher = vmLauncher
launcher(*config)
vmLauncher = _vm_launcher
if vmLauncher is None:
vmLauncher = (
"default VM launcher",
lambda vmArgs, mainClass, mainClassArgs: mx.run_java(vmArgs + [mainClass] + mainClassArgs),
)
try:
_run_tests(
args,
harness,
vmLauncher,
annotations,
testfile,
blacklist,
whitelist,
regex,
mx.suite(suite) if suite else None,
)
finally:
if os.environ.get("MX_TESTFILE") is None:
os.remove(testfile)
def runTestSuite(testsuiteproject, args, testClasses=None, vmArgs=None):
"""compile and run external testsuite projects"""
project = mx.project(testsuiteproject)
assert isinstance(project, SulongTestSuite)
project.runTestSuite(testClasses, vmArgs)
def _test_srcdir():
tp = 'com.oracle.truffle.r.test'
return join(mx.project(tp).dir, 'src', tp.replace('.', sep))
def _cran_test_project_dir():
return mx.project(_cran_test_project()).dir
def mx_post_parse_cmd_line(opts):
if mx.suite("tools", fatalIfMissing=False) and not _suite.isBinarySuite():
mx.project('truffleruby-bin').buildDependencies += [
mx.distribution('CHROMEINSPECTOR'),
mx.distribution('TRUFFLE_PROFILER')
]
def _gnur_path():
np = mx.project('com.oracle.truffle.r.native')
return join(np.dir, 'gnur', r_version(), 'bin')
def _packages_test_project_dir():
return mx.project(_packages_test_project()).dir
def sonarqube_upload(args):
"""run SonarQube scanner and upload JaCoCo results"""
sonarqube_cli = mx.library("SONARSCANNER_CLI_4_2_0_1873", True)
parser = ArgumentParser(prog='mx sonarqube-upload')
parser.add_argument('--exclude-generated', action='store_true', help='Exclude generated source files')
parser.add_argument('--skip-coverage', action='store_true', default=False, help='Do not upload coverage reports')
args, sonar_args = mx.extract_VM_args(args, useDoubleDash=True, defaultAllVMArgs=True)
args, other_args = parser.parse_known_args(args)
java_props, other_args = _parse_java_properties(other_args)
def _check_required_prop(prop):
if prop not in java_props:
mx.abort("Required property '{prop}' not present. (Format is '-D{prop}=<value>')".format(prop=prop))
_check_required_prop('sonar.projectKey')
_check_required_prop('sonar.host.url')
basedir = mx.primary_suite().dir
# collect excluded projects
excludes, includes = _jacoco_excludes_includes_projects(limit_to_primary=True)
# collect excluded classes
exclude_classes = _jacoco_exclude_classes(includes)
java_bin = []
java_src = []
java_libs = []
def _visit_deps(dep, edge):
if dep.isJARDistribution() or dep.isLibrary():
java_libs.append(dep.classpath_repr())
mx.walk_deps(includes, visit=_visit_deps)
# collect all sources and binaries -- do exclusion later
for p in includes:
java_src.extend(p.source_dirs())
if not args.exclude_generated:
gen_dir = p.source_gen_dir()
if os.path.exists(gen_dir):
java_src.append(gen_dir)
java_bin.append(p.output_dir())
java_src = [os.path.relpath(s, basedir) for s in java_src]
java_bin = [os.path.relpath(b, basedir) for b in java_bin]
# Overlayed sources and classes must be excluded
jdk_compliance = mx.get_jdk().javaCompliance
overlayed_sources = []
overlayed_classfiles = {}
for p in includes:
if hasattr(p, "multiReleaseJarVersion") and jdk_compliance not in p.javaCompliance: # JDK9+ overlays
for srcDir in p.source_dirs():
for root, _, files in os.walk(srcDir):
for name in files:
if name.endswith('.java') and name != 'package-info.java':
overlayed_sources.append(join(os.path.relpath(root, basedir), name))
elif hasattr(p, "overlayTarget"): # JDK8 overlays
target = mx.project(p.overlayTarget)
overlay_sources = []
for srcDir in p.source_dirs():
for root, _, files in os.walk(srcDir):
for name in files:
if name.endswith('.java') and name != 'package-info.java':
overlay_sources.append(join(os.path.relpath(root, srcDir), name))
print(p, target, overlay_sources)
for srcDir in target.source_dirs():
for root, _, files in os.walk(srcDir):
for name in files:
if name.endswith('.java') and name != 'package-info.java':
s = join(os.path.relpath(root, srcDir), name)
if s in overlay_sources:
overlayed = join(os.path.relpath(root, basedir), name)
overlayed_sources.append(overlayed)
for s in overlay_sources:
classfile = join(os.path.relpath(target.output_dir(), basedir), s[:-len('java')] + 'class')
with open(classfile, 'rb') as fp:
overlayed_classfiles[classfile] = fp.read()
exclude_dirs = []
for p in excludes:
exclude_dirs.extend(p.source_dirs())
exclude_dirs.append(p.source_gen_dir())
javaCompliance = max([p.javaCompliance for p in includes]) if includes else mx.JavaCompliance('1.7')
jacoco_exec = get_jacoco_dest_file()
if not os.path.exists(jacoco_exec) and not args.skip_coverage:
mx.abort('No JaCoCo report file found: ' + jacoco_exec)
def _add_default_prop(key, value):
if key not in java_props:
java_props[key] = value
# default properties
_add_default_prop('sonar.java.source', str(javaCompliance))
_add_default_prop('sonar.projectBaseDir', basedir)
if not args.skip_coverage:
_add_default_prop('sonar.jacoco.reportPaths', jacoco_exec)
_add_default_prop('sonar.sources', ','.join(java_src))
_add_default_prop('sonar.java.binaries', ','.join(java_bin))
_add_default_prop('sonar.java.libraries', ','.join(java_libs))
exclude_patterns = [os.path.relpath(e, basedir) + '**' for e in exclude_dirs] + \
overlayed_sources + \
list(set([os.path.relpath(match[0], basedir) for _, match in exclude_classes.items()]))
if exclude_patterns:
_add_default_prop('sonar.exclusions', ','.join(exclude_patterns))
_add_default_prop('sonar.coverage.exclusions', ','.join(exclude_patterns))
_add_default_prop('sonar.verbose', 'true' if mx._opts.verbose else 'false')
with tempfile.NamedTemporaryFile(suffix="-sonarqube.properties", mode="w+") as fp:
# prepare properties file
fp.writelines(('{}={}\n'.format(k, v) for k, v in java_props.items()))
fp.flush()
# Since there's no options to exclude individual classes,
# we temporarily delete the overlayed class files instead.
for classfile in overlayed_classfiles:
os.remove(classfile)
try:
# run sonarqube cli
java_args = other_args + ['-Dproject.settings=' + fp.name, '-jar', sonarqube_cli.get_path(True)] + sonar_args
exit_code = mx.run_java(java_args, nonZeroIsFatal=False)
finally:
# Restore temporarily deleted class files
for classfile, data in overlayed_classfiles.items():
with open(classfile, 'wb') as cf:
cf.write(data)
if exit_code != 0:
fp.seek(0)
mx.abort('SonarQube scanner terminated with non-zero exit code: {}\n Properties file:\n{}'.format(
exit_code, ''.join((' ' + l for l in fp.readlines()))))
def _cran_test_project():
return mx.project('com.oracle.truffle.r.test.cran').dir
def jaotc_classpath():
return mx.project('jdk.tools.jaotc.test').output_dir()
def _test_srcdir():
tp = 'com.oracle.truffle.r.test'
return join(mx.project(tp).dir, 'src', tp.replace('.', sep))
