def checkcopyrights(args):
"""run copyright check on the Maxine sources"""
for i in args:
if i == '-h' or i == '-help' or i == '--help':
print '"for help run mx :checkcopyrights -h"'
return
mx.checkcopyrights(['--', '--copyright-dir', 'mx.maxine'] + args)
def _sdk_gate_runner(args, tasks):
with Task('SDK UnitTests', tasks, tags=['test']) as t:
if t:
unittest([
'--suite', 'sdk', '--enable-timing', '--verbose', '--fail-fast'
])
with Task('Check Copyrights', tasks) as t:
if t: mx.checkcopyrights(['--primary'])
def _truffle_gate_runner(args, tasks):
jdk = mx.get_jdk(tag=mx.DEFAULT_JDK_TAG)
with Task('Jackpot check', tasks) as t:
if t: jackpot(['--fail-on-warnings'], suite=None, nonZeroIsFatal=True)
if jdk.javaCompliance < '9':
with Task('Truffle Javadoc', tasks) as t:
if t: javadoc([])
with Task('Truffle UnitTests', tasks) as t:
if t: unittest(['--suite', 'truffle', '--enable-timing', '--verbose', '--fail-fast'])
with Task('Truffle Signature Tests', tasks) as t:
if t: sigtest(['--check', 'binary'])
with Task('File name length check', tasks) as t:
if t: check_filename_length([])
with Task('Check Copyrights', tasks) as t:
if t: mx.checkcopyrights(['--primary'])
def _sdk_gate_runner(args, tasks):
with Task('SDK UnitTests', tasks, tags=['test']) as t:
if t: unittest(['--suite', 'sdk', '--enable-timing', '--verbose', '--fail-fast'])
with Task('Check Copyrights', tasks) as t:
if t:
if mx.checkcopyrights(['--primary', '--', '--projects', 'src']) != 0:
t.abort('Copyright errors found. Please run "mx checkcopyrights --primary -- --fix" to fix them.')
def _sulong_gate_runner(args, tasks):
with Task('CheckCopyright', tasks, tags=['style']) as t:
if t:
if mx.checkcopyrights(['--primary']) != 0:
t.abort('Copyright errors found. Please run "mx checkcopyrights --primary -- --fix" to fix them.')
with Task('BuildLLVMorg', tasks, tags=['style', 'clangformat']) as t:
# needed for clang-format
if t: build_llvm_org(args)
with Task('ClangFormat', tasks, tags=['style', 'clangformat']) as t:
if t: clangformatcheck()
_sulong_gate_testsuite('Benchmarks', 'shootout', tasks, args, tags=['benchmarks', 'sulongMisc'])
_sulong_gate_unittest('Types', 'SULONG_TEST', tasks, args, tags=['type', 'sulongMisc', 'sulongCoverage'], testClasses=['com.oracle.truffle.llvm.tests.types.floating'])
_sulong_gate_unittest('Pipe', 'SULONG_TEST', tasks, args, tags=['pipe', 'sulongMisc', 'sulongCoverage'], testClasses=['CaptureOutputTest'])
_sulong_gate_testsuite('LLVM', 'llvm', tasks, args, tags=['llvm', 'sulongCoverage'])
_sulong_gate_testsuite('NWCC', 'nwcc', tasks, args, tags=['nwcc', 'sulongCoverage'])
_sulong_gate_testsuite('GCCParserTorture', 'parserTorture', tasks, args, tags=['parser', 'sulongCoverage'], vmArgs=['-Dpolyglot.llvm.parseOnly=true'])
_sulong_gate_testsuite('GCC_C', 'gcc_c', tasks, args, tags=['gcc_c', 'sulongCoverage'])
_sulong_gate_testsuite('GCC_CPP', 'gcc_cpp', tasks, args, tags=['gcc_cpp', 'sulongCoverage'])
_sulong_gate_testsuite('GCC_Fortran', 'gcc_fortran', tasks, args, tags=['gcc_fortran', 'sulongCoverage'])
_sulong_gate_sulongsuite_unittest('Sulong', tasks, args, testClasses='SulongSuite', tags=['sulong', 'sulongBasic', 'sulongCoverage'])
_sulong_gate_sulongsuite_unittest('Interop', tasks, args, testClasses='com.oracle.truffle.llvm.tests.interop', tags=['interop', 'sulongBasic', 'sulongCoverage'])
_sulong_gate_sulongsuite_unittest('Linker', tasks, args, testClasses='com.oracle.truffle.llvm.tests.linker', tags=['linker', 'sulongBasic', 'sulongCoverage'])
_sulong_gate_sulongsuite_unittest('Debug', tasks, args, testClasses='LLVMDebugTest', tags=['debug', 'sulongBasic', 'sulongCoverage'])
_sulong_gate_sulongsuite_unittest('IRDebug', tasks, args, testClasses='LLVMIRDebugTest', tags=['irdebug', 'sulongBasic', 'sulongCoverage'])
_sulong_gate_sulongsuite_unittest('BitcodeFormat', tasks, args, testClasses='BitcodeFormatTest', tags=['bitcodeFormat', 'sulongBasic', 'sulongCoverage'])
_sulong_gate_sulongsuite_unittest('OtherTests', tasks, args, testClasses='com.oracle.truffle.llvm.tests.other', tags=['otherTests', 'sulongBasic', 'sulongCoverage'])
_sulong_gate_testsuite('Assembly', 'inlineassemblytests', tasks, args, testClasses='InlineAssemblyTest', tags=['assembly', 'sulongMisc', 'sulongCoverage'])
_sulong_gate_testsuite('Args', 'other', tasks, args, tags=['args', 'sulongMisc', 'sulongCoverage'], testClasses=['com.oracle.truffle.llvm.tests.MainArgsTest'])
_sulong_gate_testsuite('Callback', 'other', tasks, args, tags=['callback', 'sulongMisc', 'sulongCoverage'], testClasses=['com.oracle.truffle.llvm.tests.CallbackTest'])
_sulong_gate_testsuite('Varargs', 'other', tasks, args, tags=['vaargs', 'sulongMisc', 'sulongCoverage'], testClasses=['com.oracle.truffle.llvm.tests.VAArgsTest'])
with Task('TestToolchain', tasks, tags=['toolchain', 'sulongMisc', 'sulongCoverage']) as t:
if t:
mx.command_function('clean')(['--project', 'toolchain-launchers-tests'] + args.extra_build_args)
mx.command_function('build')(['--project', 'toolchain-launchers-tests'] + args.extra_build_args)
def _fastr_gate_runner(args, tasks):
'''
The specific additional gates tasks provided by FastR:
1. Copyright check
2. Check that ExpectedTestOutput file is in sync with unit tests
3. Unit tests
'''
# FastR has custom copyright check
with mx_gate.Task('Copyright check', tasks) as t:
if t:
if mx.checkcopyrights(['--primary']) != 0:
t.abort('copyright errors')
# check that the expected test output file is up to date
with mx_gate.Task('UnitTests: ExpectedTestOutput file check', tasks) as t:
if t:
mx_unittest.unittest(['-Dfastr.test.gen.expected=' + _test_srcdir(), '-Dfastr.test.check.expected'] + _gate_unit_tests())
with mx_gate.Task('UnitTests: no specials', tasks) as t:
if t:
mx_unittest.unittest(['-Dfastr.test.options.R.UseSpecials=false'] + _gate_noapps_unit_tests())
with mx_gate.Task('UnitTests: with specials', tasks) as t:
if t:
mx_unittest.unittest(_gate_noapps_unit_tests())
with mx_gate.Task('UnitTests: apps', tasks) as t:
if t:
mx_unittest.unittest(_apps_unit_tests())
with mx_gate.Task('Rembedded', tasks) as t:
if t:
if rembedtest([]) != 0:
t.abort("Rembedded tests failed")
def _add_copyright_checks(tasks):
with mx_gate.Task('Check Copyrights', tasks,
tags=[mx_gate.Tags.style]) as t:
if t:
if mx.checkcopyrights(['--primary']) != 0:
t.abort('Copyright errors found. Please run "mx '
'checkcopyrights --primary -- --fix" to fix them.')
def _fastr_gate_runner(args, tasks):
'''
The specific additional gates tasks provided by FastR:
1. Copyright check
2. Check that ExpectedTestOutput file is in sync with unit tests
3. Unit tests
'''
# FastR has custom copyright check
with mx_gate.Task('Copyright check', tasks) as t:
if t:
if mx.checkcopyrights(['--primary']) != 0:
t.abort('copyright errors')
# check that the expected test output file is up to date
with mx_gate.Task('UnitTests: ExpectedTestOutput file check', tasks) as t:
if t:
if junit([
'--tests',
_gate_noapps_unit_tests(), '--check-expected-output'
]) != 0:
t.abort('unit tests expected output check failed')
with mx_gate.Task('UnitTests', tasks) as t:
if t:
if junit(['--tests', _gate_noapps_unit_tests()]) != 0:
t.abort('unit tests failed')
def _graal_js_gate_runner(args, tasks):
with Task('CheckCopyrights', tasks, tags=['style']) as t:
if t:
if mx.checkcopyrights(['--primary']) != 0:
t.abort(
'Copyright errors found. Please run "mx checkcopyrights --primary -- --fix" to fix them.'
)
with Task('TestJSCommand',
tasks,
tags=[GraalJsDefaultTags.default, GraalJsDefaultTags.all]) as t:
if t:
js(['-Dpolyglot.js.profile-time=true', '-e', '""'])
with Task('UnitTests',
tasks,
tags=[GraalJsDefaultTags.default, GraalJsDefaultTags.all]) as t:
if t:
unittest(
['--enable-timing', '--very-verbose', '--suite', _suite.name])
gateTestConfigs = {
GraalJsDefaultTags.default: ['gate'],
'noic': [
'-Dpolyglot.js.property-cache-limit=0',
'-Dpolyglot.js.function-cache-limit=0', 'gate'
],
'directbytebuffer': ['-Dpolyglot.js.direct-byte-buffer=true', 'gate'],
'cloneuninitialized':
['-Dpolyglot.js.test-clone-uninitialized=true', 'gate'],
'lazytranslation': ['-Dpolyglot.js.lazy-translation=true', 'gate'],
'shareengine': ['gate', 'shareengine'],
}
gateTestCommands = {
'Test262': test262,
'TestNashorn': testnashorn,
'TestV8': testv8,
}
for testCommandName in gateTestCommands:
for testConfigName in gateTestConfigs:
testName = '%s-%s' % (testCommandName, testConfigName)
with Task(testName,
tasks,
tags=[testName, testConfigName,
GraalJsDefaultTags.all]) as t:
if t:
gateTestCommands[testCommandName](
gateTestConfigs[testConfigName])
with Task('TCK tests',
tasks,
tags=[GraalJsDefaultTags.all, GraalJsDefaultTags.tck]) as t:
if t:
import mx_truffle
mx_truffle._tck([])
def _graal_js_gate_runner(args, tasks):
with Task('CheckCopyrights', tasks, tags=['style']) as t:
if t:
if mx.checkcopyrights(['--primary']) != 0:
t.abort('Copyright errors found. Please run "mx checkcopyrights --primary -- --fix" to fix them.')
with Task('TestJSCommand', tasks, tags=[GraalJsDefaultTags.default, GraalJsDefaultTags.all]) as t:
if t:
js(['-Dpolyglot.js.profile-time=true', '-e', '""'])
webassemblyTestSuite = 'com.oracle.truffle.js.test.suite.WebAssemblySimpleTestSuite'
with Task('UnitTests', tasks, tags=[GraalJsDefaultTags.default, GraalJsDefaultTags.all]) as t:
if t:
noWebAssemblyTestSuite = '^(?!' + webassemblyTestSuite + ')'
commonOptions = ['--enable-timing', '--very-verbose', '--suite', _suite.name]
unittest(['--regex', noWebAssemblyTestSuite] + commonOptions)
unittest(['--regex', 'ZoneRulesProviderTest', '-Djava.time.zone.DefaultZoneRulesProvider=com.oracle.truffle.js.test.runtime.SimpleZoneRulesProvider'] + commonOptions)
with Task('WebAssemblyTests', tasks, tags=['webassembly', GraalJsDefaultTags.all]) as t:
if t:
unittest(['--regex', webassemblyTestSuite, '--enable-timing', '--very-verbose', '--suite', _suite.name])
gateTestConfigs = {
GraalJsDefaultTags.default: ['gate'],
'noic': ['-Dpolyglot.js.property-cache-limit=0', '-Dpolyglot.js.function-cache-limit=0', 'gate'],
'directbytebuffer': ['-Dpolyglot.js.direct-byte-buffer=true', 'gate'],
'cloneuninitialized': ['-Dpolyglot.js.test-clone-uninitialized=true', 'gate'],
'lazytranslation': ['-Dpolyglot.js.lazy-translation=true', 'gate'],
'zonerulesbasedtimezones': ['-Dpolyglot.js.zone-rules-based-time-zones=true', 'gate'],
'shareengine': ['gate', 'shareengine'],
'latestversion': ['gate', 'minesversion=2022'],
'instrument': ['gate', 'instrument', 'timeoutoverall=1800']
}
gateTestCommands = {
'Test262': test262,
'TestNashorn': testnashorn,
'TestV8': testv8,
}
for testCommandName in gateTestCommands:
for testConfigName in gateTestConfigs:
# TestNashorn is not sensitive to ES version
if testCommandName == 'TestNashorn' and testConfigName == 'latestversion':
continue
testName = '%s-%s' % (testCommandName, testConfigName)
with Task(testName, tasks, tags=[testName, testConfigName, GraalJsDefaultTags.all]) as t:
if t:
gateTestCommands[testCommandName](gateTestConfigs[testConfigName])
with Task('TCK tests', tasks, tags=[GraalJsDefaultTags.all, GraalJsDefaultTags.tck]) as t:
if t:
import mx_truffle
mx_truffle._tck([])
def checkCopyrightHeaders(args):
# create list of Java sources to check
with open('files_to_check', 'w') as f:
mx.run(['git', 'ls-files', '--', '*.java'], out=f)
# create an overrides file so that NodeProf copyright can be checked
with open(join(_suite.dir, 'mx.nodeprof/copyrights/overrides'), 'w') as f:
mx.run(['awk', '{print $0",nodeprof.copyright"}', 'files_to_check'],
out=f)
# run the actual check (overrodes file needs to be passed explicitly)
return mx.checkcopyrights(
['--primary', '--', '--file-list', 'files_to_check'])
def _sulong_gate_runner(args, tasks):
with TemporaryEnv():
with Task('CheckCopyright', tasks, tags=['style']) as t:
if t:
if mx.checkcopyrights(['--primary']) != 0:
t.abort(
'Copyright errors found. Please run "mx checkcopyrights --primary -- --fix" to fix them.'
)
with Task('ClangFormat', tasks, tags=['style', 'clangformat']) as t:
if t: clangformatcheck()
with Task('TestBenchmarks', tasks, tags=['benchmarks',
'sulongMisc']) as t:
if t: mx_testsuites.runSuite('shootout')
with Task('TestTypes', tasks, tags=['type', 'sulongMisc']) as t:
if t: mx_testsuites.runSuite('type')
with Task('TestPipe', tasks, tags=['pipe', 'sulongMisc']) as t:
if t: mx_testsuites.runSuite('pipe')
with Task('TestLLVM', tasks, tags=['llvm']) as t:
if t: mx_testsuites.runSuite('llvm')
with Task('TestNWCC', tasks, tags=['nwcc']) as t:
if t: mx_testsuites.runSuite('nwcc')
with Task('TestGCCParserTorture', tasks, tags=['parser']) as t:
if t: mx_testsuites.runSuite('parserTorture')
with Task('TestGCC_C', tasks, tags=['gcc_c']) as t:
if t: mx_testsuites.runSuite('gcc_c')
with Task('TestGCC_CPP', tasks, tags=['gcc_cpp']) as t:
if t: mx_testsuites.runSuite('gcc_cpp')
with Task('TestGCC_Fortran', tasks, tags=['gcc_fortran']) as t:
if t: mx_testsuites.runSuite('gcc_fortran')
with Task("TestSulong", tasks, tags=['sulong', 'sulongBasic']) as t:
if t: mx_unittest.unittest(['SulongSuite'])
with Task("TestInterop", tasks, tags=['interop', 'sulongBasic']) as t:
if t:
mx_unittest.unittest(['com.oracle.truffle.llvm.test.interop'])
with Task("TestDebug", tasks, tags=['debug', 'sulongBasic']) as t:
if t: mx_unittest.unittest(['LLVMDebugTest'])
with Task("TestIRDebug", tasks, tags=['irdebug', 'sulongBasic']) as t:
if t: mx_unittest.unittest(['LLVMIRDebugTest'])
with Task('TestAssembly', tasks, tags=['assembly', 'sulongMisc']) as t:
if t: mx_testsuites.runSuite('assembly')
with Task('TestArgs', tasks, tags=['args', 'sulongMisc']) as t:
if t: mx_testsuites.runSuite('args')
with Task('TestCallback', tasks, tags=['callback', 'sulongMisc']) as t:
if t: mx_testsuites.runSuite('callback')
with Task('TestVarargs', tasks, tags=['vaargs', 'sulongMisc']) as t:
if t: mx_testsuites.runSuite('vaargs')
def _sulong_gate_runner(args, tasks):
_unittest_task_factory = UnittestTaskFactory()
def _unittest(title, test_suite, tags=None, testClasses=None, unittestArgs=None, description=None):
_unittest_task_factory.add(title, test_suite, args, tags=tags, testClasses=testClasses, unittestArgs=unittestArgs, description=description)
with Task('CheckCopyright', tasks, tags=['style']) as t:
if t:
if mx.checkcopyrights(['--primary']) != 0:
t.abort('Copyright errors found. Please run "mx checkcopyrights --primary -- --fix" to fix them.')
with Task('BuildLLVMorg', tasks, tags=['style', 'clangformat']) as t:
# needed for clang-format
if t: build_llvm_org(args)
with Task('ClangFormat', tasks, tags=['style', 'clangformat']) as t:
if t: clangformat([])
# Folders not containing tests: options, services, util
_unittest('Benchmarks', 'SULONG_SHOOTOUT_TEST_SUITE', description="Language Benchmark game tests", testClasses=['ShootoutsSuite'], tags=['benchmarks', 'sulongMisc'])
_unittest('Types', 'SULONG_TEST', description="Test floating point arithmetic", testClasses=['com.oracle.truffle.llvm.tests.types.floating.'], tags=['type', 'sulongMisc', 'sulongCoverage'])
_unittest('Pipe', 'SULONG_TEST', description="Test output capturing", testClasses=['CaptureOutputTest'], tags=['pipe', 'sulongMisc', 'sulongCoverage'])
_unittest('LLVM', 'SULONG_LLVM_TEST_SUITE', description="LLVM 3.2 test suite", testClasses=['LLVMSuite'], tags=['llvm', 'sulongCoverage'])
_unittest('NWCC', 'SULONG_NWCC_TEST_SUITE', description="Test suite of the NWCC compiler v0.8.3", testClasses=['NWCCSuite'], tags=['nwcc', 'sulongCoverage'])
_unittest('GCCParserTorture', 'SULONG_PARSER_TORTURE', description="Parser test using GCC suite", testClasses=['ParserTortureSuite'], tags=['parser', 'sulongCoverage'])
_unittest('GCC_C', 'SULONG_GCC_C_TEST_SUITE', description="GCC 5.2 test suite (C tests)", testClasses=['GccCSuite'], tags=['gcc_c', 'sulongCoverage'])
_unittest('GCC_CPP', 'SULONG_GCC_CPP_TEST_SUITE', description="GCC 5.2 test suite (C++ tests)", testClasses=['GccCppSuite'], tags=['gcc_cpp', 'sulongCoverage'])
_unittest('GCC_Fortran', 'SULONG_GCC_FORTRAN_TEST_SUITE', description="GCC 5.2 test suite (Fortran tests)", testClasses=['GccFortranSuite'], tags=['gcc_fortran', 'sulongCoverage'])
_unittest('Sulong', 'SULONG_STANDALONE_TEST_SUITES', description="Sulong's internal tests", testClasses='SulongSuite', tags=['sulong', 'sulongBasic', 'sulongCoverage'])
_unittest('Interop', 'SULONG_EMBEDDED_TEST_SUITES', description="Truffle Language interoperability tests", testClasses=['com.oracle.truffle.llvm.tests.interop.'], tags=['interop', 'sulongBasic', 'sulongCoverage'])
_unittest('Linker', 'SULONG_EMBEDDED_TEST_SUITES', description=None, testClasses=['com.oracle.truffle.llvm.tests.linker.'], tags=['linker', 'sulongBasic', 'sulongCoverage'])
_unittest('Debug', 'SULONG_EMBEDDED_TEST_SUITES', description="Debug support test suite", testClasses=['com.oracle.truffle.llvm.tests.debug.LLVMDebugTest'], tags=['debug', 'sulongBasic', 'sulongCoverage'])
_unittest('IRDebug', 'SULONG_EMBEDDED_TEST_SUITES', description=None, testClasses=['com.oracle.truffle.llvm.tests.debug.LLVMIRDebugTest'], tags=['irdebug', 'sulongBasic', 'sulongCoverage'])
_unittest('BitcodeFormat', 'SULONG_EMBEDDED_TEST_SUITES', description=None, testClasses=['com.oracle.truffle.llvm.tests.bitcodeformat.'], tags=['bitcodeFormat', 'sulongBasic', 'sulongCoverage'])
_unittest('DebugExpr', 'SULONG_EMBEDDED_TEST_SUITES', description=None, testClasses=['com.oracle.truffle.llvm.tests.debug.LLVMDebugExprParserTest'], tags=['debugexpr', 'sulongBasic', 'sulongCoverage'])
_unittest('OtherTests', 'SULONG_EMBEDDED_TEST_SUITES', description=None, testClasses=['com.oracle.truffle.llvm.tests.bitcode.', 'com.oracle.truffle.llvm.tests.other.', 'com.oracle.truffle.llvm.tests.runtime.'], tags=['otherTests', 'sulongBasic', 'sulongCoverage'])
_unittest('Args', 'SULONG_EMBEDDED_TEST_SUITES', description="Tests main args passing", testClasses=['com.oracle.truffle.llvm.tests.MainArgsTest'], tags=['args', 'sulongMisc', 'sulongCoverage'])
_unittest('Callback', 'SULONG_EMBEDDED_TEST_SUITES', description="Test calling native functions", testClasses=['com.oracle.truffle.llvm.tests.CallbackTest'], tags=['callback', 'sulongMisc', 'sulongCoverage'])
_unittest('Varargs', 'SULONG_EMBEDDED_TEST_SUITES', description="Varargs tests", testClasses=['com.oracle.truffle.llvm.tests.VAArgsTest'], tags=['vaargs', 'sulongMisc', 'sulongCoverage'])
_unittest_task_factory.execute(tasks)
with Task('TestToolchain', description="build toolchain-launchers-tests project", tags=['toolchain', 'sulongMisc', 'sulongCoverage'], tasks=tasks) as t:
if t:
with SulongGateEnv():
mx.command_function('clean')(['--project', 'toolchain-launchers-tests'] + args.extra_build_args)
mx.command_function('build')(['--project', 'toolchain-launchers-tests'] + args.extra_build_args)
def updategraalinopenjdk(args):
"""updates the Graal sources in OpenJDK"""
parser = ArgumentParser(prog='mx updategraalinopenjdk')
parser.add_argument('--pretty', help='value for --pretty when logging the changes since the last JDK* tag')
parser.add_argument('jdkrepo', help='path to the local OpenJDK repo')
parser.add_argument('version', type=int, help='Java version of the OpenJDK repo')
parser.add_argument('--metro', action='store_true', help='update for Metropolis project (to build libgraal)')
args = parser.parse_args(args)
if mx_compiler.jdk.javaCompliance.value < args.version:
mx.abort('JAVA_HOME/--java-home must be Java version {} or greater: {}'.format(args.version, mx_compiler.jdk))
if not args.metro:
mx.log('Processing update for OpenJDK {}'.format(args.version))
graal_modules = [
# JDK module jdk.internal.vm.compiler is composed of sources from:
GraalJDKModule('jdk.internal.vm.compiler',
# 1. Classes in the compiler suite under the org.graalvm namespace except for packages
# or projects whose names contain terms on the specified exclude list
[SuiteJDKInfo('compiler', ['org.graalvm'], ['truffle', 'management', 'core.llvm', 'replacements.llvm', 'libgraal.jni']),
# 2. Classes in the sdk suite under the org.graalvm.collections and org.graalvm.word namespaces
SuiteJDKInfo('sdk', ['org.graalvm.collections', 'org.graalvm.word'], [])]),
# JDK module jdk.internal.vm.compiler.management is composed of sources from:
GraalJDKModule('jdk.internal.vm.compiler.management',
# 1. Classes in the compiler suite under the org.graalvm.compiler.hotspot.management namespace
[SuiteJDKInfo('compiler', ['org.graalvm.compiler.hotspot.management'], ['libgraal'])]),
]
else:
if args.version < 15:
mx.abort('Update for Metropolis works only for JDK 15 or later')
mx.log('Processing update for Metropolis JDK {}'.format(args.version))
# Metropolis needs more code from Graal
graal_modules = [
# JDK module jdk.internal.vm.compiler is composed of sources from:
GraalJDKModule('jdk.internal.vm.compiler',
# 1. Classes in the compiler suite under the org.graalvm namespace except for packages
# or projects whose names contain terms on the specified exclude list
[SuiteJDKInfo('compiler', ['org.graalvm'], ['truffle', 'management', 'core.llvm', 'replacements.llvm']),
# 2. Classes in the sdk suite under the org.graalvm.collections and org.graalvm.word namespaces
SuiteJDKInfo('sdk', ['org.graalvm.collections', 'org.graalvm.word', 'org.graalvm.nativeimage', 'org.graalvm.options'], [])]),
# JDK module jdk.internal.vm.compiler.management is composed of sources from:
GraalJDKModule('jdk.internal.vm.compiler.management',
# 1. Classes in the compiler suite under the org.graalvm.compiler.hotspot.management namespace
[SuiteJDKInfo('compiler', ['org.graalvm.compiler.hotspot.management'], ['libgraal'])]),
]
# Strings to be replaced in files copied to OpenJDK.
replacements = {
'published by the Free Software Foundation. Oracle designates this\n * particular file as subject to the "Classpath" exception as provided\n * by Oracle in the LICENSE file that accompanied this code.' : 'published by the Free Software Foundation.',
_read_sibling_file('upl_substring.txt') : _read_sibling_file('gplv2_substring.txt')
}
# Strings that must not exist in OpenJDK source files. This is applied after replacements are made.
denylist = ['"Classpath" exception']
jdkrepo = args.jdkrepo
git_repo = _is_git_repo(jdkrepo)
for m in graal_modules:
m_src_dir = join(jdkrepo, 'src', m.name)
if not exists(m_src_dir):
mx.abort(jdkrepo + ' does not look like a JDK repo - ' + m_src_dir + ' does not exist')
def run_output(args, cwd=None):
out = mx.OutputCapture()
mx.run(args, cwd=cwd, out=out, err=out)
return out.data
for m in graal_modules:
m_src_dir = join('src', m.name)
mx.log('Checking ' + m_src_dir)
if git_repo:
out = run_output(['git', 'status', '-s', m_src_dir], cwd=jdkrepo)
else:
out = run_output(['hg', 'status', m_src_dir], cwd=jdkrepo)
if out:
mx.abort(jdkrepo + ' is not "clean":' + '\n' + out[:min(200, len(out))] + '...')
mx.checkcopyrights(['--primary', '--', '--fix'])
for dirpath, _, filenames in os.walk(join(jdkrepo, 'make')):
for filename in filenames:
if filename.endswith('.gmk'):
rename_packages(join(dirpath, filename), True)
java_package_re = re.compile(r"^\s*package\s+(?P<package>[a-zA-Z_][\w\.]*)\s*;$", re.MULTILINE)
copied_source_dirs = []
jdk_internal_vm_compiler_EXCLUDES = set() # pylint: disable=invalid-name
jdk_internal_vm_compiler_test_SRC = set() # pylint: disable=invalid-name
# Add org.graalvm.compiler.processor since it is only a dependency
# for (most) Graal annotation processors and is not needed to
# run Graal.
jdk_internal_vm_compiler_EXCLUDES.add('org.graalvm.compiler.processor')
for m in graal_modules:
classes_dir = join(jdkrepo, 'src', m.name, 'share', 'classes')
for info in m.suites:
mx.log('Processing ' + m.name + ':' + info.name)
for e in os.listdir(classes_dir):
if any(inc in e for inc in info.includes) and not any(ex in e for ex in info.excludes):
project_dir = join(classes_dir, e)
shutil.rmtree(project_dir)
mx.log(' removed ' + project_dir)
suite = mx.suite(info.name)
worklist = []
for p in [e for e in suite.projects if e.isJavaProject()]:
if str(mx_compiler.jdk.javaCompliance) not in p.javaCompliance:
mx.log(' skipping {} since its compliance ({}) is not compatible with {}'.format(p, repr(p.javaCompliance), mx_compiler.jdk.javaCompliance))
continue
if any(inc in p.name for inc in info.includes) and not any(ex in p.name for ex in info.excludes):
assert len(p.source_dirs()) == 1, p
version = 0
new_project_name = p.name
if hasattr(p, 'multiReleaseJarVersion'):
version = int(getattr(p, 'multiReleaseJarVersion'))
if version <= args.version:
base_project = _find_version_base_project(p)
new_project_name = base_project.name
else:
continue
for old_name, new_name in package_renamings.items():
if new_project_name.startswith(old_name):
new_project_name = new_project_name.replace(old_name, new_name)
workitem = (version, p, new_project_name)
worklist.append(workitem)
# Ensure versioned resources are copied in the right order
# such that higher versions override lower versions.
worklist = sorted(worklist)
for version, p, new_project_name in worklist:
first_file = True
source_dir = p.source_dirs()[0]
target_dir = join(classes_dir, new_project_name, 'src')
copied_source_dirs.append(source_dir)
trailing = re.compile(r"[ \t]+\n")
src_files = run_output(['git', 'ls-files'], cwd=source_dir).split('\n')
for rel_src_file in src_files:
if not len(rel_src_file):
continue
filename = os.path.basename(rel_src_file)
src_file = join(source_dir, rel_src_file)
dst_file = join(target_dir, os.path.relpath(src_file, source_dir))
binary_flag = '' if filename.endswith('.java') else 'b'
with open(src_file, 'r' + binary_flag) as fp:
try:
contents = fp.read()
except Exception as ex: # pylint: disable=broad-except
mx.log('Error reading {}: {}'.format(src_file, ex))
if filename.endswith('.java'):
old_line_count = len(contents.split('\n'))
for old_name, new_name in package_renamings.items():
old_name_as_dir = old_name.replace('.', os.sep)
if old_name_as_dir in src_file:
new_name_as_dir = new_name.replace('.', os.sep)
dst = src_file.replace(old_name_as_dir, new_name_as_dir)
dst_file = join(target_dir, os.path.relpath(dst, source_dir))
contents = contents.replace(old_name, new_name)
for old_line, new_line in replacements.items():
contents = contents.replace(old_line, new_line)
contents = re.sub(trailing, '\n', contents)
match = java_package_re.search(contents)
if not match:
mx.abort('Could not find package declaration in {}'.format(src_file))
java_package = match.group('package')
if any(ex in java_package for ex in info.excludes):
mx.log(' excluding ' + filename)
continue
new_line_count = len(contents.split('\n'))
if new_line_count > old_line_count:
mx.abort('Pattern replacement caused line count to grow from {} to {} in {}'.format(old_line_count, new_line_count, src_file))
else:
if new_line_count < old_line_count:
contents = contents.replace('\npackage ', '\n' * (old_line_count - new_line_count) + '\npackage ')
new_line_count = len(contents.split('\n'))
if new_line_count != old_line_count:
mx.abort('Unable to correct line count for {}'.format(src_file))
for forbidden in denylist:
if forbidden in contents:
mx.abort('Found deny-listed pattern \'{}\' in {}'.format(forbidden, src_file))
dst_dir = os.path.dirname(dst_file)
if not exists(dst_dir):
os.makedirs(dst_dir)
if first_file:
mx.log(' copying: ' + source_dir)
mx.log(' to: ' + target_dir)
if p.testProject or p.definedAnnotationProcessors:
to_exclude = new_project_name
jdk_internal_vm_compiler_EXCLUDES.add(to_exclude)
if p.testProject:
jdk_internal_vm_compiler_test_SRC.add(to_exclude)
first_file = False
with open(dst_file, 'w' + binary_flag) as fp:
fp.write(contents)
def replace_line(filename, old_line, new_line):
mx.log('Updating ' + filename + '...')
old_lines = []
new_lines = []
with open(filename) as fp:
old_lines = fp.readlines()
for line in old_lines:
if line == old_line:
new_line = line.replace(old_line, new_line)
if old_line != new_line:
mx.log('Replaced \n ' + old_line.strip() + '\nwith\n ' + new_line.strip())
new_lines.append(line.replace(old_line, new_line))
with open(filename, 'w') as fp:
fp.writelines(new_lines)
def replace_lines(filename, begin_lines, end_line, replace_lines, old_line_check, preserve_indent=False, append_mode=False):
mx.log('Updating ' + filename + '...')
old_lines = []
new_lines = []
with open(filename) as fp:
for begin_line in begin_lines:
line = fp.readline()
while line:
stripped_line = line.strip()
if stripped_line == begin_line:
new_lines.append(line)
break
new_lines.append(line)
line = fp.readline()
assert line, begin_line + ' not found'
lines = fp.readlines()
line_in_def = True
indent = 0
if preserve_indent:
line = lines[0]
lstripped_line = line.lstrip()
indent = len(line) - len(lstripped_line)
if not append_mode:
for replace in replace_lines:
new_lines.append(' ' * indent + replace)
for line in lines:
stripped_line = line.strip()
if line_in_def:
if stripped_line == end_line:
line_in_def = False
new_lines.append(line)
else:
old_line_check(line)
if append_mode:
new_lines.append(line)
if append_mode and not line_in_def:
# reach end line and append new lines
for replace in replace_lines:
new_lines.append(replace)
else:
new_lines.append(line)
with open(filename, 'w') as fp:
for line in new_lines:
fp.write(line)
return old_lines
def single_column_with_continuation(line):
parts = line.split()
assert len(parts) == 2 and parts[1] == '\\', line
def do_nothing(line):
pass
# Exclude all test, benchmark and annotation processor packages.
if args.version >= 17:
# See JDK-8258407
CompileJavaModules_gmk = join(jdkrepo, 'make', 'modules', 'jdk.internal.vm.compiler', 'Java.gmk') # pylint: disable=invalid-name
begin_lines = ['EXCLUDES += \\']
else:
CompileJavaModules_gmk = join(jdkrepo, 'make', 'CompileJavaModules.gmk') # pylint: disable=invalid-name
begin_lines = ['jdk.internal.vm.compiler_EXCLUDES += \\']
new_lines = []
for pkg in sorted(jdk_internal_vm_compiler_EXCLUDES):
new_lines.append(pkg + ' \\\n')
end_line = '#'
old_line_check = single_column_with_continuation
replace_lines(CompileJavaModules_gmk, begin_lines, end_line, new_lines, old_line_check, preserve_indent=True)
# replace renamed service
compiler_module_info = join(jdkrepo, 'src', 'jdk.internal.vm.compiler', 'share', 'classes', 'module-info.java')
old_line = ' uses org.graalvm.compiler.nodes.graphbuilderconf.NodeIntrinsicPluginFactory;\n'
new_line = ' uses org.graalvm.compiler.nodes.graphbuilderconf.GeneratedPluginFactory;\n'
replace_line(compiler_module_info, old_line, new_line)
# Update 'SRC' in the 'Compile graalunit tests' section of make/test/JtregGraalUnit.gmk
# to include all test packages.
JtregGraalUnit_gmk = join(jdkrepo, 'make', 'test', 'JtregGraalUnit.gmk') # pylint: disable=invalid-name
new_lines = []
jdk_internal_vm_compiler_test_SRC.discard('org.graalvm.compiler.microbenchmarks')
jdk_internal_vm_compiler_test_SRC.discard('org.graalvm.compiler.virtual.bench')
jdk_internal_vm_compiler_test_SRC.discard('org.graalvm.micro.benchmarks')
for pkg in sorted(jdk_internal_vm_compiler_test_SRC):
new_lines.append('$(SRC_DIR)/' + pkg + '/src \\\n')
if args.version == 11:
begin_lines = ['### Compile and build graalunit tests', 'SRC := \\']
else:
begin_lines = ['### Compile graalunit tests', 'SRC := \\']
end_line = ', \\'
old_line_check = single_column_with_continuation
replace_lines(JtregGraalUnit_gmk, begin_lines, end_line, new_lines, old_line_check, preserve_indent=True)
# Updates for Metropolis
if args.metro:
# Update 'PROCESSOR_JARS' and 'PROC_SRC_SUBDIRS' in make/modules/jdk.internal.vm.compiler/Gensrc.gmk
Gensrc_gmk = join(jdkrepo, 'make', 'modules', 'jdk.internal.vm.compiler', 'Gensrc.gmk') # pylint: disable=invalid-name
begin_lines = ['PROC_SRC_SUBDIRS := \\']
end_line = 'org.graalvm.compiler.virtual \\'
new_lines = [' jdk.internal.vm.compiler.libgraal.jni \\\n']
replace_lines(Gensrc_gmk, begin_lines, end_line, new_lines, old_line_check, preserve_indent=True, append_mode=True)
begin_lines = ['PROCESSOR_JARS := \\']
end_line = '$(BUILDTOOLS_OUTPUTDIR)/jdk.vm.compiler.serviceprovider.processor.jar \\'
new_lines = [' $(BUILDTOOLS_OUTPUTDIR)/jdk.vm.compiler.libgraal.jni.processor.jar \\\n']
replace_lines(Gensrc_gmk, begin_lines, end_line, new_lines, old_line_check, preserve_indent=True, append_mode=True)
# Add 'libgraal.jni.processor' build to make/CompileToolsHotspot.gmk
CompileToolsHotspot_gmk = join(jdkrepo, 'make', 'CompileToolsHotspot.gmk') # pylint: disable=invalid-name
begin_lines = ['TARGETS += $(BUILD_VM_COMPILER_SERVICEPROVIDER_PROCESSOR)']
end_line = '##############################################################################'
new_lines = ['\n $(eval $(call SetupJavaCompilation, BUILD_VM_COMPILER_LIBGRAAL_JNI_PROCESSOR, \\\n',
' TARGET_RELEASE := $(TARGET_RELEASE_BOOTJDK), \\\n',
' SRC := \\\n',
' $(SRC_DIR)/org.graalvm.compiler.processor/src \\\n',
' $(SRC_DIR)/jdk.internal.vm.compiler.libgraal.jni.annotation/src \\\n',
' $(SRC_DIR)/jdk.internal.vm.compiler.libgraal.jni.processor/src \\\n',
' , \\\n',
' EXCLUDE_FILES := $(EXCLUDE_FILES), \\\n',
' BIN := $(BUILDTOOLS_OUTPUTDIR)/jdk.vm.compiler.libgraal.jni.processor, \\\n',
' JAR := $(BUILDTOOLS_OUTPUTDIR)/jdk.vm.compiler.libgraal.jni.processor.jar, \\\n',
' DISABLED_WARNINGS := options, \\\n',
' ))\n\n',
' TARGETS += $(BUILD_VM_COMPILER_LIBGRAAL_JNI_PROCESSOR)\n\n',
' ##############################################################################\n'] # indent is inlined
old_line_check = do_nothing
replace_lines(CompileToolsHotspot_gmk, begin_lines, end_line, new_lines, old_line_check, preserve_indent=True, append_mode=True)
# Rename packages in libgraal.jni.processor service
rename_packages(join(jdkrepo, 'src', 'jdk.internal.vm.compiler', 'share', 'classes', 'jdk.internal.vm.compiler.libgraal.jni.processor', 'src', 'META-INF', 'services', 'javax.annotation.processing.Processor'))
overwritten = ''
if not git_repo:
mx.log('Adding new files to HG...')
m_src_dirs = []
for m in graal_modules:
m_src_dirs.append(join('src', m.name))
out = run_output(['hg', 'log', '-r', 'last(keyword("Update Graal"))', '--template', '{rev}'] + m_src_dirs, cwd=jdkrepo)
last_graal_update = out.strip()
for m in graal_modules:
m_src_dir = join('src', m.name)
if last_graal_update:
overwritten += run_output(['hg', 'diff', '-r', last_graal_update, '-r', 'tip', m_src_dir], cwd=jdkrepo)
mx.run(['hg', 'add', m_src_dir], cwd=jdkrepo)
mx.log('Removing old files from HG...')
for m in graal_modules:
m_src_dir = join('src', m.name)
out = run_output(['hg', 'status', '-dn', m_src_dir], cwd=jdkrepo)
if out:
mx.run(['hg', 'rm'] + out.split(), cwd=jdkrepo)
out = run_output(['git', 'tag', '-l', 'JDK-*'], cwd=mx_compiler._suite.vc_dir)
last_jdk_tag = sorted(out.split(), reverse=True)[0]
pretty = args.pretty or 'format:%h %ad %>(20) %an %s'
out = run_output(['git', '--no-pager', 'log', '--merges', '--abbrev-commit', '--pretty=' + pretty, '--first-parent', '-r', last_jdk_tag + '..HEAD'] +
copied_source_dirs, cwd=mx_compiler._suite.vc_dir)
changes_file = 'changes-since-{}.txt'.format(last_jdk_tag)
with open(changes_file, 'w') as fp:
fp.write(out)
mx.log('Saved changes since {} to {}'.format(last_jdk_tag, os.path.abspath(changes_file)))
if overwritten:
overwritten_file = 'overwritten-diffs.txt'
with open(overwritten_file, 'w') as fp:
fp.write(overwritten)
mx.warn('Overwritten changes detected in OpenJDK Graal! See diffs in ' + os.path.abspath(overwritten_file))
def _fastr_gate_runner(args, tasks):
with mx_gate.Task('Setup no specials',
tasks,
tags=[FastRGateTags.no_specials]) as t:
if t:
os.environ['FASTR_OPTION_UseSpecials'] = 'false'
with mx_gate.Task('Setup no dsl cache',
tasks,
tags=[FastRGateTags.no_dsl_cache]) as t:
if t:
os.environ['FASTR_OPTION_DSLCacheSizeFactor'] = '0'
with mx_gate.Task('SetupLLVM', tasks, tags=[FastRGateTags.llvm]) as t:
if t:
os.environ['FASTR_RFFI'] = 'llvm'
with mx_gate.Task('GCTorture1', tasks,
tags=[FastRGateTags.gc_torture1]) as t:
if t:
os.environ['FASTR_GCTORTURE'] = '1'
with mx_gate.Task('GCTorture5', tasks,
tags=[FastRGateTags.gc_torture5]) as t:
if t:
os.environ['FASTR_GCTORTURE'] = '5'
'''
The specific additional gates tasks provided by FastR.
'''
with mx_gate.Task('ExtSoftVersions', tasks,
tags=[mx_gate.Tags.always]) as t:
if t:
new_env = os.environ.copy()
new_env['R_DEFAULT_PACKAGES'] = 'base'
run_r(['-q', '-e', 'extSoftVersion()'], 'R', env=new_env)
# ---------------------------------
# Style checks:
# FastR has custom copyright check
with mx_gate.Task('Copyright check', tasks,
tags=[mx_gate.Tags.style]) as t:
if t:
if mx.checkcopyrights(['--primary']) != 0:
t.abort('copyright errors')
# check that the expected test output file is up to date
with mx_gate.Task('UnitTests: ExpectedTestOutput file check',
tasks,
tags=[mx_gate.Tags.style]) as t:
if t:
mx_unittest.unittest([
'-Dfastr.test.gen.expected=' +
_test_srcdir(), '-Dfastr.test.check.expected=true'
] + _gate_unit_tests())
# ----------------------------------
# Basic tests:
with mx_gate.Task(
'UnitTests',
tasks,
tags=[FastRGateTags.basic_tests, FastRGateTags.unit_tests]) as t:
if t:
mx_unittest.unittest(_gate_noapps_unit_tests())
with mx_gate.Task('Rembedded', tasks,
tags=[FastRGateTags.basic_tests]) as t:
if t:
if rembedtest([]) != 0:
t.abort("Rembedded tests failed")
# ----------------------------------
# Package tests:
with mx_gate.Task('Recommended load test',
tasks,
tags=[FastRGateTags.recommended_load]) as t:
if t:
# Note: this is a convenience mx gate job for testing the loading of recommended packages
# We also test the loading of recommended pkgs in the "graalvm-tests"
if not os.path.exists(
os.path.join(_fastr_suite.dir, 'library', 'spatial')):
mx.abort(
'Recommended packages seem to be not installed in FastR. Did you forget to build with FASTR_RELEASE=true?'
)
pkgs = [
'codetools', 'MASS', 'boot', 'class', 'cluster', 'lattice',
'nnet', 'spatial', 'Matrix', 'KernSmooth', 'foreign', 'nlme',
'rpart', 'survival'
]
# TODO: removed failing "KernSmooth" and "cluster" on LLVM, GR-20406
if os.environ.get('FASTR_RFFI') == 'llvm':
pkgs.remove('KernSmooth')
pkgs.remove('cluster')
# Creates code that looks like: require(codetools) && require(MASS) && ...
require_stmts = ' && '.join(
['require(' + pkg + ')' for pkg in pkgs])
test_load = 'if (!(' + require_stmts + ')) q(status=1) else q(status=42)'
if run_r(['--vanilla', '-e', test_load], 'R',
nonZeroIsFatal=False) != 42:
mx.abort("Loading of recommended packages failed")
with mx_gate.Task('Internal pkg test',
tasks,
tags=[FastRGateTags.internal_pkgs_test]) as t:
if t:
list_file = os.path.join(
_fastr_suite.dir,
'com.oracle.truffle.r.test.native/packages/pkg-filelist')
if os.environ.get('FASTR_RFFI') == 'llvm':
list_file_llvm = list_file + '.llvm'
if os.path.exists(list_file_llvm):
list_file = list_file_llvm
if os.environ.get('FASTR_GCTORTURE') != '':
list_file_gctorture = list_file + '.gctorture'
if os.path.exists(list_file_gctorture):
list_file = list_file_gctorture
result = pkgtest(
["--verbose", "--repos", "FASTR", "--pkg-filelist", list_file])
if result != 0:
mx.abort("internal package test failed")
# CRAN packages are listed in files com.oracle.truffle.r.test.packages/gated0, gated1, ...
# We loop over all such files and crete gate task for each of them
# See also documentation in FastRGateTags.cran_pkgs_tests
for i in range(1, 1000):
list_file = os.path.join(
_fastr_suite.dir,
'com.oracle.truffle.r.test.packages/gated' + str(i))
if not os.path.exists(list_file):
break
with mx_gate.Task('CRAN pkg test: ' + str(i),
tasks,
tags=[FastRGateTags.cran_pkgs_test + str(i)]) as t:
if t:
check_last = False if mx_gate.Task.tags is None else FastRGateTags.cran_pkgs_test_check_last in mx_gate.Task.tags # pylint: disable=unsupported-membership-test
if check_last:
next_file = os.path.join(
_fastr_suite.dir,
'com.oracle.truffle.r.test.packages/gated' +
str(i + 1))
if os.path.exists(next_file):
mx.abort(
"File %s exists, but the gate thinks that %s is the last file. Did you forget to update the gate configuration?"
% (next_file, list_file))
cache_arg = os.environ.get('FASTR_PKGS_CACHE_OPT')
if cache_arg is None:
cache_arg = []
mx.warn(
"If you want to use R packages cache, export environment variable FASTR_PKGS_CACHE_OPT. See option '--cache-pkgs' of 'mx pkgtest' for the syntax."
)
else:
cache_arg = ['--cache-pkgs', cache_arg]
result = pkgtest(
["--verbose"] + cache_arg +
["--repos", "SNAPSHOT", "--pkg-filelist", list_file])
if result != 0:
mx.abort("package test failed")
def _sulong_gate_runner(args, tasks):
with Task('CheckCopyright', tasks, tags=['style']) as t:
if t:
if mx.checkcopyrights(['--primary']) != 0:
t.abort(
'Copyright errors found. Please run "mx checkcopyrights --primary -- --fix" to fix them.'
)
with Task('ClangFormat', tasks, tags=['style', 'clangformat']) as t:
if t: clangformatcheck()
_sulong_gate_testsuite('Benchmarks',
'shootout',
tasks,
args,
tags=['benchmarks', 'sulongMisc'])
_sulong_gate_unittest(
'Types',
'com.oracle.truffle.llvm.types.test',
tasks,
args,
tags=['type', 'sulongMisc', 'sulongCoverage'],
testClasses=['com.oracle.truffle.llvm.types.floating.test'])
_sulong_gate_unittest('Pipe',
'com.oracle.truffle.llvm.test',
tasks,
args,
tags=['pipe', 'sulongMisc', 'sulongCoverage'],
testClasses=['CaptureOutputTest'])
_sulong_gate_testsuite('LLVM',
'llvm',
tasks,
args,
tags=['llvm', 'sulongCoverage'])
_sulong_gate_testsuite('NWCC',
'nwcc',
tasks,
args,
tags=['nwcc', 'sulongCoverage'])
_sulong_gate_testsuite('GCCParserTorture',
'parserTorture',
tasks,
args,
tags=['parser', 'sulongCoverage'],
vmArgs=['-Dpolyglot.llvm.parseOnly=true'])
_sulong_gate_testsuite('GCC_C',
'gcc_c',
tasks,
args,
tags=['gcc_c', 'sulongCoverage'])
_sulong_gate_testsuite('GCC_CPP',
'gcc_cpp',
tasks,
args,
tags=['gcc_cpp', 'sulongCoverage'])
_sulong_gate_testsuite('GCC_Fortran',
'gcc_fortran',
tasks,
args,
tags=['gcc_fortran', 'sulongCoverage'])
_sulong_gate_sulongsuite_unittest(
'Sulong',
tasks,
args,
testClasses='SulongSuite',
tags=['sulong', 'sulongBasic', 'sulongCoverage'])
_sulong_gate_sulongsuite_unittest(
'Interop',
tasks,
args,
testClasses='com.oracle.truffle.llvm.test.interop',
tags=['interop', 'sulongBasic', 'sulongCoverage'])
_sulong_gate_sulongsuite_unittest(
'Debug',
tasks,
args,
testClasses='LLVMDebugTest',
tags=['debug', 'sulongBasic', 'sulongCoverage'])
_sulong_gate_sulongsuite_unittest(
'IRDebug',
tasks,
args,
testClasses='LLVMIRDebugTest',
tags=['irdebug', 'sulongBasic', 'sulongCoverage'])
_sulong_gate_sulongsuite_unittest(
'BitcodeFormat',
tasks,
args,
testClasses='BitcodeFormatTest',
tags=['bitcodeFormat', 'sulongBasic', 'sulongCoverage'])
_sulong_gate_sulongsuite_unittest(
'OtherTests',
tasks,
args,
testClasses='com.oracle.truffle.llvm.test.other',
tags=['otherTests', 'sulongBasic', 'sulongCoverage'])
_sulong_gate_testsuite('Assembly',
'inlineassemblytests',
tasks,
args,
testClasses='InlineAssemblyTest',
tags=['assembly', 'sulongCoverage'])
_sulong_gate_testsuite(
'Args',
'other',
tasks,
args,
tags=['args', 'sulongMisc', 'sulongCoverage'],
testClasses=['com.oracle.truffle.llvm.test.MainArgsTest'])
_sulong_gate_testsuite(
'Callback',
'other',
tasks,
args,
tags=['callback', 'sulongMisc', 'sulongCoverage'],
testClasses=['com.oracle.truffle.llvm.test.CallbackTest'])
_sulong_gate_testsuite(
'Varargs',
'other',
tasks,
args,
tags=['vaargs', 'sulongMisc', 'sulongCoverage'],
testClasses=['com.oracle.truffle.llvm.test.VAArgsTest'])
def _fastr_gate_runner(args, tasks):
with mx_gate.Task('Setup no specials',
tasks,
tags=[FastRGateTags.no_specials]) as t:
if t:
os.environ['FASTR_OPTION_UseSpecials'] = 'false'
with mx_gate.Task('Setup no dsl cache',
tasks,
tags=[FastRGateTags.no_dsl_cache]) as t:
if t:
os.environ['FASTR_OPTION_DSLCacheSizeFactor'] = '0'
with mx_gate.Task('SetupLLVM', tasks, tags=[FastRGateTags.llvm]) as t:
if t:
os.environ['FASTR_RFFI'] = 'llvm'
with mx_gate.Task('GCTorture1', tasks,
tags=[FastRGateTags.gc_torture1]) as t:
if t:
os.environ['FASTR_GCTORTURE'] = '1'
with mx_gate.Task('GCTorture3', tasks,
tags=[FastRGateTags.gc_torture3]) as t:
if t:
os.environ['FASTR_GCTORTURE'] = '3'
with mx_gate.Task('VerySlowAsserts',
tasks,
tags=[FastRGateTags.very_slow_asserts]) as t:
if t:
os.environ['FASTR_TEST_VERY_SLOW_ASSERTS'] = 'true'
'''
The specific additional gates tasks provided by FastR.
'''
with mx_gate.Task('ExtSoftVersions', tasks,
tags=[mx_gate.Tags.always]) as t:
if t:
new_env = os.environ.copy()
new_env['R_DEFAULT_PACKAGES'] = 'base'
run_r(['-q', '-e', 'extSoftVersion()'], 'R', env=new_env)
with mx_gate.Task('LibsInfo', tasks, tags=[mx_gate.Tags.always]) as t:
if t:
mx.log("Libraries captured in FASTR_HOME/lib:")
lib_dir = os.path.join(_fastr_suite.dir, 'lib')
ldd = ['otool', '-L'] if platform.system() == 'Darwin' else ['ldd']
for f in os.listdir(lib_dir):
full_path = os.path.join(lib_dir, f)
mx.run(['file', full_path], nonZeroIsFatal=False)
mx.log('---\nobjdump:')
mx.run(['objdump', '-s', '--section', '.comment', full_path],
nonZeroIsFatal=False)
mx.log('---\nlinking info:')
mx.run(ldd + [full_path], nonZeroIsFatal=False)
mx.log('---------')
# ---------------------------------
# Style checks:
# FastR has custom copyright check
with mx_gate.Task('Copyright check', tasks,
tags=[mx_gate.Tags.style]) as t:
if t:
if mx.checkcopyrights(['--primary']) != 0:
t.abort('copyright errors')
# check that the expected test output file is up to date
with mx_gate.Task('UnitTests: ExpectedTestOutput file check',
tasks,
tags=[mx_gate.Tags.style]) as t:
if t:
mx_unittest.unittest([
'-Dfastr.test.gen.expected=' +
_test_srcdir(), '-Dfastr.test.check.expected=true'
] + _gate_unit_tests())
# ----------------------------------
# Basic tests:
with mx_gate.Task(
'UnitTests',
tasks,
tags=[FastRGateTags.basic_tests, FastRGateTags.unit_tests]) as t:
if t:
mx_unittest.unittest(_gate_noapps_unit_tests())
with mx_gate.Task('Rembedded', tasks,
tags=[FastRGateTags.basic_tests]) as t:
if t:
if rembedtest([]) != 0:
t.abort("Rembedded tests failed")
# ----------------------------------
# Package tests:
with mx_gate.Task('Recommended load test',
tasks,
tags=[FastRGateTags.recommended_load]) as t:
if t:
# Note: this is a convenience mx gate job for testing the loading of recommended packages
# We also test the loading of recommended pkgs in the "graalvm-tests"
if not os.path.exists(
os.path.join(_fastr_suite.dir, 'library', 'spatial')):
mx.abort(
'Recommended packages seem to be not installed in FastR. Did you forget to build with FASTR_RELEASE=true?'
)
pkgs = [
'codetools', 'MASS', 'boot', 'class', 'cluster', 'lattice',
'nnet', 'spatial', 'Matrix', 'KernSmooth', 'foreign', 'nlme',
'rpart', 'survival'
]
# Creates code that looks like: require(codetools) && require(MASS) && ...
require_stmts = ' && '.join(
['require(' + pkg + ')' for pkg in pkgs])
test_load = 'if (!(' + require_stmts + ')) q(status=1) else q(status=42)'
if run_r(['--vanilla', '-e', test_load], 'R',
nonZeroIsFatal=False) != 42:
mx.abort("Loading of recommended packages failed")
with mx_gate.Task('Internal pkg test',
tasks,
tags=[FastRGateTags.internal_pkgs_test]) as t:
if t:
internal_pkg_tests()
# CRAN packages are listed in files com.oracle.truffle.r.test.packages/gated0, gated1, ...
# We loop over all such files and crete gate task for each of them
# See also documentation in FastRGateTags.cran_pkgs_tests
for i in range(1, 1000):
list_file = os.path.join(
_fastr_suite.dir,
'com.oracle.truffle.r.test.packages/gated' + str(i))
if not os.path.exists(list_file):
break
with mx_gate.Task('CRAN pkg test: ' + str(i),
tasks,
tags=[FastRGateTags.cran_pkgs_test + str(i)]) as t:
if t:
check_last = False if mx_gate.Task.tags is None else FastRGateTags.cran_pkgs_test_check_last in mx_gate.Task.tags # pylint: disable=unsupported-membership-test
if check_last:
next_file = os.path.join(
_fastr_suite.dir,
'com.oracle.truffle.r.test.packages/gated' +
str(i + 1))
if os.path.exists(next_file):
mx.abort(
"File %s exists, but the gate thinks that %s is the last file. Did you forget to update the gate configuration?"
% (next_file, list_file))
cran_pkg_tests(list_file)
