def checkcopyrights(args): """run copyright check on the Maxine sources""" for i in args: if i == '-h' or i == '-help' or i == '--help': print '"for help run mx :checkcopyrights -h"' return mx.checkcopyrights(['--', '--copyright-dir', 'mx.maxine'] + args)
def _sdk_gate_runner(args, tasks): with Task('SDK UnitTests', tasks, tags=['test']) as t: if t: unittest([ '--suite', 'sdk', '--enable-timing', '--verbose', '--fail-fast' ]) with Task('Check Copyrights', tasks) as t: if t: mx.checkcopyrights(['--primary'])
def _truffle_gate_runner(args, tasks): jdk = mx.get_jdk(tag=mx.DEFAULT_JDK_TAG) with Task('Jackpot check', tasks) as t: if t: jackpot(['--fail-on-warnings'], suite=None, nonZeroIsFatal=True) if jdk.javaCompliance < '9': with Task('Truffle Javadoc', tasks) as t: if t: javadoc([]) with Task('Truffle UnitTests', tasks) as t: if t: unittest(['--suite', 'truffle', '--enable-timing', '--verbose', '--fail-fast']) with Task('Truffle Signature Tests', tasks) as t: if t: sigtest(['--check', 'binary']) with Task('File name length check', tasks) as t: if t: check_filename_length([]) with Task('Check Copyrights', tasks) as t: if t: mx.checkcopyrights(['--primary'])
def _sdk_gate_runner(args, tasks): with Task('SDK UnitTests', tasks, tags=['test']) as t: if t: unittest(['--suite', 'sdk', '--enable-timing', '--verbose', '--fail-fast']) with Task('Check Copyrights', tasks) as t: if t: if mx.checkcopyrights(['--primary', '--', '--projects', 'src']) != 0: t.abort('Copyright errors found. Please run "mx checkcopyrights --primary -- --fix" to fix them.')
def _sulong_gate_runner(args, tasks): with Task('CheckCopyright', tasks, tags=['style']) as t: if t: if mx.checkcopyrights(['--primary']) != 0: t.abort('Copyright errors found. Please run "mx checkcopyrights --primary -- --fix" to fix them.') with Task('BuildLLVMorg', tasks, tags=['style', 'clangformat']) as t: # needed for clang-format if t: build_llvm_org(args) with Task('ClangFormat', tasks, tags=['style', 'clangformat']) as t: if t: clangformatcheck() _sulong_gate_testsuite('Benchmarks', 'shootout', tasks, args, tags=['benchmarks', 'sulongMisc']) _sulong_gate_unittest('Types', 'SULONG_TEST', tasks, args, tags=['type', 'sulongMisc', 'sulongCoverage'], testClasses=['com.oracle.truffle.llvm.tests.types.floating']) _sulong_gate_unittest('Pipe', 'SULONG_TEST', tasks, args, tags=['pipe', 'sulongMisc', 'sulongCoverage'], testClasses=['CaptureOutputTest']) _sulong_gate_testsuite('LLVM', 'llvm', tasks, args, tags=['llvm', 'sulongCoverage']) _sulong_gate_testsuite('NWCC', 'nwcc', tasks, args, tags=['nwcc', 'sulongCoverage']) _sulong_gate_testsuite('GCCParserTorture', 'parserTorture', tasks, args, tags=['parser', 'sulongCoverage'], vmArgs=['-Dpolyglot.llvm.parseOnly=true']) _sulong_gate_testsuite('GCC_C', 'gcc_c', tasks, args, tags=['gcc_c', 'sulongCoverage']) _sulong_gate_testsuite('GCC_CPP', 'gcc_cpp', tasks, args, tags=['gcc_cpp', 'sulongCoverage']) _sulong_gate_testsuite('GCC_Fortran', 'gcc_fortran', tasks, args, tags=['gcc_fortran', 'sulongCoverage']) _sulong_gate_sulongsuite_unittest('Sulong', tasks, args, testClasses='SulongSuite', tags=['sulong', 'sulongBasic', 'sulongCoverage']) _sulong_gate_sulongsuite_unittest('Interop', tasks, args, testClasses='com.oracle.truffle.llvm.tests.interop', tags=['interop', 'sulongBasic', 'sulongCoverage']) _sulong_gate_sulongsuite_unittest('Linker', tasks, args, testClasses='com.oracle.truffle.llvm.tests.linker', tags=['linker', 'sulongBasic', 'sulongCoverage']) _sulong_gate_sulongsuite_unittest('Debug', tasks, args, testClasses='LLVMDebugTest', tags=['debug', 'sulongBasic', 'sulongCoverage']) _sulong_gate_sulongsuite_unittest('IRDebug', tasks, args, testClasses='LLVMIRDebugTest', tags=['irdebug', 'sulongBasic', 'sulongCoverage']) _sulong_gate_sulongsuite_unittest('BitcodeFormat', tasks, args, testClasses='BitcodeFormatTest', tags=['bitcodeFormat', 'sulongBasic', 'sulongCoverage']) _sulong_gate_sulongsuite_unittest('OtherTests', tasks, args, testClasses='com.oracle.truffle.llvm.tests.other', tags=['otherTests', 'sulongBasic', 'sulongCoverage']) _sulong_gate_testsuite('Assembly', 'inlineassemblytests', tasks, args, testClasses='InlineAssemblyTest', tags=['assembly', 'sulongMisc', 'sulongCoverage']) _sulong_gate_testsuite('Args', 'other', tasks, args, tags=['args', 'sulongMisc', 'sulongCoverage'], testClasses=['com.oracle.truffle.llvm.tests.MainArgsTest']) _sulong_gate_testsuite('Callback', 'other', tasks, args, tags=['callback', 'sulongMisc', 'sulongCoverage'], testClasses=['com.oracle.truffle.llvm.tests.CallbackTest']) _sulong_gate_testsuite('Varargs', 'other', tasks, args, tags=['vaargs', 'sulongMisc', 'sulongCoverage'], testClasses=['com.oracle.truffle.llvm.tests.VAArgsTest']) with Task('TestToolchain', tasks, tags=['toolchain', 'sulongMisc', 'sulongCoverage']) as t: if t: mx.command_function('clean')(['--project', 'toolchain-launchers-tests'] + args.extra_build_args) mx.command_function('build')(['--project', 'toolchain-launchers-tests'] + args.extra_build_args)
def _fastr_gate_runner(args, tasks): ''' The specific additional gates tasks provided by FastR: 1. Copyright check 2. Check that ExpectedTestOutput file is in sync with unit tests 3. Unit tests ''' # FastR has custom copyright check with mx_gate.Task('Copyright check', tasks) as t: if t: if mx.checkcopyrights(['--primary']) != 0: t.abort('copyright errors') # check that the expected test output file is up to date with mx_gate.Task('UnitTests: ExpectedTestOutput file check', tasks) as t: if t: mx_unittest.unittest(['-Dfastr.test.gen.expected=' + _test_srcdir(), '-Dfastr.test.check.expected'] + _gate_unit_tests()) with mx_gate.Task('UnitTests: no specials', tasks) as t: if t: mx_unittest.unittest(['-Dfastr.test.options.R.UseSpecials=false'] + _gate_noapps_unit_tests()) with mx_gate.Task('UnitTests: with specials', tasks) as t: if t: mx_unittest.unittest(_gate_noapps_unit_tests()) with mx_gate.Task('UnitTests: apps', tasks) as t: if t: mx_unittest.unittest(_apps_unit_tests()) with mx_gate.Task('Rembedded', tasks) as t: if t: if rembedtest([]) != 0: t.abort("Rembedded tests failed")
def _add_copyright_checks(tasks): with mx_gate.Task('Check Copyrights', tasks, tags=[mx_gate.Tags.style]) as t: if t: if mx.checkcopyrights(['--primary']) != 0: t.abort('Copyright errors found. Please run "mx ' 'checkcopyrights --primary -- --fix" to fix them.')
def _fastr_gate_runner(args, tasks): ''' The specific additional gates tasks provided by FastR: 1. Copyright check 2. Check that ExpectedTestOutput file is in sync with unit tests 3. Unit tests ''' # FastR has custom copyright check with mx_gate.Task('Copyright check', tasks) as t: if t: if mx.checkcopyrights(['--primary']) != 0: t.abort('copyright errors') # check that the expected test output file is up to date with mx_gate.Task('UnitTests: ExpectedTestOutput file check', tasks) as t: if t: if junit([ '--tests', _gate_noapps_unit_tests(), '--check-expected-output' ]) != 0: t.abort('unit tests expected output check failed') with mx_gate.Task('UnitTests', tasks) as t: if t: if junit(['--tests', _gate_noapps_unit_tests()]) != 0: t.abort('unit tests failed')
def _graal_js_gate_runner(args, tasks): with Task('CheckCopyrights', tasks, tags=['style']) as t: if t: if mx.checkcopyrights(['--primary']) != 0: t.abort( 'Copyright errors found. Please run "mx checkcopyrights --primary -- --fix" to fix them.' ) with Task('TestJSCommand', tasks, tags=[GraalJsDefaultTags.default, GraalJsDefaultTags.all]) as t: if t: js(['-Dpolyglot.js.profile-time=true', '-e', '""']) with Task('UnitTests', tasks, tags=[GraalJsDefaultTags.default, GraalJsDefaultTags.all]) as t: if t: unittest( ['--enable-timing', '--very-verbose', '--suite', _suite.name]) gateTestConfigs = { GraalJsDefaultTags.default: ['gate'], 'noic': [ '-Dpolyglot.js.property-cache-limit=0', '-Dpolyglot.js.function-cache-limit=0', 'gate' ], 'directbytebuffer': ['-Dpolyglot.js.direct-byte-buffer=true', 'gate'], 'cloneuninitialized': ['-Dpolyglot.js.test-clone-uninitialized=true', 'gate'], 'lazytranslation': ['-Dpolyglot.js.lazy-translation=true', 'gate'], 'shareengine': ['gate', 'shareengine'], } gateTestCommands = { 'Test262': test262, 'TestNashorn': testnashorn, 'TestV8': testv8, } for testCommandName in gateTestCommands: for testConfigName in gateTestConfigs: testName = '%s-%s' % (testCommandName, testConfigName) with Task(testName, tasks, tags=[testName, testConfigName, GraalJsDefaultTags.all]) as t: if t: gateTestCommands[testCommandName]( gateTestConfigs[testConfigName]) with Task('TCK tests', tasks, tags=[GraalJsDefaultTags.all, GraalJsDefaultTags.tck]) as t: if t: import mx_truffle mx_truffle._tck([])
def _graal_js_gate_runner(args, tasks): with Task('CheckCopyrights', tasks, tags=['style']) as t: if t: if mx.checkcopyrights(['--primary']) != 0: t.abort('Copyright errors found. Please run "mx checkcopyrights --primary -- --fix" to fix them.') with Task('TestJSCommand', tasks, tags=[GraalJsDefaultTags.default, GraalJsDefaultTags.all]) as t: if t: js(['-Dpolyglot.js.profile-time=true', '-e', '""']) webassemblyTestSuite = 'com.oracle.truffle.js.test.suite.WebAssemblySimpleTestSuite' with Task('UnitTests', tasks, tags=[GraalJsDefaultTags.default, GraalJsDefaultTags.all]) as t: if t: noWebAssemblyTestSuite = '^(?!' + webassemblyTestSuite + ')' commonOptions = ['--enable-timing', '--very-verbose', '--suite', _suite.name] unittest(['--regex', noWebAssemblyTestSuite] + commonOptions) unittest(['--regex', 'ZoneRulesProviderTest', '-Djava.time.zone.DefaultZoneRulesProvider=com.oracle.truffle.js.test.runtime.SimpleZoneRulesProvider'] + commonOptions) with Task('WebAssemblyTests', tasks, tags=['webassembly', GraalJsDefaultTags.all]) as t: if t: unittest(['--regex', webassemblyTestSuite, '--enable-timing', '--very-verbose', '--suite', _suite.name]) gateTestConfigs = { GraalJsDefaultTags.default: ['gate'], 'noic': ['-Dpolyglot.js.property-cache-limit=0', '-Dpolyglot.js.function-cache-limit=0', 'gate'], 'directbytebuffer': ['-Dpolyglot.js.direct-byte-buffer=true', 'gate'], 'cloneuninitialized': ['-Dpolyglot.js.test-clone-uninitialized=true', 'gate'], 'lazytranslation': ['-Dpolyglot.js.lazy-translation=true', 'gate'], 'zonerulesbasedtimezones': ['-Dpolyglot.js.zone-rules-based-time-zones=true', 'gate'], 'shareengine': ['gate', 'shareengine'], 'latestversion': ['gate', 'minesversion=2022'], 'instrument': ['gate', 'instrument', 'timeoutoverall=1800'] } gateTestCommands = { 'Test262': test262, 'TestNashorn': testnashorn, 'TestV8': testv8, } for testCommandName in gateTestCommands: for testConfigName in gateTestConfigs: # TestNashorn is not sensitive to ES version if testCommandName == 'TestNashorn' and testConfigName == 'latestversion': continue testName = '%s-%s' % (testCommandName, testConfigName) with Task(testName, tasks, tags=[testName, testConfigName, GraalJsDefaultTags.all]) as t: if t: gateTestCommands[testCommandName](gateTestConfigs[testConfigName]) with Task('TCK tests', tasks, tags=[GraalJsDefaultTags.all, GraalJsDefaultTags.tck]) as t: if t: import mx_truffle mx_truffle._tck([])
def checkCopyrightHeaders(args): # create list of Java sources to check with open('files_to_check', 'w') as f: mx.run(['git', 'ls-files', '--', '*.java'], out=f) # create an overrides file so that NodeProf copyright can be checked with open(join(_suite.dir, 'mx.nodeprof/copyrights/overrides'), 'w') as f: mx.run(['awk', '{print $0",nodeprof.copyright"}', 'files_to_check'], out=f) # run the actual check (overrodes file needs to be passed explicitly) return mx.checkcopyrights( ['--primary', '--', '--file-list', 'files_to_check'])
def _sulong_gate_runner(args, tasks): with TemporaryEnv(): with Task('CheckCopyright', tasks, tags=['style']) as t: if t: if mx.checkcopyrights(['--primary']) != 0: t.abort( 'Copyright errors found. Please run "mx checkcopyrights --primary -- --fix" to fix them.' ) with Task('ClangFormat', tasks, tags=['style', 'clangformat']) as t: if t: clangformatcheck() with Task('TestBenchmarks', tasks, tags=['benchmarks', 'sulongMisc']) as t: if t: mx_testsuites.runSuite('shootout') with Task('TestTypes', tasks, tags=['type', 'sulongMisc']) as t: if t: mx_testsuites.runSuite('type') with Task('TestPipe', tasks, tags=['pipe', 'sulongMisc']) as t: if t: mx_testsuites.runSuite('pipe') with Task('TestLLVM', tasks, tags=['llvm']) as t: if t: mx_testsuites.runSuite('llvm') with Task('TestNWCC', tasks, tags=['nwcc']) as t: if t: mx_testsuites.runSuite('nwcc') with Task('TestGCCParserTorture', tasks, tags=['parser']) as t: if t: mx_testsuites.runSuite('parserTorture') with Task('TestGCC_C', tasks, tags=['gcc_c']) as t: if t: mx_testsuites.runSuite('gcc_c') with Task('TestGCC_CPP', tasks, tags=['gcc_cpp']) as t: if t: mx_testsuites.runSuite('gcc_cpp') with Task('TestGCC_Fortran', tasks, tags=['gcc_fortran']) as t: if t: mx_testsuites.runSuite('gcc_fortran') with Task("TestSulong", tasks, tags=['sulong', 'sulongBasic']) as t: if t: mx_unittest.unittest(['SulongSuite']) with Task("TestInterop", tasks, tags=['interop', 'sulongBasic']) as t: if t: mx_unittest.unittest(['com.oracle.truffle.llvm.test.interop']) with Task("TestDebug", tasks, tags=['debug', 'sulongBasic']) as t: if t: mx_unittest.unittest(['LLVMDebugTest']) with Task("TestIRDebug", tasks, tags=['irdebug', 'sulongBasic']) as t: if t: mx_unittest.unittest(['LLVMIRDebugTest']) with Task('TestAssembly', tasks, tags=['assembly', 'sulongMisc']) as t: if t: mx_testsuites.runSuite('assembly') with Task('TestArgs', tasks, tags=['args', 'sulongMisc']) as t: if t: mx_testsuites.runSuite('args') with Task('TestCallback', tasks, tags=['callback', 'sulongMisc']) as t: if t: mx_testsuites.runSuite('callback') with Task('TestVarargs', tasks, tags=['vaargs', 'sulongMisc']) as t: if t: mx_testsuites.runSuite('vaargs')
def _sulong_gate_runner(args, tasks): _unittest_task_factory = UnittestTaskFactory() def _unittest(title, test_suite, tags=None, testClasses=None, unittestArgs=None, description=None): _unittest_task_factory.add(title, test_suite, args, tags=tags, testClasses=testClasses, unittestArgs=unittestArgs, description=description) with Task('CheckCopyright', tasks, tags=['style']) as t: if t: if mx.checkcopyrights(['--primary']) != 0: t.abort('Copyright errors found. Please run "mx checkcopyrights --primary -- --fix" to fix them.') with Task('BuildLLVMorg', tasks, tags=['style', 'clangformat']) as t: # needed for clang-format if t: build_llvm_org(args) with Task('ClangFormat', tasks, tags=['style', 'clangformat']) as t: if t: clangformat([]) # Folders not containing tests: options, services, util _unittest('Benchmarks', 'SULONG_SHOOTOUT_TEST_SUITE', description="Language Benchmark game tests", testClasses=['ShootoutsSuite'], tags=['benchmarks', 'sulongMisc']) _unittest('Types', 'SULONG_TEST', description="Test floating point arithmetic", testClasses=['com.oracle.truffle.llvm.tests.types.floating.'], tags=['type', 'sulongMisc', 'sulongCoverage']) _unittest('Pipe', 'SULONG_TEST', description="Test output capturing", testClasses=['CaptureOutputTest'], tags=['pipe', 'sulongMisc', 'sulongCoverage']) _unittest('LLVM', 'SULONG_LLVM_TEST_SUITE', description="LLVM 3.2 test suite", testClasses=['LLVMSuite'], tags=['llvm', 'sulongCoverage']) _unittest('NWCC', 'SULONG_NWCC_TEST_SUITE', description="Test suite of the NWCC compiler v0.8.3", testClasses=['NWCCSuite'], tags=['nwcc', 'sulongCoverage']) _unittest('GCCParserTorture', 'SULONG_PARSER_TORTURE', description="Parser test using GCC suite", testClasses=['ParserTortureSuite'], tags=['parser', 'sulongCoverage']) _unittest('GCC_C', 'SULONG_GCC_C_TEST_SUITE', description="GCC 5.2 test suite (C tests)", testClasses=['GccCSuite'], tags=['gcc_c', 'sulongCoverage']) _unittest('GCC_CPP', 'SULONG_GCC_CPP_TEST_SUITE', description="GCC 5.2 test suite (C++ tests)", testClasses=['GccCppSuite'], tags=['gcc_cpp', 'sulongCoverage']) _unittest('GCC_Fortran', 'SULONG_GCC_FORTRAN_TEST_SUITE', description="GCC 5.2 test suite (Fortran tests)", testClasses=['GccFortranSuite'], tags=['gcc_fortran', 'sulongCoverage']) _unittest('Sulong', 'SULONG_STANDALONE_TEST_SUITES', description="Sulong's internal tests", testClasses='SulongSuite', tags=['sulong', 'sulongBasic', 'sulongCoverage']) _unittest('Interop', 'SULONG_EMBEDDED_TEST_SUITES', description="Truffle Language interoperability tests", testClasses=['com.oracle.truffle.llvm.tests.interop.'], tags=['interop', 'sulongBasic', 'sulongCoverage']) _unittest('Linker', 'SULONG_EMBEDDED_TEST_SUITES', description=None, testClasses=['com.oracle.truffle.llvm.tests.linker.'], tags=['linker', 'sulongBasic', 'sulongCoverage']) _unittest('Debug', 'SULONG_EMBEDDED_TEST_SUITES', description="Debug support test suite", testClasses=['com.oracle.truffle.llvm.tests.debug.LLVMDebugTest'], tags=['debug', 'sulongBasic', 'sulongCoverage']) _unittest('IRDebug', 'SULONG_EMBEDDED_TEST_SUITES', description=None, testClasses=['com.oracle.truffle.llvm.tests.debug.LLVMIRDebugTest'], tags=['irdebug', 'sulongBasic', 'sulongCoverage']) _unittest('BitcodeFormat', 'SULONG_EMBEDDED_TEST_SUITES', description=None, testClasses=['com.oracle.truffle.llvm.tests.bitcodeformat.'], tags=['bitcodeFormat', 'sulongBasic', 'sulongCoverage']) _unittest('DebugExpr', 'SULONG_EMBEDDED_TEST_SUITES', description=None, testClasses=['com.oracle.truffle.llvm.tests.debug.LLVMDebugExprParserTest'], tags=['debugexpr', 'sulongBasic', 'sulongCoverage']) _unittest('OtherTests', 'SULONG_EMBEDDED_TEST_SUITES', description=None, testClasses=['com.oracle.truffle.llvm.tests.bitcode.', 'com.oracle.truffle.llvm.tests.other.', 'com.oracle.truffle.llvm.tests.runtime.'], tags=['otherTests', 'sulongBasic', 'sulongCoverage']) _unittest('Args', 'SULONG_EMBEDDED_TEST_SUITES', description="Tests main args passing", testClasses=['com.oracle.truffle.llvm.tests.MainArgsTest'], tags=['args', 'sulongMisc', 'sulongCoverage']) _unittest('Callback', 'SULONG_EMBEDDED_TEST_SUITES', description="Test calling native functions", testClasses=['com.oracle.truffle.llvm.tests.CallbackTest'], tags=['callback', 'sulongMisc', 'sulongCoverage']) _unittest('Varargs', 'SULONG_EMBEDDED_TEST_SUITES', description="Varargs tests", testClasses=['com.oracle.truffle.llvm.tests.VAArgsTest'], tags=['vaargs', 'sulongMisc', 'sulongCoverage']) _unittest_task_factory.execute(tasks) with Task('TestToolchain', description="build toolchain-launchers-tests project", tags=['toolchain', 'sulongMisc', 'sulongCoverage'], tasks=tasks) as t: if t: with SulongGateEnv(): mx.command_function('clean')(['--project', 'toolchain-launchers-tests'] + args.extra_build_args) mx.command_function('build')(['--project', 'toolchain-launchers-tests'] + args.extra_build_args)
def updategraalinopenjdk(args): """updates the Graal sources in OpenJDK""" parser = ArgumentParser(prog='mx updategraalinopenjdk') parser.add_argument('--pretty', help='value for --pretty when logging the changes since the last JDK* tag') parser.add_argument('jdkrepo', help='path to the local OpenJDK repo') parser.add_argument('version', type=int, help='Java version of the OpenJDK repo') parser.add_argument('--metro', action='store_true', help='update for Metropolis project (to build libgraal)') args = parser.parse_args(args) if mx_compiler.jdk.javaCompliance.value < args.version: mx.abort('JAVA_HOME/--java-home must be Java version {} or greater: {}'.format(args.version, mx_compiler.jdk)) if not args.metro: mx.log('Processing update for OpenJDK {}'.format(args.version)) graal_modules = [ # JDK module jdk.internal.vm.compiler is composed of sources from: GraalJDKModule('jdk.internal.vm.compiler', # 1. Classes in the compiler suite under the org.graalvm namespace except for packages # or projects whose names contain terms on the specified exclude list [SuiteJDKInfo('compiler', ['org.graalvm'], ['truffle', 'management', 'core.llvm', 'replacements.llvm', 'libgraal.jni']), # 2. Classes in the sdk suite under the org.graalvm.collections and org.graalvm.word namespaces SuiteJDKInfo('sdk', ['org.graalvm.collections', 'org.graalvm.word'], [])]), # JDK module jdk.internal.vm.compiler.management is composed of sources from: GraalJDKModule('jdk.internal.vm.compiler.management', # 1. Classes in the compiler suite under the org.graalvm.compiler.hotspot.management namespace [SuiteJDKInfo('compiler', ['org.graalvm.compiler.hotspot.management'], ['libgraal'])]), ] else: if args.version < 15: mx.abort('Update for Metropolis works only for JDK 15 or later') mx.log('Processing update for Metropolis JDK {}'.format(args.version)) # Metropolis needs more code from Graal graal_modules = [ # JDK module jdk.internal.vm.compiler is composed of sources from: GraalJDKModule('jdk.internal.vm.compiler', # 1. Classes in the compiler suite under the org.graalvm namespace except for packages # or projects whose names contain terms on the specified exclude list [SuiteJDKInfo('compiler', ['org.graalvm'], ['truffle', 'management', 'core.llvm', 'replacements.llvm']), # 2. Classes in the sdk suite under the org.graalvm.collections and org.graalvm.word namespaces SuiteJDKInfo('sdk', ['org.graalvm.collections', 'org.graalvm.word', 'org.graalvm.nativeimage', 'org.graalvm.options'], [])]), # JDK module jdk.internal.vm.compiler.management is composed of sources from: GraalJDKModule('jdk.internal.vm.compiler.management', # 1. Classes in the compiler suite under the org.graalvm.compiler.hotspot.management namespace [SuiteJDKInfo('compiler', ['org.graalvm.compiler.hotspot.management'], ['libgraal'])]), ] # Strings to be replaced in files copied to OpenJDK. replacements = { 'published by the Free Software Foundation. Oracle designates this\n * particular file as subject to the "Classpath" exception as provided\n * by Oracle in the LICENSE file that accompanied this code.' : 'published by the Free Software Foundation.', _read_sibling_file('upl_substring.txt') : _read_sibling_file('gplv2_substring.txt') } # Strings that must not exist in OpenJDK source files. This is applied after replacements are made. denylist = ['"Classpath" exception'] jdkrepo = args.jdkrepo git_repo = _is_git_repo(jdkrepo) for m in graal_modules: m_src_dir = join(jdkrepo, 'src', m.name) if not exists(m_src_dir): mx.abort(jdkrepo + ' does not look like a JDK repo - ' + m_src_dir + ' does not exist') def run_output(args, cwd=None): out = mx.OutputCapture() mx.run(args, cwd=cwd, out=out, err=out) return out.data for m in graal_modules: m_src_dir = join('src', m.name) mx.log('Checking ' + m_src_dir) if git_repo: out = run_output(['git', 'status', '-s', m_src_dir], cwd=jdkrepo) else: out = run_output(['hg', 'status', m_src_dir], cwd=jdkrepo) if out: mx.abort(jdkrepo + ' is not "clean":' + '\n' + out[:min(200, len(out))] + '...') mx.checkcopyrights(['--primary', '--', '--fix']) for dirpath, _, filenames in os.walk(join(jdkrepo, 'make')): for filename in filenames: if filename.endswith('.gmk'): rename_packages(join(dirpath, filename), True) java_package_re = re.compile(r"^\s*package\s+(?P<package>[a-zA-Z_][\w\.]*)\s*;$", re.MULTILINE) copied_source_dirs = [] jdk_internal_vm_compiler_EXCLUDES = set() # pylint: disable=invalid-name jdk_internal_vm_compiler_test_SRC = set() # pylint: disable=invalid-name # Add org.graalvm.compiler.processor since it is only a dependency # for (most) Graal annotation processors and is not needed to # run Graal. jdk_internal_vm_compiler_EXCLUDES.add('org.graalvm.compiler.processor') for m in graal_modules: classes_dir = join(jdkrepo, 'src', m.name, 'share', 'classes') for info in m.suites: mx.log('Processing ' + m.name + ':' + info.name) for e in os.listdir(classes_dir): if any(inc in e for inc in info.includes) and not any(ex in e for ex in info.excludes): project_dir = join(classes_dir, e) shutil.rmtree(project_dir) mx.log(' removed ' + project_dir) suite = mx.suite(info.name) worklist = [] for p in [e for e in suite.projects if e.isJavaProject()]: if str(mx_compiler.jdk.javaCompliance) not in p.javaCompliance: mx.log(' skipping {} since its compliance ({}) is not compatible with {}'.format(p, repr(p.javaCompliance), mx_compiler.jdk.javaCompliance)) continue if any(inc in p.name for inc in info.includes) and not any(ex in p.name for ex in info.excludes): assert len(p.source_dirs()) == 1, p version = 0 new_project_name = p.name if hasattr(p, 'multiReleaseJarVersion'): version = int(getattr(p, 'multiReleaseJarVersion')) if version <= args.version: base_project = _find_version_base_project(p) new_project_name = base_project.name else: continue for old_name, new_name in package_renamings.items(): if new_project_name.startswith(old_name): new_project_name = new_project_name.replace(old_name, new_name) workitem = (version, p, new_project_name) worklist.append(workitem) # Ensure versioned resources are copied in the right order # such that higher versions override lower versions. worklist = sorted(worklist) for version, p, new_project_name in worklist: first_file = True source_dir = p.source_dirs()[0] target_dir = join(classes_dir, new_project_name, 'src') copied_source_dirs.append(source_dir) trailing = re.compile(r"[ \t]+\n") src_files = run_output(['git', 'ls-files'], cwd=source_dir).split('\n') for rel_src_file in src_files: if not len(rel_src_file): continue filename = os.path.basename(rel_src_file) src_file = join(source_dir, rel_src_file) dst_file = join(target_dir, os.path.relpath(src_file, source_dir)) binary_flag = '' if filename.endswith('.java') else 'b' with open(src_file, 'r' + binary_flag) as fp: try: contents = fp.read() except Exception as ex: # pylint: disable=broad-except mx.log('Error reading {}: {}'.format(src_file, ex)) if filename.endswith('.java'): old_line_count = len(contents.split('\n')) for old_name, new_name in package_renamings.items(): old_name_as_dir = old_name.replace('.', os.sep) if old_name_as_dir in src_file: new_name_as_dir = new_name.replace('.', os.sep) dst = src_file.replace(old_name_as_dir, new_name_as_dir) dst_file = join(target_dir, os.path.relpath(dst, source_dir)) contents = contents.replace(old_name, new_name) for old_line, new_line in replacements.items(): contents = contents.replace(old_line, new_line) contents = re.sub(trailing, '\n', contents) match = java_package_re.search(contents) if not match: mx.abort('Could not find package declaration in {}'.format(src_file)) java_package = match.group('package') if any(ex in java_package for ex in info.excludes): mx.log(' excluding ' + filename) continue new_line_count = len(contents.split('\n')) if new_line_count > old_line_count: mx.abort('Pattern replacement caused line count to grow from {} to {} in {}'.format(old_line_count, new_line_count, src_file)) else: if new_line_count < old_line_count: contents = contents.replace('\npackage ', '\n' * (old_line_count - new_line_count) + '\npackage ') new_line_count = len(contents.split('\n')) if new_line_count != old_line_count: mx.abort('Unable to correct line count for {}'.format(src_file)) for forbidden in denylist: if forbidden in contents: mx.abort('Found deny-listed pattern \'{}\' in {}'.format(forbidden, src_file)) dst_dir = os.path.dirname(dst_file) if not exists(dst_dir): os.makedirs(dst_dir) if first_file: mx.log(' copying: ' + source_dir) mx.log(' to: ' + target_dir) if p.testProject or p.definedAnnotationProcessors: to_exclude = new_project_name jdk_internal_vm_compiler_EXCLUDES.add(to_exclude) if p.testProject: jdk_internal_vm_compiler_test_SRC.add(to_exclude) first_file = False with open(dst_file, 'w' + binary_flag) as fp: fp.write(contents) def replace_line(filename, old_line, new_line): mx.log('Updating ' + filename + '...') old_lines = [] new_lines = [] with open(filename) as fp: old_lines = fp.readlines() for line in old_lines: if line == old_line: new_line = line.replace(old_line, new_line) if old_line != new_line: mx.log('Replaced \n ' + old_line.strip() + '\nwith\n ' + new_line.strip()) new_lines.append(line.replace(old_line, new_line)) with open(filename, 'w') as fp: fp.writelines(new_lines) def replace_lines(filename, begin_lines, end_line, replace_lines, old_line_check, preserve_indent=False, append_mode=False): mx.log('Updating ' + filename + '...') old_lines = [] new_lines = [] with open(filename) as fp: for begin_line in begin_lines: line = fp.readline() while line: stripped_line = line.strip() if stripped_line == begin_line: new_lines.append(line) break new_lines.append(line) line = fp.readline() assert line, begin_line + ' not found' lines = fp.readlines() line_in_def = True indent = 0 if preserve_indent: line = lines[0] lstripped_line = line.lstrip() indent = len(line) - len(lstripped_line) if not append_mode: for replace in replace_lines: new_lines.append(' ' * indent + replace) for line in lines: stripped_line = line.strip() if line_in_def: if stripped_line == end_line: line_in_def = False new_lines.append(line) else: old_line_check(line) if append_mode: new_lines.append(line) if append_mode and not line_in_def: # reach end line and append new lines for replace in replace_lines: new_lines.append(replace) else: new_lines.append(line) with open(filename, 'w') as fp: for line in new_lines: fp.write(line) return old_lines def single_column_with_continuation(line): parts = line.split() assert len(parts) == 2 and parts[1] == '\\', line def do_nothing(line): pass # Exclude all test, benchmark and annotation processor packages. if args.version >= 17: # See JDK-8258407 CompileJavaModules_gmk = join(jdkrepo, 'make', 'modules', 'jdk.internal.vm.compiler', 'Java.gmk') # pylint: disable=invalid-name begin_lines = ['EXCLUDES += \\'] else: CompileJavaModules_gmk = join(jdkrepo, 'make', 'CompileJavaModules.gmk') # pylint: disable=invalid-name begin_lines = ['jdk.internal.vm.compiler_EXCLUDES += \\'] new_lines = [] for pkg in sorted(jdk_internal_vm_compiler_EXCLUDES): new_lines.append(pkg + ' \\\n') end_line = '#' old_line_check = single_column_with_continuation replace_lines(CompileJavaModules_gmk, begin_lines, end_line, new_lines, old_line_check, preserve_indent=True) # replace renamed service compiler_module_info = join(jdkrepo, 'src', 'jdk.internal.vm.compiler', 'share', 'classes', 'module-info.java') old_line = ' uses org.graalvm.compiler.nodes.graphbuilderconf.NodeIntrinsicPluginFactory;\n' new_line = ' uses org.graalvm.compiler.nodes.graphbuilderconf.GeneratedPluginFactory;\n' replace_line(compiler_module_info, old_line, new_line) # Update 'SRC' in the 'Compile graalunit tests' section of make/test/JtregGraalUnit.gmk # to include all test packages. JtregGraalUnit_gmk = join(jdkrepo, 'make', 'test', 'JtregGraalUnit.gmk') # pylint: disable=invalid-name new_lines = [] jdk_internal_vm_compiler_test_SRC.discard('org.graalvm.compiler.microbenchmarks') jdk_internal_vm_compiler_test_SRC.discard('org.graalvm.compiler.virtual.bench') jdk_internal_vm_compiler_test_SRC.discard('org.graalvm.micro.benchmarks') for pkg in sorted(jdk_internal_vm_compiler_test_SRC): new_lines.append('$(SRC_DIR)/' + pkg + '/src \\\n') if args.version == 11: begin_lines = ['### Compile and build graalunit tests', 'SRC := \\'] else: begin_lines = ['### Compile graalunit tests', 'SRC := \\'] end_line = ', \\' old_line_check = single_column_with_continuation replace_lines(JtregGraalUnit_gmk, begin_lines, end_line, new_lines, old_line_check, preserve_indent=True) # Updates for Metropolis if args.metro: # Update 'PROCESSOR_JARS' and 'PROC_SRC_SUBDIRS' in make/modules/jdk.internal.vm.compiler/Gensrc.gmk Gensrc_gmk = join(jdkrepo, 'make', 'modules', 'jdk.internal.vm.compiler', 'Gensrc.gmk') # pylint: disable=invalid-name begin_lines = ['PROC_SRC_SUBDIRS := \\'] end_line = 'org.graalvm.compiler.virtual \\' new_lines = [' jdk.internal.vm.compiler.libgraal.jni \\\n'] replace_lines(Gensrc_gmk, begin_lines, end_line, new_lines, old_line_check, preserve_indent=True, append_mode=True) begin_lines = ['PROCESSOR_JARS := \\'] end_line = '$(BUILDTOOLS_OUTPUTDIR)/jdk.vm.compiler.serviceprovider.processor.jar \\' new_lines = [' $(BUILDTOOLS_OUTPUTDIR)/jdk.vm.compiler.libgraal.jni.processor.jar \\\n'] replace_lines(Gensrc_gmk, begin_lines, end_line, new_lines, old_line_check, preserve_indent=True, append_mode=True) # Add 'libgraal.jni.processor' build to make/CompileToolsHotspot.gmk CompileToolsHotspot_gmk = join(jdkrepo, 'make', 'CompileToolsHotspot.gmk') # pylint: disable=invalid-name begin_lines = ['TARGETS += $(BUILD_VM_COMPILER_SERVICEPROVIDER_PROCESSOR)'] end_line = '##############################################################################' new_lines = ['\n $(eval $(call SetupJavaCompilation, BUILD_VM_COMPILER_LIBGRAAL_JNI_PROCESSOR, \\\n', ' TARGET_RELEASE := $(TARGET_RELEASE_BOOTJDK), \\\n', ' SRC := \\\n', ' $(SRC_DIR)/org.graalvm.compiler.processor/src \\\n', ' $(SRC_DIR)/jdk.internal.vm.compiler.libgraal.jni.annotation/src \\\n', ' $(SRC_DIR)/jdk.internal.vm.compiler.libgraal.jni.processor/src \\\n', ' , \\\n', ' EXCLUDE_FILES := $(EXCLUDE_FILES), \\\n', ' BIN := $(BUILDTOOLS_OUTPUTDIR)/jdk.vm.compiler.libgraal.jni.processor, \\\n', ' JAR := $(BUILDTOOLS_OUTPUTDIR)/jdk.vm.compiler.libgraal.jni.processor.jar, \\\n', ' DISABLED_WARNINGS := options, \\\n', ' ))\n\n', ' TARGETS += $(BUILD_VM_COMPILER_LIBGRAAL_JNI_PROCESSOR)\n\n', ' ##############################################################################\n'] # indent is inlined old_line_check = do_nothing replace_lines(CompileToolsHotspot_gmk, begin_lines, end_line, new_lines, old_line_check, preserve_indent=True, append_mode=True) # Rename packages in libgraal.jni.processor service rename_packages(join(jdkrepo, 'src', 'jdk.internal.vm.compiler', 'share', 'classes', 'jdk.internal.vm.compiler.libgraal.jni.processor', 'src', 'META-INF', 'services', 'javax.annotation.processing.Processor')) overwritten = '' if not git_repo: mx.log('Adding new files to HG...') m_src_dirs = [] for m in graal_modules: m_src_dirs.append(join('src', m.name)) out = run_output(['hg', 'log', '-r', 'last(keyword("Update Graal"))', '--template', '{rev}'] + m_src_dirs, cwd=jdkrepo) last_graal_update = out.strip() for m in graal_modules: m_src_dir = join('src', m.name) if last_graal_update: overwritten += run_output(['hg', 'diff', '-r', last_graal_update, '-r', 'tip', m_src_dir], cwd=jdkrepo) mx.run(['hg', 'add', m_src_dir], cwd=jdkrepo) mx.log('Removing old files from HG...') for m in graal_modules: m_src_dir = join('src', m.name) out = run_output(['hg', 'status', '-dn', m_src_dir], cwd=jdkrepo) if out: mx.run(['hg', 'rm'] + out.split(), cwd=jdkrepo) out = run_output(['git', 'tag', '-l', 'JDK-*'], cwd=mx_compiler._suite.vc_dir) last_jdk_tag = sorted(out.split(), reverse=True)[0] pretty = args.pretty or 'format:%h %ad %>(20) %an %s' out = run_output(['git', '--no-pager', 'log', '--merges', '--abbrev-commit', '--pretty=' + pretty, '--first-parent', '-r', last_jdk_tag + '..HEAD'] + copied_source_dirs, cwd=mx_compiler._suite.vc_dir) changes_file = 'changes-since-{}.txt'.format(last_jdk_tag) with open(changes_file, 'w') as fp: fp.write(out) mx.log('Saved changes since {} to {}'.format(last_jdk_tag, os.path.abspath(changes_file))) if overwritten: overwritten_file = 'overwritten-diffs.txt' with open(overwritten_file, 'w') as fp: fp.write(overwritten) mx.warn('Overwritten changes detected in OpenJDK Graal! See diffs in ' + os.path.abspath(overwritten_file))
def _fastr_gate_runner(args, tasks): with mx_gate.Task('Setup no specials', tasks, tags=[FastRGateTags.no_specials]) as t: if t: os.environ['FASTR_OPTION_UseSpecials'] = 'false' with mx_gate.Task('Setup no dsl cache', tasks, tags=[FastRGateTags.no_dsl_cache]) as t: if t: os.environ['FASTR_OPTION_DSLCacheSizeFactor'] = '0' with mx_gate.Task('SetupLLVM', tasks, tags=[FastRGateTags.llvm]) as t: if t: os.environ['FASTR_RFFI'] = 'llvm' with mx_gate.Task('GCTorture1', tasks, tags=[FastRGateTags.gc_torture1]) as t: if t: os.environ['FASTR_GCTORTURE'] = '1' with mx_gate.Task('GCTorture5', tasks, tags=[FastRGateTags.gc_torture5]) as t: if t: os.environ['FASTR_GCTORTURE'] = '5' ''' The specific additional gates tasks provided by FastR. ''' with mx_gate.Task('ExtSoftVersions', tasks, tags=[mx_gate.Tags.always]) as t: if t: new_env = os.environ.copy() new_env['R_DEFAULT_PACKAGES'] = 'base' run_r(['-q', '-e', 'extSoftVersion()'], 'R', env=new_env) # --------------------------------- # Style checks: # FastR has custom copyright check with mx_gate.Task('Copyright check', tasks, tags=[mx_gate.Tags.style]) as t: if t: if mx.checkcopyrights(['--primary']) != 0: t.abort('copyright errors') # check that the expected test output file is up to date with mx_gate.Task('UnitTests: ExpectedTestOutput file check', tasks, tags=[mx_gate.Tags.style]) as t: if t: mx_unittest.unittest([ '-Dfastr.test.gen.expected=' + _test_srcdir(), '-Dfastr.test.check.expected=true' ] + _gate_unit_tests()) # ---------------------------------- # Basic tests: with mx_gate.Task( 'UnitTests', tasks, tags=[FastRGateTags.basic_tests, FastRGateTags.unit_tests]) as t: if t: mx_unittest.unittest(_gate_noapps_unit_tests()) with mx_gate.Task('Rembedded', tasks, tags=[FastRGateTags.basic_tests]) as t: if t: if rembedtest([]) != 0: t.abort("Rembedded tests failed") # ---------------------------------- # Package tests: with mx_gate.Task('Recommended load test', tasks, tags=[FastRGateTags.recommended_load]) as t: if t: # Note: this is a convenience mx gate job for testing the loading of recommended packages # We also test the loading of recommended pkgs in the "graalvm-tests" if not os.path.exists( os.path.join(_fastr_suite.dir, 'library', 'spatial')): mx.abort( 'Recommended packages seem to be not installed in FastR. Did you forget to build with FASTR_RELEASE=true?' ) pkgs = [ 'codetools', 'MASS', 'boot', 'class', 'cluster', 'lattice', 'nnet', 'spatial', 'Matrix', 'KernSmooth', 'foreign', 'nlme', 'rpart', 'survival' ] # TODO: removed failing "KernSmooth" and "cluster" on LLVM, GR-20406 if os.environ.get('FASTR_RFFI') == 'llvm': pkgs.remove('KernSmooth') pkgs.remove('cluster') # Creates code that looks like: require(codetools) && require(MASS) && ... require_stmts = ' && '.join( ['require(' + pkg + ')' for pkg in pkgs]) test_load = 'if (!(' + require_stmts + ')) q(status=1) else q(status=42)' if run_r(['--vanilla', '-e', test_load], 'R', nonZeroIsFatal=False) != 42: mx.abort("Loading of recommended packages failed") with mx_gate.Task('Internal pkg test', tasks, tags=[FastRGateTags.internal_pkgs_test]) as t: if t: list_file = os.path.join( _fastr_suite.dir, 'com.oracle.truffle.r.test.native/packages/pkg-filelist') if os.environ.get('FASTR_RFFI') == 'llvm': list_file_llvm = list_file + '.llvm' if os.path.exists(list_file_llvm): list_file = list_file_llvm if os.environ.get('FASTR_GCTORTURE') != '': list_file_gctorture = list_file + '.gctorture' if os.path.exists(list_file_gctorture): list_file = list_file_gctorture result = pkgtest( ["--verbose", "--repos", "FASTR", "--pkg-filelist", list_file]) if result != 0: mx.abort("internal package test failed") # CRAN packages are listed in files com.oracle.truffle.r.test.packages/gated0, gated1, ... # We loop over all such files and crete gate task for each of them # See also documentation in FastRGateTags.cran_pkgs_tests for i in range(1, 1000): list_file = os.path.join( _fastr_suite.dir, 'com.oracle.truffle.r.test.packages/gated' + str(i)) if not os.path.exists(list_file): break with mx_gate.Task('CRAN pkg test: ' + str(i), tasks, tags=[FastRGateTags.cran_pkgs_test + str(i)]) as t: if t: check_last = False if mx_gate.Task.tags is None else FastRGateTags.cran_pkgs_test_check_last in mx_gate.Task.tags # pylint: disable=unsupported-membership-test if check_last: next_file = os.path.join( _fastr_suite.dir, 'com.oracle.truffle.r.test.packages/gated' + str(i + 1)) if os.path.exists(next_file): mx.abort( "File %s exists, but the gate thinks that %s is the last file. Did you forget to update the gate configuration?" % (next_file, list_file)) cache_arg = os.environ.get('FASTR_PKGS_CACHE_OPT') if cache_arg is None: cache_arg = [] mx.warn( "If you want to use R packages cache, export environment variable FASTR_PKGS_CACHE_OPT. See option '--cache-pkgs' of 'mx pkgtest' for the syntax." ) else: cache_arg = ['--cache-pkgs', cache_arg] result = pkgtest( ["--verbose"] + cache_arg + ["--repos", "SNAPSHOT", "--pkg-filelist", list_file]) if result != 0: mx.abort("package test failed")
def _sulong_gate_runner(args, tasks): with Task('CheckCopyright', tasks, tags=['style']) as t: if t: if mx.checkcopyrights(['--primary']) != 0: t.abort( 'Copyright errors found. Please run "mx checkcopyrights --primary -- --fix" to fix them.' ) with Task('ClangFormat', tasks, tags=['style', 'clangformat']) as t: if t: clangformatcheck() _sulong_gate_testsuite('Benchmarks', 'shootout', tasks, args, tags=['benchmarks', 'sulongMisc']) _sulong_gate_unittest( 'Types', 'com.oracle.truffle.llvm.types.test', tasks, args, tags=['type', 'sulongMisc', 'sulongCoverage'], testClasses=['com.oracle.truffle.llvm.types.floating.test']) _sulong_gate_unittest('Pipe', 'com.oracle.truffle.llvm.test', tasks, args, tags=['pipe', 'sulongMisc', 'sulongCoverage'], testClasses=['CaptureOutputTest']) _sulong_gate_testsuite('LLVM', 'llvm', tasks, args, tags=['llvm', 'sulongCoverage']) _sulong_gate_testsuite('NWCC', 'nwcc', tasks, args, tags=['nwcc', 'sulongCoverage']) _sulong_gate_testsuite('GCCParserTorture', 'parserTorture', tasks, args, tags=['parser', 'sulongCoverage'], vmArgs=['-Dpolyglot.llvm.parseOnly=true']) _sulong_gate_testsuite('GCC_C', 'gcc_c', tasks, args, tags=['gcc_c', 'sulongCoverage']) _sulong_gate_testsuite('GCC_CPP', 'gcc_cpp', tasks, args, tags=['gcc_cpp', 'sulongCoverage']) _sulong_gate_testsuite('GCC_Fortran', 'gcc_fortran', tasks, args, tags=['gcc_fortran', 'sulongCoverage']) _sulong_gate_sulongsuite_unittest( 'Sulong', tasks, args, testClasses='SulongSuite', tags=['sulong', 'sulongBasic', 'sulongCoverage']) _sulong_gate_sulongsuite_unittest( 'Interop', tasks, args, testClasses='com.oracle.truffle.llvm.test.interop', tags=['interop', 'sulongBasic', 'sulongCoverage']) _sulong_gate_sulongsuite_unittest( 'Debug', tasks, args, testClasses='LLVMDebugTest', tags=['debug', 'sulongBasic', 'sulongCoverage']) _sulong_gate_sulongsuite_unittest( 'IRDebug', tasks, args, testClasses='LLVMIRDebugTest', tags=['irdebug', 'sulongBasic', 'sulongCoverage']) _sulong_gate_sulongsuite_unittest( 'BitcodeFormat', tasks, args, testClasses='BitcodeFormatTest', tags=['bitcodeFormat', 'sulongBasic', 'sulongCoverage']) _sulong_gate_sulongsuite_unittest( 'OtherTests', tasks, args, testClasses='com.oracle.truffle.llvm.test.other', tags=['otherTests', 'sulongBasic', 'sulongCoverage']) _sulong_gate_testsuite('Assembly', 'inlineassemblytests', tasks, args, testClasses='InlineAssemblyTest', tags=['assembly', 'sulongCoverage']) _sulong_gate_testsuite( 'Args', 'other', tasks, args, tags=['args', 'sulongMisc', 'sulongCoverage'], testClasses=['com.oracle.truffle.llvm.test.MainArgsTest']) _sulong_gate_testsuite( 'Callback', 'other', tasks, args, tags=['callback', 'sulongMisc', 'sulongCoverage'], testClasses=['com.oracle.truffle.llvm.test.CallbackTest']) _sulong_gate_testsuite( 'Varargs', 'other', tasks, args, tags=['vaargs', 'sulongMisc', 'sulongCoverage'], testClasses=['com.oracle.truffle.llvm.test.VAArgsTest'])
def _fastr_gate_runner(args, tasks): with mx_gate.Task('Setup no specials', tasks, tags=[FastRGateTags.no_specials]) as t: if t: os.environ['FASTR_OPTION_UseSpecials'] = 'false' with mx_gate.Task('Setup no dsl cache', tasks, tags=[FastRGateTags.no_dsl_cache]) as t: if t: os.environ['FASTR_OPTION_DSLCacheSizeFactor'] = '0' with mx_gate.Task('SetupLLVM', tasks, tags=[FastRGateTags.llvm]) as t: if t: os.environ['FASTR_RFFI'] = 'llvm' with mx_gate.Task('GCTorture1', tasks, tags=[FastRGateTags.gc_torture1]) as t: if t: os.environ['FASTR_GCTORTURE'] = '1' with mx_gate.Task('GCTorture3', tasks, tags=[FastRGateTags.gc_torture3]) as t: if t: os.environ['FASTR_GCTORTURE'] = '3' with mx_gate.Task('VerySlowAsserts', tasks, tags=[FastRGateTags.very_slow_asserts]) as t: if t: os.environ['FASTR_TEST_VERY_SLOW_ASSERTS'] = 'true' ''' The specific additional gates tasks provided by FastR. ''' with mx_gate.Task('ExtSoftVersions', tasks, tags=[mx_gate.Tags.always]) as t: if t: new_env = os.environ.copy() new_env['R_DEFAULT_PACKAGES'] = 'base' run_r(['-q', '-e', 'extSoftVersion()'], 'R', env=new_env) with mx_gate.Task('LibsInfo', tasks, tags=[mx_gate.Tags.always]) as t: if t: mx.log("Libraries captured in FASTR_HOME/lib:") lib_dir = os.path.join(_fastr_suite.dir, 'lib') ldd = ['otool', '-L'] if platform.system() == 'Darwin' else ['ldd'] for f in os.listdir(lib_dir): full_path = os.path.join(lib_dir, f) mx.run(['file', full_path], nonZeroIsFatal=False) mx.log('---\nobjdump:') mx.run(['objdump', '-s', '--section', '.comment', full_path], nonZeroIsFatal=False) mx.log('---\nlinking info:') mx.run(ldd + [full_path], nonZeroIsFatal=False) mx.log('---------') # --------------------------------- # Style checks: # FastR has custom copyright check with mx_gate.Task('Copyright check', tasks, tags=[mx_gate.Tags.style]) as t: if t: if mx.checkcopyrights(['--primary']) != 0: t.abort('copyright errors') # check that the expected test output file is up to date with mx_gate.Task('UnitTests: ExpectedTestOutput file check', tasks, tags=[mx_gate.Tags.style]) as t: if t: mx_unittest.unittest([ '-Dfastr.test.gen.expected=' + _test_srcdir(), '-Dfastr.test.check.expected=true' ] + _gate_unit_tests()) # ---------------------------------- # Basic tests: with mx_gate.Task( 'UnitTests', tasks, tags=[FastRGateTags.basic_tests, FastRGateTags.unit_tests]) as t: if t: mx_unittest.unittest(_gate_noapps_unit_tests()) with mx_gate.Task('Rembedded', tasks, tags=[FastRGateTags.basic_tests]) as t: if t: if rembedtest([]) != 0: t.abort("Rembedded tests failed") # ---------------------------------- # Package tests: with mx_gate.Task('Recommended load test', tasks, tags=[FastRGateTags.recommended_load]) as t: if t: # Note: this is a convenience mx gate job for testing the loading of recommended packages # We also test the loading of recommended pkgs in the "graalvm-tests" if not os.path.exists( os.path.join(_fastr_suite.dir, 'library', 'spatial')): mx.abort( 'Recommended packages seem to be not installed in FastR. Did you forget to build with FASTR_RELEASE=true?' ) pkgs = [ 'codetools', 'MASS', 'boot', 'class', 'cluster', 'lattice', 'nnet', 'spatial', 'Matrix', 'KernSmooth', 'foreign', 'nlme', 'rpart', 'survival' ] # Creates code that looks like: require(codetools) && require(MASS) && ... require_stmts = ' && '.join( ['require(' + pkg + ')' for pkg in pkgs]) test_load = 'if (!(' + require_stmts + ')) q(status=1) else q(status=42)' if run_r(['--vanilla', '-e', test_load], 'R', nonZeroIsFatal=False) != 42: mx.abort("Loading of recommended packages failed") with mx_gate.Task('Internal pkg test', tasks, tags=[FastRGateTags.internal_pkgs_test]) as t: if t: internal_pkg_tests() # CRAN packages are listed in files com.oracle.truffle.r.test.packages/gated0, gated1, ... # We loop over all such files and crete gate task for each of them # See also documentation in FastRGateTags.cran_pkgs_tests for i in range(1, 1000): list_file = os.path.join( _fastr_suite.dir, 'com.oracle.truffle.r.test.packages/gated' + str(i)) if not os.path.exists(list_file): break with mx_gate.Task('CRAN pkg test: ' + str(i), tasks, tags=[FastRGateTags.cran_pkgs_test + str(i)]) as t: if t: check_last = False if mx_gate.Task.tags is None else FastRGateTags.cran_pkgs_test_check_last in mx_gate.Task.tags # pylint: disable=unsupported-membership-test if check_last: next_file = os.path.join( _fastr_suite.dir, 'com.oracle.truffle.r.test.packages/gated' + str(i + 1)) if os.path.exists(next_file): mx.abort( "File %s exists, but the gate thinks that %s is the last file. Did you forget to update the gate configuration?" % (next_file, list_file)) cran_pkg_tests(list_file)