def travisTestSulong(args=None):
"""executes the Sulong test cases (which also stress compilation)"""
tasks = []
with Task('BuildJavaWithJavac', tasks) as t:
if t: mx.command_function('build')(['-p', '--warning-as-error', '--force-javac'])
with Task('TestSulong', tasks) as t:
if t: runTruffleTestCases()
def travis1(args=None):
tasks = []
with Task('BuildJavaWithEcj', tasks) as t:
if t:
if mx.get_env('JDT'):
mx.command_function('build')(['-p', '--no-native', '--warning-as-error'])
gate_clean([], tasks, name='CleanAfterEcjBuild')
else:
mx._warn_or_abort('JDT environment variable not set. Cannot execute BuildJavaWithEcj task.', args.strict_mode)
with Task('BuildJavaWithJavac', tasks) as t:
if t: mx.command_function('build')(['-p', '--warning-as-error', '--no-native', '--force-javac'])
with Task('Findbugs', tasks) as t:
if t and mx_findbugs.findbugs([]) != 0:
t.abort('FindBugs warnings were found')
with Task('TestBenchmarks', tasks) as t:
if t: runBenchmarkTestCases()
with Task('TestPolglot', tasks) as t:
if t: runPolyglotTestCases()
with Task('TestInterop', tasks) as t:
if t: runInteropTestCases()
with Task('TestAsm', tasks) as t:
if t: runAsmTestCases()
with Task('TestTypes', tasks) as t:
if t: runTypeTestCases()
with Task('TestSulong', tasks) as t:
if t: runTruffleTestCases()
with Task('TestLLVM', tasks) as t:
if t: runLLVMTestCases()
def travis2(args=None):
"""executes the second Travis job (Javac build, GCC execution test cases)"""
tasks = []
with Task('BuildJavaWithJavac', tasks) as t:
if t: mx.command_function('build')(['-p', '--warning-as-error', '--force-javac'])
with Task('TestGCC', tasks) as t:
if t: runGCCTestCases()
def travis1(args=None):
"""executes the first Travis job (Javac build, benchmarks, polyglot, interop, tck, asm, types, and LLVM test cases)"""
tasks = []
with Task("BuildJavaWithJavac", tasks) as t:
if t:
mx.command_function("build")(["-p", "--warning-as-error", "--force-javac"])
with Task("TestBenchmarks", tasks) as t:
if t:
runBenchmarkTestCases()
with Task("TestPolglot", tasks) as t:
if t:
runPolyglotTestCases()
with Task("TestInterop", tasks) as t:
if t:
runInteropTestCases()
with Task("TestTck", tasks) as t:
if t:
runTckTestCases()
with Task("TestAsm", tasks) as t:
if t:
runAsmTestCases()
with Task("TestTypes", tasks) as t:
if t:
runTypeTestCases()
with Task("TestLLVM", tasks) as t:
if t:
runLLVMTestCases()
with Task("TestMainArgs", tasks) as t:
if t:
runMainArgTestCases()
with Task("TestPipe", tasks) as t:
if t:
runPipeTestCases()
def compileWithEcjStrict(args=None):
"""build project with the option --warning-as-error"""
if mx.get_env('JDT'):
mx.clean([])
mx.command_function('build')(['-p', '--warning-as-error'])
else:
exit('JDT environment variable not set. Cannot execute BuildJavaWithEcj task.')
def travisArgon2(args=None):
"""executes the argon2 Travis job (Javac build, argon2 test cases)"""
tasks = []
with Task('BuildJavaWithJavac', tasks) as t:
if t: mx.command_function('build')(['-p', '--warning-as-error', '--force-javac'])
with Task('TestArgon2', tasks) as t:
if t: runTestArgon2(optimize=False)
def compileWithEcjStrict(args=None):
"""build project with the option --warning-as-error"""
if mx.get_env("JDT"):
mx.clean([])
mx.command_function("build")(["-p", "--warning-as-error"])
else:
exit("JDT environment variable not set. Cannot execute BuildJavaWithEcj task.")
def travisJRuby(args=None):
"""executes the JRuby Travis job (Javac build, JRuby test cases)"""
tasks = []
with Task('BuildJavaWithJavac', tasks) as t:
if t: mx.command_function('build')(['-p', '--warning-as-error', '--no-native', '--force-javac'])
with Task('TestJRuby', tasks) as t:
if t: runTestJRuby()
def travis3(args=None):
tasks = []
with Task('BuildJavaWithJavac', tasks) as t:
if t: mx.command_function('build')(['-p', '--warning-as-error', '--no-native', '--force-javac'])
with Task('TestNWCC', tasks) as t:
if t: runNWCCTestCases()
with Task('TestGCCSuiteCompile', tasks) as t:
if t: runCompileTestCases()
def travis3(args=None):
"""executes the third Travis job (Javac build, NWCC, GCC compilation test cases)"""
tasks = []
with Task('BuildJavaWithJavac', tasks) as t:
if t: mx.command_function('build')(['-p', '--warning-as-error', '--no-native', '--force-javac'])
with Task('TestNWCC', tasks) as t:
if t: runNWCCTestCases()
with Task('TestGCCSuiteCompile', tasks) as t:
if t: runCompileTestCases()
def travisArgon2(args=None):
"""executes the argon2 Travis job (Javac build, argon2 test cases)"""
tasks = []
with Task("BuildJavaWithJavac", tasks) as t:
if t:
mx.command_function("build")(["-p", "--warning-as-error", "--force-javac"])
with Task("TestArgon2", tasks) as t:
if t:
runTestArgon2(optimize=False)
def travisTestSulong(args=None):
"""executes the Sulong test cases (which also stress compilation)"""
tasks = []
with Task("BuildJavaWithJavac", tasks) as t:
if t:
mx.command_function("build")(["-p", "--warning-as-error", "--force-javac"])
with Task("TestSulong", tasks) as t:
if t:
runTruffleTestCases()
def travis2(args=None):
"""executes the second Travis job (Javac build, GCC execution test cases)"""
tasks = []
with Task("BuildJavaWithJavac", tasks) as t:
if t:
mx.command_function("build")(["-p", "--warning-as-error", "--force-javac"])
with Task("TestGCC", tasks) as t:
if t:
runGCCTestCases()
def travis4(args=None):
"""executes the fourth Travis job (Javac build, LLVM and GCC test cases with BitCode parser)"""
tasks = []
with Task('BuildJavaWithJavac', tasks) as t:
if t: mx.command_function('build')(['-p', '--warning-as-error', '--force-javac'])
with Task('TestLLVMBC', tasks) as t:
if t: runLLVMTestCases(['-Dsulong.TestBinaryParser=true'])
with Task('TestGCCBC', tasks) as t:
if t: runGCCTestCases(['-Dsulong.TestBinaryParser=true'])
def travis4(args=None):
"""executes the fourth Travis job (Javac build, LLVM and GCC test cases with BitCode parser)"""
tasks = []
with Task("BuildJavaWithJavac", tasks) as t:
if t:
mx.command_function("build")(["-p", "--warning-as-error", "--force-javac"])
with Task("TestLLVMBC", tasks) as t:
if t:
runLLVMTestCases(["-Dsulong.TestBinaryParser=true"])
with Task("TestGCCBC", tasks) as t:
if t:
runGCCTestCases(["-Dsulong.TestBinaryParser=true"])
def travis3(args=None):
"""executes the third Travis job (Javac build, NWCC, GCC compilation test cases)"""
tasks = []
with Task("BuildJavaWithJavac", tasks) as t:
if t:
mx.command_function("build")(["-p", "--warning-as-error", "--force-javac"])
with Task("TestNWCC", tasks) as t:
if t:
runNWCCTestCases()
with Task("TestGCCSuiteCompile", tasks) as t:
if t:
runCompileTestCases()
with Task("TestLifetime", tasks) as t:
if t:
runLifetimeTestCases()
def travisRunSuite(args=None):
"""executes all the test suites or selected ones (see -h or --help)"""
vmArgs, otherArgs = mx_sulong.truffle_extract_VM_args(args)
parser = argparse.ArgumentParser(description="Compiles all or selected test suites.")
parser.add_argument('suite', nargs='*', help=' '.join(testSuites.keys()), default=testSuites.keys())
parsedArgs = parser.parse_args(otherArgs)
tasks = []
with mx_gate.Task('BuildJavaWithJavac', tasks) as t:
if t: mx.command_function('build')(['-p', '--warning-as-error', '--force-javac'])
for testSuiteName in parsedArgs.suite:
with mx_gate.Task('Test%s' % testSuiteName.capitalize(), tasks) as t:
if t:
_, runCommand = testSuites[testSuiteName]
runCommand(vmArgs)
def deploy_binary_if_truffle_head(args):
"""If the active branch is 'truffle-head', deploy binaries for the primary suite to remote maven repository."""
primary_branch = 'truffle-head'
active_branch = mx.VC.get_vc(_suite.dir).active_branch(_suite.dir)
if active_branch == primary_branch:
return mx.command_function('deploy-binary')(args)
else:
mx.log('The active branch is "%s". Binaries are deployed only if the active branch is "%s".' % (active_branch, primary_branch))
return 0
def deploy_binary_if_truffle_head(args):
"""If the active branch is 'truffle-head', deploy binaries for the primary suite to remote maven repository."""
primary_branch = 'truffle-head'
active_branch = mx.VC.get_vc(_suite.dir).active_branch(_suite.dir)
if active_branch == primary_branch:
return mx.command_function('deploy-binary')(args)
else:
mx.log('The active branch is "%s". Binaries are deployed only if the active branch is "%s".' % (active_branch, primary_branch))
return 0
def travis1(args=None):
tasks = []
with Task('BuildJavaWithEcj', tasks) as t:
if t:
if mx.get_env('JDT'):
mx.command_function('build')(['-p', '--no-native', '--warning-as-error'])
gate_clean([], tasks, name='CleanAfterEcjBuild')
else:
mx._warn_or_abort('JDT environment variable not set. Cannot execute BuildJavaWithEcj task.', args.strict_mode)
with Task('BuildJavaWithJavac', tasks) as t:
if t: mx.command_function('build')(['-p', '--warning-as-error', '--no-native', '--force-javac'])
with Task('BuildHotSpotGraalServer: product', tasks) as t:
if t: buildvms(['-c', '--vms', 'server', '--builds', 'product'])
with VM('server', 'product'):
with Task('Findbugs', tasks) as t:
if t and mx_findbugs.findbugs([]) != 0:
t.abort('FindBugs warnings were found')
with VM('server', 'product'):
with Task('TestBenchmarks', tasks) as t:
if t: runBenchmarkTestCases()
with VM('server', 'product'):
with Task('TestPolglot', tasks) as t:
if t: runPolyglotTestCases()
with VM('server', 'product'):
with Task('TestInterop', tasks) as t:
if t: runInteropTestCases()
with VM('server', 'product'):
with Task('TestAsm', tasks) as t:
if t: runAsmTestCases()
with VM('server', 'product'):
with Task('TestTypes', tasks) as t:
if t: runTypeTestCases()
with VM('server', 'product'):
with Task('TestSulong', tasks) as t:
if t: runTruffleTestCases()
with VM('server', 'product'):
with Task('TestLLVM', tasks) as t:
if t: runLLVMTestCases()
with VM('server', 'product'):
with Task('TestNWCC', tasks) as t:
if t: runNWCCTestCases()
with VM('server', 'product'):
with Task('TestGCCSuiteCompile', tasks) as t:
if t: runCompileTestCases()
def travis1(args=None):
"""executes a Travis job"""
tasks = []
with Task('BuildJavaWithJavac', tasks) as t:
if t:
mx.command_function('build')(
['-p', '--warning-as-error', '--force-javac'])
with Task('TestBenchmarks', tasks) as t:
if t: mx_testsuites.runSuite(['shootout'])
with Task('TestPolglot', tasks) as t:
if t: mx_testsuites.runSuite(['polyglot'])
with Task('TestTypes', tasks) as t:
if t: mx_testsuites.runSuite(['type'])
with Task('TestPipe', tasks) as t:
if t: mx_testsuites.runSuite(['pipe'])
with Task('TestLLVM', tasks) as t:
if t: mx_testsuites.runSuite(['llvm'])
with Task('TestSulong', tasks) as t:
if t: mx_testsuites.runSuite(['sulong'])
def jacocoreport(args):
"""create a JaCoCo coverage report
Creates the report from the 'jacoco.exec' file in the current directory.
Default output directory is 'coverage', but an alternative can be provided as an argument."""
dist_name = "MX_JACOCO_REPORT"
mx.command_function("build")(['--dependencies', dist_name])
dist = mx.distribution(dist_name)
jdk = mx.get_jdk(dist.javaCompliance)
parser = ArgumentParser(prog='mx jacocoreport')
parser.add_argument('--format', help='Export format (HTML or XML)', default='html', choices=['html', 'xml'])
parser.add_argument('--omit-excluded', action='store_true', help='omit excluded files from report')
parser.add_argument('output_directory', help='Output directory', default='coverage', nargs='?')
args = parser.parse_args(args)
# list of strings of the form "project-dir:binary-dir"
includedirs = []
for p in mx.projects():
projsetting = getattr(p, 'jacoco', '')
if (projsetting == 'include' or projsetting == '') and _jacoco_is_package_whitelisted(p.name):
if isinstance(p, mx.ClasspathDependency):
source_dirs = []
if p.isJavaProject():
source_dirs += p.source_dirs() + [p.source_gen_dir()]
includedirs.append(":".join([p.dir, p.classpath_repr(jdk)] + source_dirs))
def _run_reporter(extra_args=None):
mx.run_java(['-cp', mx.classpath([dist_name], jdk=jdk), '-jar', dist.path, '--in', 'jacoco.exec', '--out',
args.output_directory, '--format', args.format] +
(extra_args or []) +
sorted(includedirs),
jdk=jdk, addDefaultArgs=False)
if not args.omit_excluded:
_run_reporter()
else:
with tempfile.NamedTemporaryFile(suffix="jacoco-report-exclude", mode="w") as fp:
excludes, _ = _jacoco_excludes_includes()
fp.writelines((e + "\n" for e in excludes))
fp.flush()
_run_reporter(['--exclude-file', fp.name])
def _sulong_gate_runner(args, tasks):
_unittest_task_factory = UnittestTaskFactory()
def _unittest(title, test_suite, tags=None, testClasses=None, unittestArgs=None, description=None):
_unittest_task_factory.add(title, test_suite, args, tags=tags, testClasses=testClasses, unittestArgs=unittestArgs, description=description)
with Task('CheckCopyright', tasks, tags=['style']) as t:
if t:
if mx.checkcopyrights(['--primary']) != 0:
t.abort('Copyright errors found. Please run "mx checkcopyrights --primary -- --fix" to fix them.')
with Task('BuildLLVMorg', tasks, tags=['style', 'clangformat']) as t:
# needed for clang-format
if t: build_llvm_org(args)
with Task('ClangFormat', tasks, tags=['style', 'clangformat']) as t:
if t: clangformat([])
# Folders not containing tests: options, services, util
_unittest('Benchmarks', 'SULONG_SHOOTOUT_TEST_SUITE', description="Language Benchmark game tests", testClasses=['ShootoutsSuite'], tags=['benchmarks', 'sulongMisc'])
_unittest('Types', 'SULONG_TEST', description="Test floating point arithmetic", testClasses=['com.oracle.truffle.llvm.tests.types.floating.'], tags=['type', 'sulongMisc', 'sulongCoverage'])
_unittest('Pipe', 'SULONG_TEST', description="Test output capturing", testClasses=['CaptureOutputTest'], tags=['pipe', 'sulongMisc', 'sulongCoverage'])
_unittest('LLVM', 'SULONG_LLVM_TEST_SUITE', description="LLVM 3.2 test suite", testClasses=['LLVMSuite'], tags=['llvm', 'sulongCoverage'])
_unittest('NWCC', 'SULONG_NWCC_TEST_SUITE', description="Test suite of the NWCC compiler v0.8.3", testClasses=['NWCCSuite'], tags=['nwcc', 'sulongCoverage'])
_unittest('GCCParserTorture', 'SULONG_PARSER_TORTURE', description="Parser test using GCC suite", testClasses=['ParserTortureSuite'], tags=['parser', 'sulongCoverage'])
_unittest('GCC_C', 'SULONG_GCC_C_TEST_SUITE', description="GCC 5.2 test suite (C tests)", testClasses=['GccCSuite'], tags=['gcc_c', 'sulongCoverage'])
_unittest('GCC_CPP', 'SULONG_GCC_CPP_TEST_SUITE', description="GCC 5.2 test suite (C++ tests)", testClasses=['GccCppSuite'], tags=['gcc_cpp', 'sulongCoverage'])
_unittest('GCC_Fortran', 'SULONG_GCC_FORTRAN_TEST_SUITE', description="GCC 5.2 test suite (Fortran tests)", testClasses=['GccFortranSuite'], tags=['gcc_fortran', 'sulongCoverage'])
_unittest('Sulong', 'SULONG_STANDALONE_TEST_SUITES', description="Sulong's internal tests", testClasses='SulongSuite', tags=['sulong', 'sulongBasic', 'sulongCoverage'])
_unittest('Interop', 'SULONG_EMBEDDED_TEST_SUITES', description="Truffle Language interoperability tests", testClasses=['com.oracle.truffle.llvm.tests.interop.'], tags=['interop', 'sulongBasic', 'sulongCoverage'])
_unittest('Linker', 'SULONG_EMBEDDED_TEST_SUITES', description=None, testClasses=['com.oracle.truffle.llvm.tests.linker.'], tags=['linker', 'sulongBasic', 'sulongCoverage'])
_unittest('Debug', 'SULONG_EMBEDDED_TEST_SUITES', description="Debug support test suite", testClasses=['com.oracle.truffle.llvm.tests.debug.LLVMDebugTest'], tags=['debug', 'sulongBasic', 'sulongCoverage'])
_unittest('IRDebug', 'SULONG_EMBEDDED_TEST_SUITES', description=None, testClasses=['com.oracle.truffle.llvm.tests.debug.LLVMIRDebugTest'], tags=['irdebug', 'sulongBasic', 'sulongCoverage'])
_unittest('BitcodeFormat', 'SULONG_EMBEDDED_TEST_SUITES', description=None, testClasses=['com.oracle.truffle.llvm.tests.bitcodeformat.'], tags=['bitcodeFormat', 'sulongBasic', 'sulongCoverage'])
_unittest('DebugExpr', 'SULONG_EMBEDDED_TEST_SUITES', description=None, testClasses=['com.oracle.truffle.llvm.tests.debug.LLVMDebugExprParserTest'], tags=['debugexpr', 'sulongBasic', 'sulongCoverage'])
_unittest('OtherTests', 'SULONG_EMBEDDED_TEST_SUITES', description=None, testClasses=['com.oracle.truffle.llvm.tests.bitcode.', 'com.oracle.truffle.llvm.tests.other.', 'com.oracle.truffle.llvm.tests.runtime.'], tags=['otherTests', 'sulongBasic', 'sulongCoverage'])
_unittest('Args', 'SULONG_EMBEDDED_TEST_SUITES', description="Tests main args passing", testClasses=['com.oracle.truffle.llvm.tests.MainArgsTest'], tags=['args', 'sulongMisc', 'sulongCoverage'])
_unittest('Callback', 'SULONG_EMBEDDED_TEST_SUITES', description="Test calling native functions", testClasses=['com.oracle.truffle.llvm.tests.CallbackTest'], tags=['callback', 'sulongMisc', 'sulongCoverage'])
_unittest('Varargs', 'SULONG_EMBEDDED_TEST_SUITES', description="Varargs tests", testClasses=['com.oracle.truffle.llvm.tests.VAArgsTest'], tags=['vaargs', 'sulongMisc', 'sulongCoverage'])
_unittest_task_factory.execute(tasks)
with Task('TestToolchain', description="build toolchain-launchers-tests project", tags=['toolchain', 'sulongMisc', 'sulongCoverage'], tasks=tasks) as t:
if t:
with SulongGateEnv():
mx.command_function('clean')(['--project', 'toolchain-launchers-tests'] + args.extra_build_args)
mx.command_function('build')(['--project', 'toolchain-launchers-tests'] + args.extra_build_args)
def python_build_watch(args):
"""
Watch the suite and on any changes to .class, .jar, .h, or .c files rebuild.
By default, rebuilds only the archives and non-Java projects.
"""
parser = ArgumentParser(prog='mx python-build-watch')
parser.add_argument('--full', action='store_true', help='Run a full mx build', required=False)
parser.add_argument('--graalvm', action='store_true', help='Build a graalvm', required=False)
parser.add_argument('--no-java', action='store_true', help='Build only archives and native projects [default]', required=False)
args = parser.parse_args(args)
if sum([args.full, args.graalvm, args.no_java]) > 1:
mx.abort("Only one of --full, --graalvm, --no-java can be specified")
if args.full:
suffixes = [".c", ".h", ".class", ".jar", ".java"]
excludes = [".*\\.py$"]
elif args.graalvm:
suffixes = [".c", ".h", ".class", ".jar", ".java", ".py"]
excludes = ["mx_.*\\.py$"]
else:
suffixes = [".c", ".h", ".class", ".jar"]
excludes = [".*\\.py$", ".*\\.java$"]
cmd = ["inotifywait", "-q", "-e", "close_write,moved_to", "-r", "--format=%f"]
for e in excludes:
cmd += ["--exclude", e]
cmd += ["@%s" % os.path.join(SUITE.dir, ".git"), SUITE.dir]
while True:
out = mx.OutputCapture()
mx.run(cmd, out=out)
changed_file = out.data.strip()
mx.logv(changed_file)
if any(changed_file.endswith(ext) for ext in [".c", ".h", ".class", ".jar"]):
mx.log("Build needed ...")
time.sleep(2)
if args.full:
mx.command_function("build")()
elif args.graalvm:
mx.log(python_gvm())
else:
nativebuild([])
mx.log("Build done.")
def _run_downstream_svm(commands, binary=False):
new_rewrites = None
if binary:
localmvn = "/tmp/graalpythonsnapshots"
localmvnrepl = "file://%s" % localmvn
publicmvn = mx.repository("python-public-snapshots").url
publicmvnpattern = re.compile(publicmvn)
git = mx.GitConfig()
new_rewrites = [{publicmvnpattern.pattern: {"replacement": localmvnrepl}}]
for rewrite in _urlrewrites:
if rewrite.pattern.match(publicmvn):
# we replace rewrites of our public repo
pass
elif publicmvnpattern.match(rewrite.replacement):
# we rewrite to what we want
new_rewrites.append({rewrite.pattern.pattern: {"replacement": localmvnrepl}})
else:
new_rewrites.append({rewrite.pattern.pattern: {"replacement": rewrite.replacement}})
os.environ["TRUFFLE_PYTHON_VERSION"] = git.tip(_suite.dir).strip()
os.environ["TRUFFLE_SULONG_VERSION"] = git.tip(_sulong.dir).strip()
prev_urlrewrites = os.environ.get("MX_URLREWRITES")
os.environ["MX_URLREWRITES"] = json.dumps(new_rewrites)
mx.command_function("deploy-binary")(["--all-suites", "python-local-snapshots", localmvnrepl])
try:
mx.log(str(dict(os.environ)))
testdownstream(
_suite,
[mx.suite("truffle").vc._remote_url(mx.suite("truffle").dir, "origin")],
"substratevm",
commands)
finally:
if binary:
os.environ.pop("TRUFFLE_PYTHON_VERSION")
os.environ.pop("TRUFFLE_SULONG_VERSION")
if prev_urlrewrites:
os.environ["MX_URLREWRITES"] = prev_urlrewrites
else:
os.environ.pop("MX_URLREWRITES")
shutil.rmtree(localmvn, ignore_errors=True)
def build_truffleruby(args=[]):
# Only build the necessary distributions to avoid building Sulong tests
truffleruby_dists = [
'TRUFFLERUBY',
'TRUFFLERUBY-LAUNCHER',
'TRUFFLERUBY-ZIP',
'TRUFFLERUBY_GRAALVM_SUPPORT',
'TRUFFLERUBY-TEST',
'TRUFFLERUBY-SPECS',
]
mx.command_function('sversions')([])
mx.command_function('build')([
'--force-javac',
'--warning-as-error',
'--force-deprecation-as-warning',
'-A-Xmaxerrs',
'-A1000', # show more than default 100 errors not to hide actual errors under many missing symbols
'--dependencies',
','.join(truffleruby_dists)
])
def gate_unittests(args=[], subdir=""):
_graalpytest_driver = "graalpython/com.oracle.graal.python.test/src/graalpytest.py"
_test_project = "graalpython/com.oracle.graal.python.test/"
for idx, arg in enumerate(args):
if arg.startswith("--subdir="):
subdir = args.pop(idx).split("=")[1]
break
test_args = [_graalpytest_driver, "-v", _test_project + "src/tests/" + subdir]
if "--" in args:
idx = args.index("--")
pre_args = args[:idx]
post_args = args[idx + 1:]
else:
pre_args = []
post_args = args
mx.command_function("python")(["--python.CatchAllExceptions=true"] + pre_args + test_args + post_args)
if platform.system() != 'Darwin':
# TODO: re-enable when python3 is available on darwin
mx.log("Running tests with CPython")
mx.run(["python3"] + test_args, nonZeroIsFatal=True)
def python_gate(args):
if not os.environ.get("JDT"):
find_jdt()
if not os.environ.get("ECLIPSE_EXE"):
find_eclipse()
if "--tags" not in args:
args += [
"--tags",
"fullbuild,style,python-junit,python-unittest,python-license,python-downstream"
]
return mx.command_function("gate")(args)
def deploy_binary_if_master(args):
"""If the active branch is 'master', deploy binaries for the primary suite to remote maven repository."""
primary_branch = 'master'
_, vc_root = mx.VC.get_vc_root(_suite.dir)
active_branch = mx.VC.get_vc(vc_root).active_branch(_suite.dir)
deploy_binary = mx.command_function('deploy-binary')
if active_branch == primary_branch:
return deploy_binary(args)
else:
mx.warn('The active branch is "%s". Binaries are deployed only if the active branch is "%s".' % (active_branch, primary_branch))
return 0
def deploy_binary_if_master(args):
"""if the active branch is 'master', deploy binaries for the primary suite to remote maven repository."""
master_branch = 'master'
active_branch = mx.VC.get_vc(_suite.dir).active_branch(_suite.dir)
if active_branch == master_branch:
if sys.platform == "darwin":
args.insert(0, "--platform-dependent")
return mx.command_function('deploy-binary')(args)
else:
mx.log('The active branch is "%s". Binaries are deployed only if the active branch is "%s".' % (
active_branch, master_branch))
return 0
def test_ruby(args):
if len(args) < 1 or len(args) > 2:
mx.abort('mx svm_test_ruby <ruby_svm_image_path> [<debug_build>=release]')
aot_bin = args[0]
debug_build = args[1] if len(args) >= 2 else 'release'
truffleruby_suite = truffle_language_ensure('ruby', extract=False)
suite_dir = truffleruby_suite.dir
distsToExtract = ['TRUFFLERUBY-ZIP', 'TRUFFLERUBY-SPECS']
lib = join(suite_dir, 'lib')
if not exists(lib):
# Binary suite, extract the distributions
for dist_name in distsToExtract:
mx.log('Extract distribution {} to {}'.format(dist_name, suite_dir))
dist = mx.distribution(dist_name)
with tarfile.open(dist.path, 'r:') as archive:
archive.extractall(suite_dir)
mx.command_function('ruby_testdownstream_aot')([aot_bin, 'spec', debug_build])
def ruby_deploy_binaries(args):
"""Deploy a binary suite for truffleruby"""
assert len(args) == 0
deploy_binary = mx.command_function('deploy-binary')
# Deploy platform-independent distributions only on Linux to avoid duplicates
if sys.platform.startswith('linux'):
return deploy_binary(
['--skip-existing', 'truffleruby-binary-snapshots'])
else:
return deploy_binary([
'--skip-existing', '--platform-dependent',
'truffleruby-binary-snapshots'
])
def _sdk_gate_runner(args, tasks):
with Task('SDK UnitTests', tasks, tags=['test']) as t:
if t:
unittest([
'--suite', 'sdk', '--enable-timing', '--verbose', '--fail-fast'
])
with Task('Check Copyrights', tasks) as t:
if t:
if mx.command_function('checkcopyrights')(
['--primary', '--', '--projects', 'src']) != 0:
t.abort(
'Copyright errors found. Please run "mx checkcopyrights --primary -- --fix" to fix them.'
)
def runSuite(args=None):
"""executes all the test suites or selected ones (see -h or --help)"""
vmArgs, otherArgs = mx_sulong.truffle_extract_VM_args(args)
parser = argparse.ArgumentParser(
description="Compiles all or selected test suites.")
parser.add_argument('suite',
nargs='*',
help=' '.join(testSuites.keys()),
default=testSuites.keys())
parsedArgs = parser.parse_args(otherArgs)
tasks = []
with mx_gate.Task('BuildJavaWithJavac', tasks) as t:
if t:
mx.command_function('build')(
['-p', '--warning-as-error', '--force-javac'])
for testSuiteName in parsedArgs.suite:
with mx_gate.Task('Test%s' % testSuiteName.capitalize(), tasks) as t:
if t:
_, runCommand = testSuites[testSuiteName]
runCommand(vmArgs)
def deploy_binary_if_master_or_release(args):
"""If the active branch is 'master' or starts with 'release', deploy binaries for the primary suite."""
assert len(args) == 0
active_branch = mx.VC.get_vc(root).active_branch(root)
deploy_binary = mx.command_function('deploy-binary')
if active_branch == 'master' or active_branch.startswith('release'):
# Deploy platform-independent distributions only on Linux to avoid duplicates
if sys.platform.startswith('linux'):
return deploy_binary(['--skip-existing', 'truffleruby-binary-snapshots'])
else:
return deploy_binary(['--skip-existing', '--platform-dependent', 'truffleruby-binary-snapshots'])
else:
mx.log('The active branch is "%s". Binaries are deployed only if the active branch is "master" or starts with "release".' % (active_branch))
return 0
def travis1(args=None):
"""executes the first Travis job (Javac build, benchmarks, polyglot, interop, tck, asm, types, and LLVM test cases)"""
tasks = []
with Task('BuildJavaWithJavac', tasks) as t:
if t: mx.command_function('build')(['-p', '--warning-as-error', '--force-javac'])
with Task('TestBenchmarks', tasks) as t:
if t: runBenchmarkTestCases()
with Task('TestPolglot', tasks) as t:
if t: runPolyglotTestCases()
with Task('TestInterop', tasks) as t:
if t: runInteropTestCases()
with Task('TestTck', tasks) as t:
if t: runTckTestCases()
with Task('TestAsm', tasks) as t:
if t: runAsmTestCases()
with Task('TestTypes', tasks) as t:
if t: runTypeTestCases()
with Task('TestLLVM', tasks) as t:
if t: runLLVMTestCases()
with Task('TestMainArgs', tasks) as t:
if t: runMainArgTestCases()
with Task('TestPipe', tasks) as t:
if t: runPipeTestCases()
def _sulong_gate_runner(args, tasks):
with Task('CheckCopyright', tasks, tags=['style']) as t:
if t:
if mx.checkcopyrights(['--primary']) != 0:
t.abort('Copyright errors found. Please run "mx checkcopyrights --primary -- --fix" to fix them.')
with Task('BuildLLVMorg', tasks, tags=['style', 'clangformat']) as t:
# needed for clang-format
if t: build_llvm_org(args)
with Task('ClangFormat', tasks, tags=['style', 'clangformat']) as t:
if t: clangformatcheck()
_sulong_gate_testsuite('Benchmarks', 'shootout', tasks, args, tags=['benchmarks', 'sulongMisc'])
_sulong_gate_unittest('Types', 'SULONG_TEST', tasks, args, tags=['type', 'sulongMisc', 'sulongCoverage'], testClasses=['com.oracle.truffle.llvm.tests.types.floating'])
_sulong_gate_unittest('Pipe', 'SULONG_TEST', tasks, args, tags=['pipe', 'sulongMisc', 'sulongCoverage'], testClasses=['CaptureOutputTest'])
_sulong_gate_testsuite('LLVM', 'llvm', tasks, args, tags=['llvm', 'sulongCoverage'])
_sulong_gate_testsuite('NWCC', 'nwcc', tasks, args, tags=['nwcc', 'sulongCoverage'])
_sulong_gate_testsuite('GCCParserTorture', 'parserTorture', tasks, args, tags=['parser', 'sulongCoverage'], vmArgs=['-Dpolyglot.llvm.parseOnly=true'])
_sulong_gate_testsuite('GCC_C', 'gcc_c', tasks, args, tags=['gcc_c', 'sulongCoverage'])
_sulong_gate_testsuite('GCC_CPP', 'gcc_cpp', tasks, args, tags=['gcc_cpp', 'sulongCoverage'])
_sulong_gate_testsuite('GCC_Fortran', 'gcc_fortran', tasks, args, tags=['gcc_fortran', 'sulongCoverage'])
_sulong_gate_sulongsuite_unittest('Sulong', tasks, args, testClasses='SulongSuite', tags=['sulong', 'sulongBasic', 'sulongCoverage'])
_sulong_gate_unittest('SulongLL', 'SULONG_LL_TEST_SUITES', tasks, args, testClasses='com.oracle.truffle.llvm.tests.bitcode.', tags=['sulongLL', 'sulongCoverage'])
_sulong_gate_sulongsuite_unittest('Interop', tasks, args, testClasses='com.oracle.truffle.llvm.tests.interop', tags=['interop', 'sulongBasic', 'sulongCoverage'])
_sulong_gate_sulongsuite_unittest('Linker', tasks, args, testClasses='com.oracle.truffle.llvm.tests.linker', tags=['linker', 'sulongBasic', 'sulongCoverage'])
_sulong_gate_sulongsuite_unittest('Debug', tasks, args, testClasses='LLVMDebugTest', tags=['debug', 'sulongBasic', 'sulongCoverage'])
_sulong_gate_sulongsuite_unittest('IRDebug', tasks, args, testClasses='LLVMIRDebugTest', tags=['irdebug', 'sulongBasic', 'sulongCoverage'])
_sulong_gate_sulongsuite_unittest('BitcodeFormat', tasks, args, testClasses='BitcodeFormatTest', tags=['bitcodeFormat', 'sulongBasic', 'sulongCoverage'])
_sulong_gate_sulongsuite_unittest('DebugExpr', tasks, args, testClasses='LLVMDebugExprParserTest', tags=['debugexpr', 'sulongBasic', 'sulongCoverage'])
_sulong_gate_sulongsuite_unittest('OtherTests', tasks, args, testClasses='com.oracle.truffle.llvm.tests.other', tags=['otherTests', 'sulongBasic', 'sulongCoverage'])
_sulong_gate_testsuite('Assembly', 'inlineassemblytests', tasks, args, testClasses='InlineAssemblyTest', tags=['assembly', 'sulongMisc', 'sulongCoverage'])
_sulong_gate_testsuite('Args', 'other', tasks, args, tags=['args', 'sulongMisc', 'sulongCoverage'], testClasses=['com.oracle.truffle.llvm.tests.MainArgsTest'])
_sulong_gate_testsuite('Callback', 'other', tasks, args, tags=['callback', 'sulongMisc', 'sulongCoverage'], testClasses=['com.oracle.truffle.llvm.tests.CallbackTest'])
_sulong_gate_testsuite('Varargs', 'other', tasks, args, tags=['vaargs', 'sulongMisc', 'sulongCoverage'], testClasses=['com.oracle.truffle.llvm.tests.VAArgsTest'])
with Task('TestToolchain', tasks, tags=['toolchain', 'sulongMisc', 'sulongCoverage']) as t:
if t:
mx.command_function('clean')(['--project', 'toolchain-launchers-tests'] + args.extra_build_args)
mx.command_function('build')(['--project', 'toolchain-launchers-tests'] + args.extra_build_args)
def travis1(args=None):
"""executes the first Travis job (ECJ and Javac build, findbugs, benchmarks, polyglot, interop, tck, asm, types, Sulong, and LLVM test cases)"""
tasks = []
with Task('BuildJavaWithEcj', tasks) as t:
if t:
if mx.get_env('JDT'):
mx.command_function('build')(
['-p', '--no-native', '--warning-as-error'])
gate_clean([], tasks, name='CleanAfterEcjBuild')
else:
mx._warn_or_abort(
'JDT environment variable not set. Cannot execute BuildJavaWithEcj task.',
args.strict_mode)
with Task('BuildJavaWithJavac', tasks) as t:
if t:
mx.command_function('build')(
['-p', '--warning-as-error', '--no-native', '--force-javac'])
with Task('Findbugs', tasks) as t:
if t and mx_findbugs.findbugs([]) != 0:
t.abort('FindBugs warnings were found')
with Task('TestBenchmarks', tasks) as t:
if t: runBenchmarkTestCases()
with Task('TestPolglot', tasks) as t:
if t: runPolyglotTestCases()
with Task('TestInterop', tasks) as t:
if t: runInteropTestCases()
with Task('TestTck', tasks) as t:
if t: runTckTestCases()
with Task('TestAsm', tasks) as t:
if t: runAsmTestCases()
with Task('TestTypes', tasks) as t:
if t: runTypeTestCases()
with Task('TestSulong', tasks) as t:
if t: runTruffleTestCases()
with Task('TestLLVM', tasks) as t:
if t: runLLVMTestCases()
def python_gate(args):
if not os.environ.get("JDT"):
find_jdt()
if not os.environ.get("ECLIPSE_EXE"):
find_eclipse()
if "--tags" not in args:
args += ["--tags"]
tags = ["style"]
include_sandboxed = mx.suite("sulong-managed", fatalIfMissing=False) is not None
for x in dir(GraalPythonTags):
v = getattr(GraalPythonTags, x)
if isinstance(v, str) and v.startswith("python-"):
if include_sandboxed and "sandboxed" in v:
tags.append(v)
elif not include_sandboxed and "sandboxed" not in v:
tags.append(v)
args.append(",".join(tags))
mx.log("Running mx python-gate " + " ".join(args))
return mx.command_function("gate")(args)
def python_checkcopyrights(args):
# we wan't to ignore lib-python/3, because that's just crazy
listfilename = tempfile.mktemp()
with open(listfilename, "w") as listfile:
mx.run(["git", "ls-tree", "-r", "HEAD", "--name-only"], out=listfile)
with open(listfilename, "r") as listfile:
content = listfile.read()
with open(listfilename, "w") as listfile:
for line in content.split("\n"):
if "lib-python/3" in line:
pass
elif os.path.splitext(line)[1] in [".py", ".java", ".c", ".h", ".sh"]:
listfile.write(line)
listfile.write("\n")
try:
r = mx.command_function("checkcopyrights")(["--primary", "--", "--file-list", listfilename] + args)
if r != 0:
mx.abort("copyrights check failed")
finally:
os.unlink(listfilename)
def _run_gate(cleanArgs, args, tasks):
global _jacoco
with Task('Versions', tasks, tags=[Tags.always]) as t:
if t:
mx.command_function('version')(['--oneline'])
mx.command_function('sversions')([])
mx.log("Python version: {}".format(sys.version_info))
with Task('JDKReleaseInfo', tasks, tags=[Tags.always]) as t:
if t:
jdkDirs = os.pathsep.join([mx.get_env('JAVA_HOME', ''), mx.get_env('EXTRA_JAVA_HOMES', '')])
for jdkDir in jdkDirs.split(os.pathsep):
release = join(jdkDir, 'release')
if exists(release):
mx.log('==== ' + jdkDir + ' ====')
with open(release) as fp:
mx.log(fp.read().strip())
if mx.primary_suite() is mx._mx_suite:
_run_mx_suite_tests()
with Task('VerifyMultiReleaseProjects', tasks, tags=[Tags.always]) as t:
if t:
mx.command_function('verifymultireleaseprojects')([])
for suiteRunner in _pre_gate_runners:
suite, runner = suiteRunner
if args.all_suites or suite is mx.primary_suite():
runner(args, tasks)
with Task('Pylint', tasks, tags=[Tags.style]) as t:
if t:
if mx.command_function('pylint')(['--primary']) != 0:
_warn_or_abort('Pylint not configured correctly. Cannot execute Pylint task.', args.strict_mode)
gate_clean(cleanArgs, tasks, tags=[Tags.build, Tags.fullbuild, Tags.ecjbuild])
with Task('Distribution Overlap Check', tasks, tags=[Tags.style]) as t:
if t:
if mx.command_function('checkoverlap')([]) != 0:
t.abort('Found overlapping distributions.')
with Task('Canonicalization Check', tasks, tags=[Tags.style]) as t:
if t:
mx.log(time.strftime('%d %b %Y %H:%M:%S - Ensuring mx/projects files are canonicalized...'))
if mx.command_function('canonicalizeprojects')([]) != 0:
t.abort('Rerun "mx canonicalizeprojects" and modify the suite.py files as suggested.')
with Task('Verify Java Sources in Project', tasks, tags=[Tags.style]) as t:
if t:
mx.log(time.strftime('%d %b %Y %H:%M:%S - Ensuring all Java sources are in a Java project directory...'))
if mx.command_function('verifysourceinproject')([]) != 0:
t.abort('Move or delete the Java sources that are not in a Java project directory.')
if mx._is_supported_by_jdt(mx.DEFAULT_JDK_TAG):
with Task('BuildWithEcj', tasks, tags=[Tags.fullbuild, Tags.ecjbuild], legacyTitles=['BuildJavaWithEcj']) as t:
if t:
defaultBuildArgs = ['-p']
fullbuild = True if Task.tags is None else Tags.fullbuild in Task.tags # pylint: disable=unsupported-membership-test
# Using ecj alone is not compatible with --warning-as-error (see GR-3969)
if not args.no_warning_as_error and fullbuild:
defaultBuildArgs += ['--warning-as-error']
if mx.get_env('JDT'):
mx.command_function('build')(defaultBuildArgs + args.extra_build_args)
if fullbuild:
gate_clean(cleanArgs, tasks, name='CleanAfterEcjBuild', tags=[Tags.fullbuild])
else:
_warn_or_abort('JDT environment variable not set. Cannot execute BuildWithEcj task.', args.strict_mode)
with Task('BuildWithJavac', tasks, tags=[Tags.build, Tags.fullbuild], legacyTitles=['BuildJavaWithJavac']) as t:
if t:
defaultBuildArgs = ['-p']
if not args.no_warning_as_error:
defaultBuildArgs += ['--warning-as-error']
mx.command_function('build')(defaultBuildArgs + ['--force-javac'] + args.extra_build_args)
with Task('IDEConfigCheck', tasks, tags=[Tags.fullbuild]) as t:
if t:
if args.cleanIDE:
mx.command_function('ideclean')([])
mx.command_function('ideinit')([])
with Task('CodeFormatCheck', tasks, tags=[Tags.style]) as t:
if t:
eclipse_exe = mx.get_env('ECLIPSE_EXE')
if eclipse_exe is not None:
if mx.command_function('eclipseformat')(['-e', eclipse_exe, '--primary']) != 0:
t.abort('Formatter modified files - run "mx eclipseformat", check in changes and repush')
else:
_warn_or_abort('ECLIPSE_EXE environment variable not set. Cannot execute CodeFormatCheck task.', args.strict_mode)
with Task('Checkstyle', tasks, tags=[Tags.style]) as t:
if t and mx.command_function('checkstyle')(['--primary']) != 0:
t.abort('Checkstyle warnings were found')
with Task('SpotBugs', tasks, tags=[Tags.fullbuild]) as t:
if t and mx.command_function('spotbugs')([]) != 0:
t.abort('FindBugs warnings were found')
with Task('VerifyLibraryURLs', tasks, tags=[Tags.fullbuild]) as t:
if t:
mx.command_function('verifylibraryurls')([])
jacoco_exec = get_jacoco_dest_file()
if exists(jacoco_exec):
os.unlink(jacoco_exec)
if args.jacocout is not None:
_jacoco = 'append'
else:
_jacoco = 'off'
for suiteRunner in _gate_runners:
suite, runner = suiteRunner
if args.all_suites or suite is mx.primary_suite():
runner(args, tasks)
if args.jacocout is not None:
jacoco_args = [args.jacocout]
if args.jacoco_omit_excluded:
jacoco_args = ['--omit-excluded'] + jacoco_args
mx.command_function('jacocoreport')(jacoco_args)
_jacoco = 'off'
if args.jacoco_zip is not None:
mx.log('Creating JaCoCo report archive: {}'.format(args.jacoco_zip))
with zipfile.ZipFile(args.jacoco_zip, 'w', compression=zipfile.ZIP_DEFLATED) as zf:
zf.write(jacoco_exec, join(args.jacocout, jacoco_exec))
for root, _, files in os.walk(args.jacocout):
for f in files:
zf.write(os.path.join(root, f))
mx.log('Archiving done.')
def gate_clean(cleanArgs, tasks, name='Clean', tags=None):
with Task(name, tasks, tags=tags) as t:
if t:
mx.command_function('clean')(cleanArgs)
def travisJRuby(args=None):
tasks = []
with Task('BuildJavaWithJavac', tasks) as t:
if t: mx.command_function('build')(['-p', '--warning-as-error', '--no-native', '--force-javac'])
with Task('TestJRuby', tasks) as t:
if t: runTestJRuby()
def rbench(args):
if _bm_suite:
mx.command_function('r_benchmarks:rbench')(args)
else:
mx.abort("no benchmarks available")
def junit_gate(args):
return mx.command_function('junit')(['--tests', _gate_unit_tests()] + args)
def junit_default(args):
return mx.command_function('junit')(['--tests', _all_unit_tests()] + args)
def build_truffleruby(args):
mx.command_function('sversions')([])
jt('build', '--no-sforceimports')
def gate_clean(cleanArgs, tasks, name='Clean'):
with Task(name, tasks) as t:
if t:
mx.command_function('clean')(cleanArgs)
def gate(args):
"""run the tests used to validate a push
If this command exits with a 0 exit code, then the gate passed."""
parser = ArgumentParser(prog='mx gate')
add_omit_clean_args(parser)
parser.add_argument('--all-suites', action='store_true', help='run gate tasks for all suites, not just the primary suite')
parser.add_argument('--dry-run', action='store_true', help='just show the tasks that will be run without running them')
parser.add_argument('-x', action='store_true', help='makes --task-filter an exclusion instead of inclusion filter')
parser.add_argument('--jacocout', help='specify the output directory for jacoco report')
parser.add_argument('--strict-mode', action='store_true', help='abort if a task cannot be executed due to missing tool configuration')
filtering = parser.add_mutually_exclusive_group()
filtering.add_argument('-t', '--task-filter', help='comma separated list of substrings to select subset of tasks to be run')
filtering.add_argument('-s', '--start-at', help='substring to select starting task')
for a, k in _extra_gate_arguments:
parser.add_argument(*a, **k)
args = parser.parse_args(args)
cleanArgs = check_gate_noclean_arg(args)
global _jacoco
if args.dry_run:
Task.dryRun = True
if args.start_at:
Task.startAtFilter = args.start_at
elif args.task_filter:
Task.filters = args.task_filter.split(',')
Task.filtersExclude = args.x
elif args.x:
mx.abort('-x option cannot be used without --task-filter option')
tasks = []
total = Task('Gate')
try:
with Task('Versions', tasks) as t:
if t:
mx.command_function('version')(['--oneline'])
mx.command_function('sversions')([])
with Task('JDKReleaseInfo', tasks) as t:
if t:
jdkDirs = os.pathsep.join([mx.get_env('JAVA_HOME', ''), mx.get_env('EXTRA_JAVA_HOMES', '')])
for jdkDir in jdkDirs.split(os.pathsep):
release = join(jdkDir, 'release')
if exists(release):
mx.log('==== ' + jdkDir + ' ====')
with open(release) as fp:
mx.log(fp.read().strip())
with Task('Pylint', tasks) as t:
if t:
if mx.command_function('pylint')(['--primary']) != 0:
_warn_or_abort('Pylint not configured correctly. Cannot execute Pylint task.', args.strict_mode)
gate_clean(cleanArgs, tasks)
with Task('Distribution Overlap Check', tasks) as t:
if t:
if mx.command_function('checkoverlap')([]) != 0:
t.abort('Found overlapping distributions.')
with Task('Canonicalization Check', tasks) as t:
if t:
mx.log(time.strftime('%d %b %Y %H:%M:%S - Ensuring mx/projects files are canonicalized...'))
if mx.command_function('canonicalizeprojects')([]) != 0:
t.abort('Rerun "mx canonicalizeprojects" and check-in the modified mx/suite*.py files.')
with Task('BuildJavaWithEcj', tasks) as t:
if t:
if mx.get_env('JDT'):
mx.command_function('build')(['-p', '--no-native', '--warning-as-error'])
gate_clean(cleanArgs, tasks, name='CleanAfterEcjBuild')
else:
_warn_or_abort('JDT environment variable not set. Cannot execute BuildJavaWithEcj task.', args.strict_mode)
with Task('BuildJavaWithJavac', tasks) as t:
if t: mx.command_function('build')(['-p', '--warning-as-error', '--no-native', '--force-javac'])
with Task('IDEConfigCheck', tasks) as t:
if t:
if args.cleanIDE:
mx.command_function('ideclean')([])
mx.command_function('ideinit')([])
with Task('CodeFormatCheck', tasks) as t:
if t:
eclipse_exe = mx.get_env('ECLIPSE_EXE')
if eclipse_exe is not None:
if mx.command_function('eclipseformat')(['-e', eclipse_exe, '--primary']) != 0:
t.abort('Formatter modified files - run "mx eclipseformat", check in changes and repush')
else:
_warn_or_abort('ECLIPSE_EXE environment variable not set. Cannot execute CodeFormatCheck task.', args.strict_mode)
with Task('Checkstyle', tasks) as t:
if t and mx.command_function('checkstyle')(['--primary']) != 0:
t.abort('Checkstyle warnings were found')
with Task('Checkheaders', tasks) as t:
if t and mx.command_function('checkheaders')([]) != 0:
t.abort('Checkheaders warnings were found')
with Task('FindBugs', tasks) as t:
if t and mx.command_function('findbugs')([]) != 0:
t.abort('FindBugs warnings were found')
if exists('jacoco.exec'):
os.unlink('jacoco.exec')
if args.jacocout is not None:
_jacoco = 'append'
else:
_jacoco = 'off'
for suiteRunner in _gate_runners:
suite, runner = suiteRunner
if args.all_suites or suite is mx.primary_suite():
runner(args, tasks)
if args.jacocout is not None:
mx.command_function('jacocoreport')([args.jacocout])
_jacoco = 'off'
except KeyboardInterrupt:
total.abort(1)
except BaseException as e:
import traceback
traceback.print_exc()
total.abort(str(e))
total.stop()
mx.log('Gate task times:')
for t in tasks:
mx.log(' ' + str(t.duration) + '\t' + t.title)
mx.log(' =======')
mx.log(' ' + str(total.duration))
if args.task_filter:
Task.filters = None
'-Dgraal.CompileGraalWithC1Only=false',
'-XX:CICompilerCount=' + str(JVM_COMPILER_THREADS),
'-Dtruffle.TrustAllTruffleRuntimeProviders=true', # GR-7046
'-Dgraal.VerifyGraalGraphs=false',
'-Dgraal.VerifyGraalGraphEdges=false',
'-Dgraal.VerifyGraalPhasesSize=false',
'-Dgraal.VerifyPhases=false'
]
IMAGE_ASSERTION_FLAGS = [
'-H:+VerifyGraalGraphs', '-H:+VerifyGraalGraphEdges', '-H:+VerifyPhases'
]
suite = mx.suite('substratevm')
svmSuites = [suite]
orig_command_build = mx.command_function('build')
allow_native_image_build = True
def build(args, vm=None):
if any([opt in args for opt in ['-h', '--help']]):
orig_command_build(args, vm)
mx.log('build: Checking SubstrateVM requirements for building ...')
if not _host_os_supported():
mx.abort(
'build: SubstrateVM can be built only on Darwin and Linux platforms'
)
def graalpython_gate_runner(args, tasks):
with Task('GraalPython JUnit', tasks,
tags=[GraalPythonTags.junit]) as task:
if task:
punittest(['--verbose'])
with Task('GraalPython Python tests',
tasks,
tags=[GraalPythonTags.unittest]) as task:
if task:
test_args = [
"graalpython/com.oracle.graal.python.test/src/graalpytest.py",
"-v", "graalpython/com.oracle.graal.python.test/src/tests/"
]
mx.command_function("python")(test_args)
if platform.system() != 'Darwin':
# TODO: re-enable when python3 is available on darwin
mx.run(["python3"] + test_args, nonZeroIsFatal=True)
with Task('GraalPython downstream R tests',
tasks,
tags=[GraalPythonTags.downstream, GraalPythonTags.R]) as task:
script_r2p = os.path.join(_suite.dir, "graalpython", "benchmarks",
"src", "benchmarks", "interop",
"r_python_image_demo.r")
script_p2r = os.path.join(_suite.dir, "graalpython", "benchmarks",
"src", "benchmarks", "interop",
"python_r_image_demo.py")
pythonjars = os.pathsep.join([
os.path.join(_suite.dir, "mxbuild", "dists", "graalpython.jar"),
os.path.join(_suite.dir, "mxbuild", "dists", "graalpython-env.jar")
])
if task:
rrepo = os.environ["FASTR_REPO_URL"]
testdownstream(_suite, [
rrepo,
mx.suite("truffle").vc._remote_url(
mx.suite("truffle").dir, "origin")
], ".", [[
"--dynamicimports", "graalpython",
"--version-conflict-resolution", "latest_all", "build",
"--force-deprecation-as-warning"
],
[
"--cp-sfx", pythonjars, "r", "--polyglot",
"--file=%s" % script_r2p
]])
testdownstream(_suite, [
rrepo,
mx.suite("truffle").vc._remote_url(
mx.suite("truffle").dir, "origin")
], ".", [[
"--dynamicimports", "graalpython",
"--version-conflict-resolution", "latest_all", "build",
"--force-deprecation-as-warning"
],
[
"-v", "--cp-sfx", pythonjars, "r", "--jvm",
"--polyglot", "-e",
"eval.polyglot('python', path='%s')" % str(script_p2r)
]])
with Task('GraalPython license header update',
tasks,
tags=[GraalPythonTags.license]) as task:
if task:
python_license_headers_update([])
with Task('GraalPython downstream svm binary tests',
tasks,
tags=[GraalPythonTags.downstream,
GraalPythonTags.svmbinary]) as task:
if task:
_run_downstream_svm(
[[
"--dynamicimports", "graalpython",
"delete-graalpython-if-testdownstream"
],
[
"gate", '-B--force-deprecation-as-warning', "--tags",
"build,python"
]],
binary=True)
with Task('GraalPython downstream svm source tests',
tasks,
tags=[GraalPythonTags.downstream,
GraalPythonTags.svmsource]) as task:
if task:
_run_downstream_svm([[
"--dynamicimports", "graalpython", "--strict-compliance",
"gate", '-B--force-deprecation-as-warning', "--strict-mode",
"--tags", "build,python"
]])
for name, iterations in sorted(python_test_benchmarks.iteritems()):
with Task('PythonBenchmarksTest:' + name,
tasks,
tags=[GraalPythonTags.benchmarks]) as task:
if task:
_gate_python_benchmarks_tests(
"graalpython/benchmarks/src/benchmarks/" + name + ".py",
iterations)
checkCases = {
'gitlog' : logCheck,
'mdl' : mdlCheck,
'ecj' : compileWithEcjStrict,
'checkstyle' : mx.checkstyle,
'findbugs' : findBugs,
'canonicalizeprojects' : mx.canonicalizeprojects,
'httpcheck' : checkNoHttp,
'checkoverlap' : mx.checkoverlap,
'clangformatcheck' : clangformatcheck,
'pylint' : mx.pylint,
'eclipseformat' : (lambda args: mx.eclipseformat(['--primary'] + args))
}
originalBuildCommand = mx.command_function('build')
mx.update_commands(_suite, {
'suoptbench' : [suOptBench, ''],
'subench' : [suBench, ''],
'clangbench' : [clangBench, ''],
'gccbench' : [gccBench, ''],
'build' : [sulongBuild, ''],
'su-options' : [printOptions, ''],
'su-pullbenchmarkgame' : [pullBenchmarkGame, ''],
'su-pulldeps' : [downloadDependencies, ''],
'su-pullllvmbinaries' : [pullLLVMBinaries, ''],
'su-pullnwccsuite' : [pullNWCCSuite, ''],
'su-pullgccsuite' : [pullGCCSuite, ''],
'su-pullllvmsuite' : [pullLLVMSuite, ''],
'su-pulltools' : [pullTools, ''],
def build_llvm_org(args=None):
defaultBuildArgs = ['-p']
if not args.no_warning_as_error:
defaultBuildArgs += ['--warning-as-error']
mx.command_function('build')(defaultBuildArgs + ['--project', 'SULONG_LLVM_ORG'] + args.extra_build_args)
def junit_simple(args):
return mx.command_function('junit')(['--tests', _library_unit_tests()] + args)
def compileTestSuite(testsuiteproject, extra_build_args):
defaultBuildArgs = ['--project', testsuiteproject]
mx.command_function('build')(defaultBuildArgs + extra_build_args)
GRAAL_COMPILER_FLAGS += ['--add-opens', 'java.base/java.lang.ref=ALL-UNNAMED']
# Reflective access to org.graalvm.nativeimage.impl.ImageSingletonsSupport.
GRAAL_COMPILER_FLAGS += ['--add-exports', 'org.graalvm.graal_sdk/org.graalvm.nativeimage.impl=ALL-UNNAMED']
# Reflective access to jdk.internal.ref.CleanerImpl$PhantomCleanableRef.
GRAAL_COMPILER_FLAGS += ['--add-opens', 'java.base/jdk.internal.ref=ALL-UNNAMED']
# Disable the check for JDK-8 graal version.
GRAAL_COMPILER_FLAGS += ['-Dsubstratevm.IgnoreGraalVersionCheck=true']
# Reflective access to java.net.URL.getURLStreamHandler.
GRAAL_COMPILER_FLAGS += ['--add-opens', 'java.base/java.net=ALL-UNNAMED']
IMAGE_ASSERTION_FLAGS = ['-H:+VerifyGraalGraphs', '-H:+VerifyGraalGraphEdges', '-H:+VerifyPhases']
suite = mx.suite('substratevm')
svmSuites = [suite]
orig_command_gate = mx.command_function('gate')
orig_command_build = mx.command_function('build')
gate_run = False
def gate(args):
global gate_run
gate_run = True
orig_command_gate(args)
def build(args, vm=None):
if any([opt in args for opt in ['-h', '--help']]):
orig_command_build(args, vm)
mx.log('build: Checking SubstrateVM requirements for building ...')
def junit_default(args):
return mx.command_function(
'junit')(['--tests', _default_unit_tests()] + args)
def junit_nopkgs(args):
return mx.command_function('junit')(['--tests', ','.join([_simple_unit_tests(), _nodes_unit_tests()])] + args)
def build_truffleruby(args):
mx.command_function('sversions')([])
jt('build')
def gate(args):
"""run the tests used to validate a push
If this command exits with a 0 exit code, then the gate passed."""
parser = ArgumentParser(prog='mx gate')
add_omit_clean_args(parser)
parser.add_argument('--all-suites', action='store_true', help='run gate tasks for all suites, not just the primary suite')
parser.add_argument('--dry-run', action='store_true', help='just show the tasks that will be run without running them')
parser.add_argument('-x', action='store_true', help='makes --task-filter or --tags an exclusion instead of inclusion filter')
parser.add_argument('--jacocout', help='specify the output directory for jacoco report')
parser.add_argument('--strict-mode', action='store_true', help='abort if a task cannot be executed due to missing tool configuration')
parser.add_argument('-B', dest='extra_build_args', action='append', metavar='<build_args>', help='append additional arguments to mx build commands used in the gate')
filtering = parser.add_mutually_exclusive_group()
filtering.add_argument('-t', '--task-filter', help='comma separated list of substrings to select subset of tasks to be run')
filtering.add_argument('-s', '--start-at', help='substring to select starting task')
filtering.add_argument('--tags', help='comma separated list of tags to select subset of tasks to be run. Tags can have a range specifier `name[:from:[to]]`.'
'If present only the [from,to) tasks are executed. If `to` is omitted all tasks starting with `from` are executed.')
for a, k in _extra_gate_arguments:
parser.add_argument(*a, **k)
args = parser.parse_args(args)
cleanArgs = check_gate_noclean_arg(args)
global _jacoco
if args.dry_run:
Task.dryRun = True
if args.start_at:
Task.startAtFilter = args.start_at
elif args.task_filter:
Task.filters = args.task_filter.split(',')
Task.filtersExclude = args.x
elif args.tags:
parse_tags_argument(args.tags, args.x)
Task.tagsExclude = args.x
if not Task.tagsExclude:
# implicitly include 'always'
Task.tags += [Tags.always]
elif args.x:
mx.abort('-x option cannot be used without --task-filter or the --tags option')
if not args.extra_build_args:
args.extra_build_args = []
tasks = []
total = Task('Gate')
try:
with Task('Versions', tasks, tags=[Tags.always]) as t:
if t:
mx.command_function('version')(['--oneline'])
mx.command_function('sversions')([])
with Task('JDKReleaseInfo', tasks, tags=[Tags.always]) as t:
if t:
jdkDirs = os.pathsep.join([mx.get_env('JAVA_HOME', ''), mx.get_env('EXTRA_JAVA_HOMES', '')])
for jdkDir in jdkDirs.split(os.pathsep):
release = join(jdkDir, 'release')
if exists(release):
mx.log('==== ' + jdkDir + ' ====')
with open(release) as fp:
mx.log(fp.read().strip())
for suiteRunner in _pre_gate_runners:
suite, runner = suiteRunner
if args.all_suites or suite is mx.primary_suite():
runner(args, tasks)
with Task('Pylint', tasks, tags=[Tags.style]) as t:
if t:
if mx.command_function('pylint')(['--primary']) != 0:
_warn_or_abort('Pylint not configured correctly. Cannot execute Pylint task.', args.strict_mode)
gate_clean(cleanArgs, tasks, tags=[Tags.build, Tags.fullbuild])
with Task('Distribution Overlap Check', tasks, tags=[Tags.style]) as t:
if t:
if mx.command_function('checkoverlap')([]) != 0:
t.abort('Found overlapping distributions.')
with Task('Canonicalization Check', tasks, tags=[Tags.style]) as t:
if t:
mx.log(time.strftime('%d %b %Y %H:%M:%S - Ensuring mx/projects files are canonicalized...'))
if mx.command_function('canonicalizeprojects')([]) != 0:
t.abort('Rerun "mx canonicalizeprojects" and modify the suite.py files as suggested.')
with Task('Verify Java Sources in Project', tasks, tags=[Tags.style]) as t:
if t:
mx.log(time.strftime('%d %b %Y %H:%M:%S - Ensuring all Java sources are in a Java project directory...'))
if mx.command_function('verifysourceinproject')([]) != 0:
t.abort('Move or delete the Java sources that are not in a Java project directory.')
if mx._is_supported_by_jdt(mx.DEFAULT_JDK_TAG):
with Task('BuildWithEcj', tasks, tags=[Tags.fullbuild], legacyTitles=['BuildJavaWithEcj']) as t:
if t:
if mx.get_env('JDT'):
mx.command_function('build')(['-p', '--warning-as-error'] + args.extra_build_args)
gate_clean(cleanArgs, tasks, name='CleanAfterEcjBuild', tags=[Tags.fullbuild])
else:
_warn_or_abort('JDT environment variable not set. Cannot execute BuildWithEcj task.', args.strict_mode)
with Task('BuildWithJavac', tasks, tags=[Tags.build, Tags.fullbuild], legacyTitles=['BuildJavaWithJavac']) as t:
if t: mx.command_function('build')(['-p', '--warning-as-error', '--force-javac'] + args.extra_build_args)
with Task('IDEConfigCheck', tasks, tags=[Tags.fullbuild]) as t:
if t:
if args.cleanIDE:
mx.command_function('ideclean')([])
mx.command_function('ideinit')([])
with Task('CodeFormatCheck', tasks, tags=[Tags.style]) as t:
if t:
eclipse_exe = mx.get_env('ECLIPSE_EXE')
if eclipse_exe is not None:
if mx.command_function('eclipseformat')(['-e', eclipse_exe, '--primary']) != 0:
t.abort('Formatter modified files - run "mx eclipseformat", check in changes and repush')
else:
_warn_or_abort('ECLIPSE_EXE environment variable not set. Cannot execute CodeFormatCheck task.', args.strict_mode)
with Task('Checkstyle', tasks, tags=[Tags.style]) as t:
if t and mx.command_function('checkstyle')(['--primary']) != 0:
t.abort('Checkstyle warnings were found')
with Task('Checkheaders', tasks, tags=[Tags.style]) as t:
if t and mx.command_function('checkheaders')([]) != 0:
t.abort('Checkheaders warnings were found')
with Task('FindBugs', tasks, tags=[Tags.fullbuild]) as t:
if t and mx.command_function('findbugs')([]) != 0:
t.abort('FindBugs warnings were found')
with Task('VerifyLibraryURLs', tasks, tags=[Tags.fullbuild]) as t:
if t:
mx.command_function('verifylibraryurls')([])
if mx._primary_suite is mx._mx_suite:
with Task('TestJMH', tasks, tags=[Tags.fullbuild]) as t:
if t: mx_microbench.get_microbenchmark_executor().microbench(['--', '-foe', 'true', 'com.oracle.mxtool.bench.TestJMH'])
if exists('jacoco.exec'):
os.unlink('jacoco.exec')
if args.jacocout is not None:
_jacoco = 'append'
else:
_jacoco = 'off'
for suiteRunner in _gate_runners:
suite, runner = suiteRunner
if args.all_suites or suite is mx.primary_suite():
runner(args, tasks)
if args.jacocout is not None:
mx.command_function('jacocoreport')([args.jacocout])
_jacoco = 'off'
except KeyboardInterrupt:
total.abort(1)
except BaseException as e:
import traceback
traceback.print_exc()
total.abort(str(e))
total.stop()
mx.log('Gate task times:')
for t in tasks:
mx.log(' ' + str(t.duration) + '\t' + t.title + ("" if not (Task.verbose and t.tags) else (' [' + ','.join(t.tags) + ']')))
mx.log(' =======')
mx.log(' ' + str(total.duration))
if args.task_filter:
Task.filters = None
