def graalpython_gate_runner(args, tasks):
with Task('GraalPython JUnit', tasks, tags=[GraalPythonTags.junit]) as task:
if task:
punittest(['--verbose'])
with Task('GraalPython Python tests', tasks, tags=[GraalPythonTags.unittest]) as task:
if task:
test_args = ["graalpython/com.oracle.graal.python.test/src/graalpytest.py", "-v",
"graalpython/com.oracle.graal.python.test/src/tests/"]
mx.command_function("python")(test_args)
if platform.system() != 'Darwin':
# TODO: re-enable when python3 is available on darwin
mx.run(["python3"] + test_args, nonZeroIsFatal=True)
with Task('GraalPython downstream R tests', tasks, tags=[GraalPythonTags.downstream, GraalPythonTags.R]) as task:
script_r2p = os.path.join(_suite.dir, "graalpython", "benchmarks", "src", "benchmarks", "interop", "r_python_image_demo.r")
script_p2r = os.path.join(_suite.dir, "graalpython", "benchmarks", "src", "benchmarks", "interop", "python_r_image_demo.py")
pythonjars = os.pathsep.join([
os.path.join(_suite.dir, "mxbuild", "dists", "graalpython.jar"),
os.path.join(_suite.dir, "mxbuild", "dists", "graalpython-env.jar")
])
if task:
rrepo = os.environ["FASTR_REPO_URL"]
testdownstream(
_suite,
[rrepo, mx.suite("truffle").vc._remote_url(mx.suite("truffle").dir, "origin")],
".",
[["--dynamicimports", "graalpython", "--version-conflict-resolution", "latest_all", "build", "--force-deprecation-as-warning"],
["--cp-sfx", pythonjars, "r", "--polyglot", "--file=%s" % script_r2p]
])
testdownstream(
_suite,
[rrepo, mx.suite("truffle").vc._remote_url(mx.suite("truffle").dir, "origin")],
".",
[["--dynamicimports", "graalpython", "--version-conflict-resolution", "latest_all", "build", "--force-deprecation-as-warning"],
["-v", "--cp-sfx", pythonjars, "r", "--jvm", "--polyglot", "-e", "eval.polyglot('python', path='%s')" % str(script_p2r)]
])
with Task('GraalPython license header update', tasks, tags=[GraalPythonTags.license]) as task:
if task:
python_license_headers_update([])
with Task('GraalPython downstream svm binary tests', tasks, tags=[GraalPythonTags.downstream, GraalPythonTags.svmbinary]) as task:
if task:
_run_downstream_svm(
[["--dynamicimports", "graalpython", "delete-graalpython-if-testdownstream"],
["gate", '-B--force-deprecation-as-warning', "--tags", "build,python"]],
binary=True
)
with Task('GraalPython downstream svm source tests', tasks, tags=[GraalPythonTags.downstream, GraalPythonTags.svmsource]) as task:
if task:
_run_downstream_svm([[
"--dynamicimports", "graalpython", "--strict-compliance", "gate", '-B--force-deprecation-as-warning', "--strict-mode", "--tags", "build,python"
]])
for name, iterations in sorted(python_test_benchmarks.iteritems()):
with Task('PythonBenchmarksTest:' + name, tasks, tags=[GraalPythonTags.benchmarks]) as task:
if task:
_gate_python_benchmarks_tests("graalpython/benchmarks/src/benchmarks/" + name + ".py", iterations)
def _run_downstream_svm(commands, binary=False):
new_rewrites = None
if binary:
localmvn = "/tmp/graalpythonsnapshots"
localmvnrepl = "file://%s" % localmvn
publicmvn = mx.repository("python-public-snapshots").url
publicmvnpattern = re.compile(publicmvn)
git = mx.GitConfig()
new_rewrites = [{
publicmvnpattern.pattern: {
"replacement": localmvnrepl
}
}]
for rewrite in _urlrewrites:
if rewrite.pattern.match(publicmvn):
# we replace rewrites of our public repo
pass
elif publicmvnpattern.match(rewrite.replacement):
# we rewrite to what we want
new_rewrites.append(
{rewrite.pattern.pattern: {
"replacement": localmvnrepl
}})
else:
new_rewrites.append({
rewrite.pattern.pattern: {
"replacement": rewrite.replacement
}
})
os.environ["TRUFFLE_PYTHON_VERSION"] = git.tip(_suite.dir).strip()
os.environ["TRUFFLE_SULONG_VERSION"] = git.tip(_sulong.dir).strip()
prev_urlrewrites = os.environ.get("MX_URLREWRITES")
os.environ["MX_URLREWRITES"] = json.dumps(new_rewrites)
mx.command_function("deploy-binary")(
["--all-suites", "python-local-snapshots", localmvnrepl])
try:
mx.log(str(dict(os.environ)))
testdownstream(_suite, [
mx.suite("truffle").vc._remote_url(
mx.suite("truffle").dir, "origin")
], "substratevm", commands)
finally:
if binary:
os.environ.pop("TRUFFLE_PYTHON_VERSION")
os.environ.pop("TRUFFLE_SULONG_VERSION")
if prev_urlrewrites:
os.environ["MX_URLREWRITES"] = prev_urlrewrites
else:
os.environ.pop("MX_URLREWRITES")
shutil.rmtree(localmvn, ignore_errors=True)
def graalpython_gate_runner(args, tasks):
_graalpytest_driver = "graalpython/com.oracle.graal.python.test/src/graalpytest.py"
_test_project = "graalpython/com.oracle.graal.python.test/"
with Task('GraalPython JUnit', tasks,
tags=[GraalPythonTags.junit]) as task:
if task:
punittest(['--verbose'])
with Task('GraalPython Python tests',
tasks,
tags=[GraalPythonTags.unittest]) as task:
if task:
test_args = [
_graalpytest_driver, "-v", _test_project + "src/tests/"
]
mx.command_function(
"python")(["--python.CatchAllExceptions=true"] + test_args)
if platform.system() != 'Darwin':
# TODO: re-enable when python3 is available on darwin
mx.log("Running tests with CPython")
mx.run(["python3"] + test_args, nonZeroIsFatal=True)
with Task('GraalPython C extension tests',
tasks,
tags=[GraalPythonTags.cpyext]) as task:
if task:
test_args = [
_graalpytest_driver, "-v", _test_project + "src/tests/cpyext/"
]
mx.command_function("python")(test_args)
if platform.system() != 'Darwin':
# TODO: re-enable when python3 is available on darwin
mx.log("Running tests with CPython")
mx.run(["python3"] + test_args, nonZeroIsFatal=True)
with Task('GraalPython Python tests on SVM',
tasks,
tags=[GraalPythonTags.svmunit]) as task:
if task:
if not os.path.exists("./graalpython-svm"):
python_svm(["-h"])
if os.path.exists("./graalpython-svm"):
langhome = mx_subst.path_substitutions.substitute(
'--native.Dllvm.home=<path:SULONG_LIBS>')
# tests root directory
tests_folder = "graalpython/com.oracle.graal.python.test/src/tests/"
# list of excluded tests
excluded = ["test_interop.py"]
def is_included(path):
if path.endswith(".py"):
basename = path.rpartition("/")[2]
return basename.startswith(
"test_") and basename not in excluded
return False
# list all 1st-level tests and exclude the SVM-incompatible ones
testfiles = []
paths = [tests_folder]
while paths:
path = paths.pop()
if is_included(path):
testfiles.append(path)
else:
try:
paths += [
(path + f if path.endswith("/") else "%s/%s" %
(path, f)) for f in os.listdir(path)
]
except OSError:
pass
test_args = [
"graalpython/com.oracle.graal.python.test/src/graalpytest.py",
"-v"
] + testfiles
mx.run([
"./graalpython-svm",
"--python.CoreHome=graalpython/lib-graalpython",
"--python.StdLibHome=graalpython/lib-python/3", langhome
] + test_args,
nonZeroIsFatal=True)
with Task('GraalPython downstream R tests',
tasks,
tags=[GraalPythonTags.downstream, GraalPythonTags.R]) as task:
script_r2p = os.path.join(_suite.dir, "graalpython", "benchmarks",
"src", "benchmarks", "interop",
"r_python_image_demo.r")
script_p2r = os.path.join(_suite.dir, "graalpython", "benchmarks",
"src", "benchmarks", "interop",
"python_r_image_demo.py")
pythonjars = os.pathsep.join([
os.path.join(_suite.dir, "mxbuild", "dists", "graalpython.jar"),
os.path.join(_suite.dir, "mxbuild", "dists", "graalpython-env.jar")
])
if task:
rrepo = os.environ["FASTR_REPO_URL"]
testdownstream(_suite, [
rrepo,
mx.suite("truffle").vc._remote_url(
mx.suite("truffle").dir, "origin")
], ".", [[
"--dynamicimports", "graalpython",
"--version-conflict-resolution", "latest_all", "build",
"--force-deprecation-as-warning"
],
[
"--cp-sfx", pythonjars, "r", "--polyglot",
"--file=%s" % script_r2p
]])
testdownstream(_suite, [
rrepo,
mx.suite("truffle").vc._remote_url(
mx.suite("truffle").dir, "origin")
], ".", [[
"--dynamicimports", "graalpython",
"--version-conflict-resolution", "latest_all", "build",
"--force-deprecation-as-warning"
],
[
"-v", "--cp-sfx", pythonjars, "r", "--jvm",
"--polyglot", "-e",
"eval.polyglot('python', path='%s')" % str(script_p2r)
]])
with Task('GraalPython apptests', tasks,
tags=[GraalPythonTags.apptests]) as task:
if task:
apprepo = os.environ["GRAALPYTHON_APPTESTS_REPO_URL"]
_apptest_suite = _suite.import_suite(
"graalpython-apptests",
urlinfos=[
mx.SuiteImportURLInfo(mx_urlrewrites.rewriteurl(apprepo),
"git", mx.vc_system("git"))
])
mx.run_mx(["-p", _apptest_suite.dir, "graalpython-apptests"])
with Task('GraalPython license header update',
tasks,
tags=[GraalPythonTags.license]) as task:
if task:
python_checkcopyrights([])
with Task('GraalPython GraalVM shared-library build',
tasks,
tags=[GraalPythonTags.downstream,
GraalPythonTags.graalvm]) as task:
if task:
run_shared_lib_test()
with Task('GraalPython GraalVM build',
tasks,
tags=[GraalPythonTags.downstream,
GraalPythonTags.graalvm]) as task:
if task:
svm_image = python_svm(["--version"])
benchmark = os.path.join("graalpython", "benchmarks", "src",
"benchmarks", "image_magix.py")
out = mx.OutputCapture()
mx.run([svm_image, benchmark],
nonZeroIsFatal=True,
out=mx.TeeOutputCapture(out))
success = "\n".join([
"[0, 0, 0, 0, 0, 0, 20, 20, 20, 0, 0, 20, 20, 20, 0, 0, 20, 20, 20, 0, 0, 0, 0, 0, 0]",
"[11, 12, 13, 14, 15, 21, 22, 23, 24, 25, 31, 32, 33, 34, 35, 41, 42, 43, 44, 45, 51, 52, 53, 54, 55]",
"[11, 12, 13, 14, 15, 21, 22, 23, 24, 25, 31, 32, 36, 36, 35, 41, 41, 40, 40, 45, 51, 52, 53, 54, 55]"
])
if success not in out.data:
mx.abort('Output from generated SVM image "' + svm_image +
'" did not match success pattern:\n' + success)
for name, iterations in sorted(python_test_benchmarks.iteritems()):
with Task('PythonBenchmarksTest:' + name,
tasks,
tags=[GraalPythonTags.benchmarks]) as task:
if task:
_gate_python_benchmarks_tests(
"graalpython/benchmarks/src/benchmarks/" + name + ".py",
iterations)
def graalpython_gate_runner(args, tasks):
with Task('GraalPython JUnit', tasks,
tags=[GraalPythonTags.junit]) as task:
if task:
punittest(['--verbose'])
with Task('GraalPython Python tests',
tasks,
tags=[GraalPythonTags.unittest]) as task:
if task:
test_args = [
"graalpython/com.oracle.graal.python.test/src/graalpytest.py",
"-v", "graalpython/com.oracle.graal.python.test/src/tests/"
]
mx.command_function("python")(test_args)
if platform.system() != 'Darwin':
# TODO: re-enable when python3 is available on darwin
mx.run(["python3"] + test_args, nonZeroIsFatal=True)
with Task('GraalPython downstream R tests',
tasks,
tags=[GraalPythonTags.downstream, GraalPythonTags.R]) as task:
script_r2p = os.path.join(_suite.dir, "graalpython", "benchmarks",
"src", "benchmarks", "interop",
"r_python_image_demo.r")
script_p2r = os.path.join(_suite.dir, "graalpython", "benchmarks",
"src", "benchmarks", "interop",
"python_r_image_demo.py")
pythonjars = os.pathsep.join([
os.path.join(_suite.dir, "mxbuild", "dists", "graalpython.jar"),
os.path.join(_suite.dir, "mxbuild", "dists", "graalpython-env.jar")
])
if task:
rrepo = os.environ["FASTR_REPO_URL"]
testdownstream(_suite, [
rrepo,
mx.suite("truffle").vc._remote_url(
mx.suite("truffle").dir, "origin")
], ".", [[
"--dynamicimports", "graalpython",
"--version-conflict-resolution", "latest_all", "build",
"--force-deprecation-as-warning"
],
[
"--cp-sfx", pythonjars, "r", "--polyglot",
"--file=%s" % script_r2p
]])
testdownstream(_suite, [
rrepo,
mx.suite("truffle").vc._remote_url(
mx.suite("truffle").dir, "origin")
], ".", [[
"--dynamicimports", "graalpython",
"--version-conflict-resolution", "latest_all", "build",
"--force-deprecation-as-warning"
],
[
"-v", "--cp-sfx", pythonjars, "r", "--jvm",
"--polyglot", "-e",
"eval.polyglot('python', path='%s')" % str(script_p2r)
]])
with Task('GraalPython license header update',
tasks,
tags=[GraalPythonTags.license]) as task:
if task:
python_checkcopyrights([])
with Task('GraalPython GraalVM shared-library build',
tasks,
tags=[GraalPythonTags.downstream,
GraalPythonTags.graalvm]) as task:
if task:
run_shared_lib_test()
with Task('GraalPython GraalVM build',
tasks,
tags=[GraalPythonTags.downstream,
GraalPythonTags.graalvm]) as task:
if task:
svm_image = python_svm(["--version"])
benchmark = os.path.join("graalpython", "benchmarks", "src",
"benchmarks", "image_magix.py")
out = mx.OutputCapture()
mx.run([svm_image, benchmark],
nonZeroIsFatal=True,
out=mx.TeeOutputCapture(out))
success = "\n".join([
"[0, 0, 0, 0, 0, 0, 20, 20, 20, 0, 0, 20, 20, 20, 0, 0, 20, 20, 20, 0, 0, 0, 0, 0, 0]",
"[11, 12, 13, 14, 15, 21, 22, 23, 24, 25, 31, 32, 33, 34, 35, 41, 42, 43, 44, 45, 51, 52, 53, 54, 55]",
"[11, 12, 13, 14, 15, 21, 22, 23, 24, 25, 31, 32, 36, 36, 35, 41, 41, 40, 40, 45, 51, 52, 53, 54, 55]"
])
if success not in out.data:
mx.abort('Output from generated SVM image "' + svm_image +
'" did not match success pattern:\n' + success)
for name, iterations in sorted(python_test_benchmarks.iteritems()):
with Task('PythonBenchmarksTest:' + name,
tasks,
tags=[GraalPythonTags.benchmarks]) as task:
if task:
_gate_python_benchmarks_tests(
"graalpython/benchmarks/src/benchmarks/" + name + ".py",
iterations)