def graalpython_gate_runner(args, tasks):
    with Task('GraalPython JUnit', tasks, tags=[GraalPythonTags.junit]) as task:
        if task:
            punittest(['--verbose'])

    with Task('GraalPython Python tests', tasks, tags=[GraalPythonTags.unittest]) as task:
        if task:
            test_args = ["graalpython/com.oracle.graal.python.test/src/graalpytest.py", "-v",
                         "graalpython/com.oracle.graal.python.test/src/tests/"]
            mx.command_function("python")(test_args)
            if platform.system() != 'Darwin':
                # TODO: re-enable when python3 is available on darwin
                mx.run(["python3"] + test_args, nonZeroIsFatal=True)

    with Task('GraalPython downstream R tests', tasks, tags=[GraalPythonTags.downstream, GraalPythonTags.R]) as task:
        script_r2p = os.path.join(_suite.dir, "graalpython", "benchmarks", "src", "benchmarks", "interop", "r_python_image_demo.r")
        script_p2r = os.path.join(_suite.dir, "graalpython", "benchmarks", "src", "benchmarks", "interop", "python_r_image_demo.py")
        pythonjars = os.pathsep.join([
            os.path.join(_suite.dir, "mxbuild", "dists", "graalpython.jar"),
            os.path.join(_suite.dir, "mxbuild", "dists", "graalpython-env.jar")
        ])
        if task:
            rrepo = os.environ["FASTR_REPO_URL"]
            testdownstream(
                _suite,
                [rrepo, mx.suite("truffle").vc._remote_url(mx.suite("truffle").dir, "origin")],
                ".",
                [["--dynamicimports", "graalpython", "--version-conflict-resolution", "latest_all", "build", "--force-deprecation-as-warning"],
                 ["--cp-sfx", pythonjars, "r", "--polyglot", "--file=%s" % script_r2p]
                 ])
            testdownstream(
                _suite,
                [rrepo, mx.suite("truffle").vc._remote_url(mx.suite("truffle").dir, "origin")],
                ".",
                [["--dynamicimports", "graalpython", "--version-conflict-resolution", "latest_all", "build", "--force-deprecation-as-warning"],
                 ["-v", "--cp-sfx", pythonjars, "r", "--jvm", "--polyglot", "-e", "eval.polyglot('python', path='%s')" % str(script_p2r)]
                 ])

    with Task('GraalPython license header update', tasks, tags=[GraalPythonTags.license]) as task:
        if task:
            python_license_headers_update([])

    with Task('GraalPython downstream svm binary tests', tasks, tags=[GraalPythonTags.downstream, GraalPythonTags.svmbinary]) as task:
        if task:
            _run_downstream_svm(
                [["--dynamicimports", "graalpython", "delete-graalpython-if-testdownstream"],
                 ["gate", '-B--force-deprecation-as-warning', "--tags", "build,python"]],
                binary=True
            )

    with Task('GraalPython downstream svm source tests', tasks, tags=[GraalPythonTags.downstream, GraalPythonTags.svmsource]) as task:
        if task:
            _run_downstream_svm([[
                "--dynamicimports", "graalpython", "--strict-compliance", "gate", '-B--force-deprecation-as-warning', "--strict-mode", "--tags", "build,python"
            ]])

    for name, iterations in sorted(python_test_benchmarks.iteritems()):
        with Task('PythonBenchmarksTest:' + name, tasks, tags=[GraalPythonTags.benchmarks]) as task:
            if task:
                _gate_python_benchmarks_tests("graalpython/benchmarks/src/benchmarks/" + name + ".py", iterations)
Exemple #2
0
def _run_downstream_svm(commands, binary=False):
    new_rewrites = None
    if binary:
        localmvn = "/tmp/graalpythonsnapshots"
        localmvnrepl = "file://%s" % localmvn
        publicmvn = mx.repository("python-public-snapshots").url
        publicmvnpattern = re.compile(publicmvn)
        git = mx.GitConfig()

        new_rewrites = [{
            publicmvnpattern.pattern: {
                "replacement": localmvnrepl
            }
        }]
        for rewrite in _urlrewrites:
            if rewrite.pattern.match(publicmvn):
                # we replace rewrites of our public repo
                pass
            elif publicmvnpattern.match(rewrite.replacement):
                # we rewrite to what we want
                new_rewrites.append(
                    {rewrite.pattern.pattern: {
                        "replacement": localmvnrepl
                    }})
            else:
                new_rewrites.append({
                    rewrite.pattern.pattern: {
                        "replacement": rewrite.replacement
                    }
                })
        os.environ["TRUFFLE_PYTHON_VERSION"] = git.tip(_suite.dir).strip()
        os.environ["TRUFFLE_SULONG_VERSION"] = git.tip(_sulong.dir).strip()
        prev_urlrewrites = os.environ.get("MX_URLREWRITES")
        os.environ["MX_URLREWRITES"] = json.dumps(new_rewrites)

        mx.command_function("deploy-binary")(
            ["--all-suites", "python-local-snapshots", localmvnrepl])

    try:
        mx.log(str(dict(os.environ)))
        testdownstream(_suite, [
            mx.suite("truffle").vc._remote_url(
                mx.suite("truffle").dir, "origin")
        ], "substratevm", commands)
    finally:
        if binary:
            os.environ.pop("TRUFFLE_PYTHON_VERSION")
            os.environ.pop("TRUFFLE_SULONG_VERSION")
            if prev_urlrewrites:
                os.environ["MX_URLREWRITES"] = prev_urlrewrites
            else:
                os.environ.pop("MX_URLREWRITES")
            shutil.rmtree(localmvn, ignore_errors=True)
Exemple #3
0
def graalpython_gate_runner(args, tasks):
    _graalpytest_driver = "graalpython/com.oracle.graal.python.test/src/graalpytest.py"
    _test_project = "graalpython/com.oracle.graal.python.test/"
    with Task('GraalPython JUnit', tasks,
              tags=[GraalPythonTags.junit]) as task:
        if task:
            punittest(['--verbose'])

    with Task('GraalPython Python tests',
              tasks,
              tags=[GraalPythonTags.unittest]) as task:
        if task:
            test_args = [
                _graalpytest_driver, "-v", _test_project + "src/tests/"
            ]
            mx.command_function(
                "python")(["--python.CatchAllExceptions=true"] + test_args)
            if platform.system() != 'Darwin':
                # TODO: re-enable when python3 is available on darwin
                mx.log("Running tests with CPython")
                mx.run(["python3"] + test_args, nonZeroIsFatal=True)

    with Task('GraalPython C extension tests',
              tasks,
              tags=[GraalPythonTags.cpyext]) as task:
        if task:
            test_args = [
                _graalpytest_driver, "-v", _test_project + "src/tests/cpyext/"
            ]
            mx.command_function("python")(test_args)
            if platform.system() != 'Darwin':
                # TODO: re-enable when python3 is available on darwin
                mx.log("Running tests with CPython")
                mx.run(["python3"] + test_args, nonZeroIsFatal=True)

    with Task('GraalPython Python tests on SVM',
              tasks,
              tags=[GraalPythonTags.svmunit]) as task:
        if task:
            if not os.path.exists("./graalpython-svm"):
                python_svm(["-h"])
            if os.path.exists("./graalpython-svm"):
                langhome = mx_subst.path_substitutions.substitute(
                    '--native.Dllvm.home=<path:SULONG_LIBS>')

                # tests root directory
                tests_folder = "graalpython/com.oracle.graal.python.test/src/tests/"

                # list of excluded tests
                excluded = ["test_interop.py"]

                def is_included(path):
                    if path.endswith(".py"):
                        basename = path.rpartition("/")[2]
                        return basename.startswith(
                            "test_") and basename not in excluded
                    return False

                # list all 1st-level tests and exclude the SVM-incompatible ones
                testfiles = []
                paths = [tests_folder]
                while paths:
                    path = paths.pop()
                    if is_included(path):
                        testfiles.append(path)
                    else:
                        try:
                            paths += [
                                (path + f if path.endswith("/") else "%s/%s" %
                                 (path, f)) for f in os.listdir(path)
                            ]
                        except OSError:
                            pass

                test_args = [
                    "graalpython/com.oracle.graal.python.test/src/graalpytest.py",
                    "-v"
                ] + testfiles
                mx.run([
                    "./graalpython-svm",
                    "--python.CoreHome=graalpython/lib-graalpython",
                    "--python.StdLibHome=graalpython/lib-python/3", langhome
                ] + test_args,
                       nonZeroIsFatal=True)

    with Task('GraalPython downstream R tests',
              tasks,
              tags=[GraalPythonTags.downstream, GraalPythonTags.R]) as task:
        script_r2p = os.path.join(_suite.dir, "graalpython", "benchmarks",
                                  "src", "benchmarks", "interop",
                                  "r_python_image_demo.r")
        script_p2r = os.path.join(_suite.dir, "graalpython", "benchmarks",
                                  "src", "benchmarks", "interop",
                                  "python_r_image_demo.py")
        pythonjars = os.pathsep.join([
            os.path.join(_suite.dir, "mxbuild", "dists", "graalpython.jar"),
            os.path.join(_suite.dir, "mxbuild", "dists", "graalpython-env.jar")
        ])
        if task:
            rrepo = os.environ["FASTR_REPO_URL"]
            testdownstream(_suite, [
                rrepo,
                mx.suite("truffle").vc._remote_url(
                    mx.suite("truffle").dir, "origin")
            ], ".", [[
                "--dynamicimports", "graalpython",
                "--version-conflict-resolution", "latest_all", "build",
                "--force-deprecation-as-warning"
            ],
                     [
                         "--cp-sfx", pythonjars, "r", "--polyglot",
                         "--file=%s" % script_r2p
                     ]])
            testdownstream(_suite, [
                rrepo,
                mx.suite("truffle").vc._remote_url(
                    mx.suite("truffle").dir, "origin")
            ], ".", [[
                "--dynamicimports", "graalpython",
                "--version-conflict-resolution", "latest_all", "build",
                "--force-deprecation-as-warning"
            ],
                     [
                         "-v", "--cp-sfx", pythonjars, "r", "--jvm",
                         "--polyglot", "-e",
                         "eval.polyglot('python', path='%s')" % str(script_p2r)
                     ]])

    with Task('GraalPython apptests', tasks,
              tags=[GraalPythonTags.apptests]) as task:
        if task:
            apprepo = os.environ["GRAALPYTHON_APPTESTS_REPO_URL"]
            _apptest_suite = _suite.import_suite(
                "graalpython-apptests",
                urlinfos=[
                    mx.SuiteImportURLInfo(mx_urlrewrites.rewriteurl(apprepo),
                                          "git", mx.vc_system("git"))
                ])
            mx.run_mx(["-p", _apptest_suite.dir, "graalpython-apptests"])

    with Task('GraalPython license header update',
              tasks,
              tags=[GraalPythonTags.license]) as task:
        if task:
            python_checkcopyrights([])

    with Task('GraalPython GraalVM shared-library build',
              tasks,
              tags=[GraalPythonTags.downstream,
                    GraalPythonTags.graalvm]) as task:
        if task:
            run_shared_lib_test()

    with Task('GraalPython GraalVM build',
              tasks,
              tags=[GraalPythonTags.downstream,
                    GraalPythonTags.graalvm]) as task:
        if task:
            svm_image = python_svm(["--version"])
            benchmark = os.path.join("graalpython", "benchmarks", "src",
                                     "benchmarks", "image_magix.py")
            out = mx.OutputCapture()
            mx.run([svm_image, benchmark],
                   nonZeroIsFatal=True,
                   out=mx.TeeOutputCapture(out))
            success = "\n".join([
                "[0, 0, 0, 0, 0, 0, 20, 20, 20, 0, 0, 20, 20, 20, 0, 0, 20, 20, 20, 0, 0, 0, 0, 0, 0]",
                "[11, 12, 13, 14, 15, 21, 22, 23, 24, 25, 31, 32, 33, 34, 35, 41, 42, 43, 44, 45, 51, 52, 53, 54, 55]",
                "[11, 12, 13, 14, 15, 21, 22, 23, 24, 25, 31, 32, 36, 36, 35, 41, 41, 40, 40, 45, 51, 52, 53, 54, 55]"
            ])
            if success not in out.data:
                mx.abort('Output from generated SVM image "' + svm_image +
                         '" did not match success pattern:\n' + success)

    for name, iterations in sorted(python_test_benchmarks.iteritems()):
        with Task('PythonBenchmarksTest:' + name,
                  tasks,
                  tags=[GraalPythonTags.benchmarks]) as task:
            if task:
                _gate_python_benchmarks_tests(
                    "graalpython/benchmarks/src/benchmarks/" + name + ".py",
                    iterations)
def graalpython_gate_runner(args, tasks):
    with Task('GraalPython JUnit', tasks,
              tags=[GraalPythonTags.junit]) as task:
        if task:
            punittest(['--verbose'])

    with Task('GraalPython Python tests',
              tasks,
              tags=[GraalPythonTags.unittest]) as task:
        if task:
            test_args = [
                "graalpython/com.oracle.graal.python.test/src/graalpytest.py",
                "-v", "graalpython/com.oracle.graal.python.test/src/tests/"
            ]
            mx.command_function("python")(test_args)
            if platform.system() != 'Darwin':
                # TODO: re-enable when python3 is available on darwin
                mx.run(["python3"] + test_args, nonZeroIsFatal=True)

    with Task('GraalPython downstream R tests',
              tasks,
              tags=[GraalPythonTags.downstream, GraalPythonTags.R]) as task:
        script_r2p = os.path.join(_suite.dir, "graalpython", "benchmarks",
                                  "src", "benchmarks", "interop",
                                  "r_python_image_demo.r")
        script_p2r = os.path.join(_suite.dir, "graalpython", "benchmarks",
                                  "src", "benchmarks", "interop",
                                  "python_r_image_demo.py")
        pythonjars = os.pathsep.join([
            os.path.join(_suite.dir, "mxbuild", "dists", "graalpython.jar"),
            os.path.join(_suite.dir, "mxbuild", "dists", "graalpython-env.jar")
        ])
        if task:
            rrepo = os.environ["FASTR_REPO_URL"]
            testdownstream(_suite, [
                rrepo,
                mx.suite("truffle").vc._remote_url(
                    mx.suite("truffle").dir, "origin")
            ], ".", [[
                "--dynamicimports", "graalpython",
                "--version-conflict-resolution", "latest_all", "build",
                "--force-deprecation-as-warning"
            ],
                     [
                         "--cp-sfx", pythonjars, "r", "--polyglot",
                         "--file=%s" % script_r2p
                     ]])
            testdownstream(_suite, [
                rrepo,
                mx.suite("truffle").vc._remote_url(
                    mx.suite("truffle").dir, "origin")
            ], ".", [[
                "--dynamicimports", "graalpython",
                "--version-conflict-resolution", "latest_all", "build",
                "--force-deprecation-as-warning"
            ],
                     [
                         "-v", "--cp-sfx", pythonjars, "r", "--jvm",
                         "--polyglot", "-e",
                         "eval.polyglot('python', path='%s')" % str(script_p2r)
                     ]])

    with Task('GraalPython license header update',
              tasks,
              tags=[GraalPythonTags.license]) as task:
        if task:
            python_checkcopyrights([])

    with Task('GraalPython GraalVM shared-library build',
              tasks,
              tags=[GraalPythonTags.downstream,
                    GraalPythonTags.graalvm]) as task:
        if task:
            run_shared_lib_test()

    with Task('GraalPython GraalVM build',
              tasks,
              tags=[GraalPythonTags.downstream,
                    GraalPythonTags.graalvm]) as task:
        if task:
            svm_image = python_svm(["--version"])
            benchmark = os.path.join("graalpython", "benchmarks", "src",
                                     "benchmarks", "image_magix.py")
            out = mx.OutputCapture()
            mx.run([svm_image, benchmark],
                   nonZeroIsFatal=True,
                   out=mx.TeeOutputCapture(out))
            success = "\n".join([
                "[0, 0, 0, 0, 0, 0, 20, 20, 20, 0, 0, 20, 20, 20, 0, 0, 20, 20, 20, 0, 0, 0, 0, 0, 0]",
                "[11, 12, 13, 14, 15, 21, 22, 23, 24, 25, 31, 32, 33, 34, 35, 41, 42, 43, 44, 45, 51, 52, 53, 54, 55]",
                "[11, 12, 13, 14, 15, 21, 22, 23, 24, 25, 31, 32, 36, 36, 35, 41, 41, 40, 40, 45, 51, 52, 53, 54, 55]"
            ])
            if success not in out.data:
                mx.abort('Output from generated SVM image "' + svm_image +
                         '" did not match success pattern:\n' + success)

    for name, iterations in sorted(python_test_benchmarks.iteritems()):
        with Task('PythonBenchmarksTest:' + name,
                  tasks,
                  tags=[GraalPythonTags.benchmarks]) as task:
            if task:
                _gate_python_benchmarks_tests(
                    "graalpython/benchmarks/src/benchmarks/" + name + ".py",
                    iterations)