Пример #1
0
def _vc_tip(args):
    parser = ArgumentParser(prog='mx mxt-vc-tip')
    parser.add_argument('--dir', action='store', help='repo url', default=os.getcwd())
    parser.add_argument('--kind', action='store', help='vc kind (hg, git, binary)', default='hg')
    args = parser.parse_args(args)
    vc = mx.vc_system(args.kind)
    rc = vc.tip(args.dir)
    print rc
Пример #2
0
def _vc_pull(args):
    parser = ArgumentParser(prog='mx mxt-vc-pull')
    parser.add_argument('--no-update', action='store_true', help='do not update')
    parser.add_argument('--dir', action='store', help='repo url', default=os.getcwd())
    parser.add_argument('--kind', action='store', help='vc kind (hg, git)', default='hg')
    args = parser.parse_args(args)
    vc = mx.vc_system(args.kind)
    vc.pull(args.dir, update=not args.no_update)
Пример #3
0
def _vc_locate(args):
    parser = ArgumentParser(prog='mx mxt-vc-locate')
    parser.add_argument('--dir', action='store', help='repo url', default=os.getcwd())
    parser.add_argument('--vind', action='store', help='vc kind (hg, git)', default='hg')
    parser.add_argument('--patterns', action='store', help='patterns)')
    args = parser.parse_args(args)
    vc = mx.vc_system(args.kind)
    lines = vc.locate(args.dir, patterns=args.patterns)
    for line in lines:
        print line
Пример #4
0
def _vc_clone(args):
    parser = ArgumentParser(prog='mx mxt-vc-clone')
    parser.add_argument('--url', action='store', help='repo url', required=True)
    parser.add_argument('--target', action='store', help='target dir')
    parser.add_argument('--rev', action='store', help='revision')
    parser.add_argument('--kind', action='store', help='vc kind (hg, git)', default='hg')
    parser.add_argument('--log', action='store', help='log command output', default='True')
    args = parser.parse_args(args)
    vc = mx.vc_system(args.kind)
    rc = vc.clone(args.url, args.target, args.rev, args.log == 'True')
    print rc
Пример #5
0
def load_optional_suite(name, rev, kind='hg', build=True, url=None):
    if not url:
        hg_base = mx.get_env('MX_' + kind.upper() + '_BASE')
        if hg_base is None:
            url = None
        else:
            url = join(hg_base, name)
    urlinfos = None if url is None else [mx.SuiteImportURLInfo(url, kind, mx.vc_system(kind))]
    opt_suite = _fastr_suite.import_suite(name, version=rev, urlinfos=urlinfos)
    if opt_suite and build:
        mx.build_suite(opt_suite)
    return opt_suite
Пример #6
0
def load_optional_suite(name, rev):
    hg_base = mx.get_env('MX_HG_BASE')
    urlinfos = None if hg_base is None else [mx.SuiteImportURLInfo(join(hg_base, name), 'hg', mx.vc_system('hg'))]
    opt_suite = _fastr_suite.import_suite(name, version=rev, urlinfos=urlinfos)
    if opt_suite:
        mx.build_suite(opt_suite)
    return opt_suite
Пример #7
0
def truffle_language_ensure(language_flag, version=None, native_image_root=None, early_exit=False, extract=True, debug_gr_8964=False):
    """
    Ensures that we have a valid suite for the given language_flag, by downloading a binary if necessary
    and providing the suite distribution artifacts in the native-image directory hierachy (via symlinks).
    :param language_flag: native-image language_flag whose truffle-language we want to use
    :param version: if not specified and no TRUFFLE_<LANG>_VERSION set latest binary deployed master revision gets downloaded
    :param native_image_root: the native_image_root directory where the the artifacts get installed to
    :return: language suite for the given language_flag
    """
    if not native_image_root:
        native_image_root = suite_native_image_root()

    version_env_var = 'TRUFFLE_' + language_flag.upper() + '_VERSION'
    if not version and os.environ.has_key(version_env_var):
        version = os.environ[version_env_var]

    if language_flag not in flag_suitename_map:
        mx.abort('No truffle-language uses language_flag \'' + language_flag + '\'')

    language_dir = join('languages', language_flag)
    if early_exit and exists(join(native_image_root, language_dir)):
        mx.logv('Early exit mode: Language subdir \'' + language_flag + '\' exists. Skip suite.import_suite.')
        return None

    language_entry = flag_suitename_map[language_flag]

    language_suite_name = language_entry[0]
    language_repo_name = language_entry[3] if len(language_entry) > 3 else None

    urlinfos = [
        mx.SuiteImportURLInfo(mx_urlrewrites.rewriteurl('https://curio.ssw.jku.at/nexus/content/repositories/snapshots'),
                              'binary',
                              mx.vc_system('binary'))
    ]

    failure_warning = None
    if not version and not mx.suite(language_suite_name, fatalIfMissing=False):
        # If no specific version requested use binary import of last recently deployed master version
        repo_suite_name = language_repo_name if language_repo_name else language_suite_name
        repo_url = mx_urlrewrites.rewriteurl('https://github.com/graalvm/{0}.git'.format(repo_suite_name))
        version = mx.SuiteImport.resolve_git_branchref(repo_url, 'binary', abortOnError=False)
        if not version:
            failure_warning = 'Resolving \'binary\' against ' + repo_url + ' failed'

    language_suite = suite.import_suite(
        language_suite_name,
        version=version,
        urlinfos=urlinfos,
        kind=None,
        in_subdir=bool(language_repo_name)
    )

    if not language_suite:
        if failure_warning:
            mx.warn(failure_warning)
        mx.abort('Binary suite not found and no local copy of ' + language_suite_name + ' available.')

    if not extract:
        if not exists(join(native_image_root, language_dir)):
            mx.abort('Language subdir \'' + language_flag + '\' should already exist with extract=False')
        return language_suite

    language_suite_depnames = language_entry[1]
    language_deps = language_suite.dists + language_suite.libs
    language_deps = [dep for dep in language_deps if dep.name in language_suite_depnames]
    native_image_layout(language_deps, language_dir, native_image_root, debug_gr_8964=debug_gr_8964)

    language_suite_nativedistnames = language_entry[2]
    language_nativedists = [dist for dist in language_suite.dists if dist.name in language_suite_nativedistnames]
    native_image_extract(language_nativedists, language_dir, native_image_root)

    option_properties = join(language_suite.mxDir, 'native-image.properties')
    target_path = remove_existing_symlink(join(native_image_root, language_dir, 'native-image.properties'))
    if exists(option_properties):
        if not exists(target_path):
            mx.logv('Add symlink to ' + str(option_properties))
            symlink_or_copy(option_properties, target_path, debug_gr_8964=debug_gr_8964)
    else:
        native_image_option_properties('languages', language_flag, native_image_root)
    return language_suite
Пример #8
0
def truffle_language_ensure(language_flag,
                            version=None,
                            native_image_root=None,
                            early_exit=False,
                            extract=True):
    """
    Ensures that we have a valid suite for the given language_flag, by downloading a binary if necessary
    and providing the suite distribution artifacts in the native-image directory hierachy (via symlinks).
    :param language_flag: native-image language_flag whose truffle-language we want to use
    :param version: if not specified and no TRUFFLE_<LANG>_VERSION set latest binary deployed master revision gets downloaded
    :param native_image_root: the native_image_root directory where the the artifacts get installed to
    :return: language suite for the given language_flag
    """
    if not native_image_root:
        native_image_root = suite_native_image_root()

    version_env_var = 'TRUFFLE_' + language_flag.upper() + '_VERSION'
    if not version and os.environ.has_key(version_env_var):
        version = os.environ[version_env_var]

    if language_flag not in flag_suitename_map:
        mx.abort('No truffle-language uses language_flag \'' + language_flag +
                 '\'')

    language_dir = join('languages', language_flag)
    if early_exit and exists(join(native_image_root, language_dir)):
        mx.logv('Early exit mode: Language subdir \'' + language_flag +
                '\' exists. Skip suite.import_suite.')
        return None

    language_entry = flag_suitename_map[language_flag]

    language_suite_name = language_entry[0]
    language_repo_name = language_entry[3] if len(language_entry) > 3 else None

    urlinfos = [
        mx.SuiteImportURLInfo(
            mx_urlrewrites.rewriteurl(
                'https://curio.ssw.jku.at/nexus/content/repositories/snapshots'
            ), 'binary', mx.vc_system('binary'))
    ]

    failure_warning = None
    if not version and not mx.suite(language_suite_name, fatalIfMissing=False):
        # If no specific version requested use binary import of last recently deployed master version
        repo_suite_name = language_repo_name if language_repo_name else language_suite_name
        repo_url = mx_urlrewrites.rewriteurl(
            'https://github.com/graalvm/{0}.git'.format(repo_suite_name))
        version = mx.SuiteImport.resolve_git_branchref(repo_url,
                                                       'binary',
                                                       abortOnError=False)
        if not version:
            failure_warning = 'Resolving \'binary\' against ' + repo_url + ' failed'

    language_suite = suite.import_suite(language_suite_name,
                                        version=version,
                                        urlinfos=urlinfos,
                                        kind=None,
                                        in_subdir=bool(language_repo_name))

    if not language_suite:
        if failure_warning:
            mx.warn(failure_warning)
        mx.abort('Binary suite not found and no local copy of ' +
                 language_suite_name + ' available.')

    if not extract:
        if not exists(join(native_image_root, language_dir)):
            mx.abort('Language subdir \'' + language_flag +
                     '\' should already exist with extract=False')
        return language_suite

    language_suite_depnames = language_entry[1]
    language_deps = language_suite.dists + language_suite.libs
    language_deps = [
        dep for dep in language_deps if dep.name in language_suite_depnames
    ]
    native_image_layout(language_deps, language_dir, native_image_root)

    language_suite_nativedistnames = language_entry[2]
    language_nativedists = [
        dist for dist in language_suite.dists
        if dist.name in language_suite_nativedistnames
    ]
    native_image_extract(language_nativedists, language_dir, native_image_root)

    option_properties = join(language_suite.mxDir, 'native-image.properties')
    target_path = remove_existing_symlink(
        join(native_image_root, language_dir, 'native-image.properties'))
    if exists(option_properties):
        if not exists(target_path):
            mx.logv('Add symlink to ' + str(option_properties))
            symlink_or_copy(option_properties, target_path)
    else:
        native_image_option_properties('languages', language_flag,
                                       native_image_root)
    return language_suite
Пример #9
0
def graalpython_gate_runner(args, tasks):
    _graalpytest_driver = "graalpython/com.oracle.graal.python.test/src/graalpytest.py"
    _test_project = "graalpython/com.oracle.graal.python.test/"
    with Task('GraalPython JUnit', tasks,
              tags=[GraalPythonTags.junit]) as task:
        if task:
            punittest(['--verbose'])

    with Task('GraalPython Python tests',
              tasks,
              tags=[GraalPythonTags.unittest]) as task:
        if task:
            test_args = [
                _graalpytest_driver, "-v", _test_project + "src/tests/"
            ]
            mx.command_function(
                "python")(["--python.CatchAllExceptions=true"] + test_args)
            if platform.system() != 'Darwin':
                # TODO: re-enable when python3 is available on darwin
                mx.log("Running tests with CPython")
                mx.run(["python3"] + test_args, nonZeroIsFatal=True)

    with Task('GraalPython C extension tests',
              tasks,
              tags=[GraalPythonTags.cpyext]) as task:
        if task:
            test_args = [
                _graalpytest_driver, "-v", _test_project + "src/tests/cpyext/"
            ]
            mx.command_function("python")(test_args)
            if platform.system() != 'Darwin':
                # TODO: re-enable when python3 is available on darwin
                mx.log("Running tests with CPython")
                mx.run(["python3"] + test_args, nonZeroIsFatal=True)

    with Task('GraalPython Python tests on SVM',
              tasks,
              tags=[GraalPythonTags.svmunit]) as task:
        if task:
            if not os.path.exists("./graalpython-svm"):
                python_svm(["-h"])
            if os.path.exists("./graalpython-svm"):
                langhome = mx_subst.path_substitutions.substitute(
                    '--native.Dllvm.home=<path:SULONG_LIBS>')

                # tests root directory
                tests_folder = "graalpython/com.oracle.graal.python.test/src/tests/"

                # list of excluded tests
                excluded = ["test_interop.py"]

                def is_included(path):
                    if path.endswith(".py"):
                        basename = path.rpartition("/")[2]
                        return basename.startswith(
                            "test_") and basename not in excluded
                    return False

                # list all 1st-level tests and exclude the SVM-incompatible ones
                testfiles = []
                paths = [tests_folder]
                while paths:
                    path = paths.pop()
                    if is_included(path):
                        testfiles.append(path)
                    else:
                        try:
                            paths += [
                                (path + f if path.endswith("/") else "%s/%s" %
                                 (path, f)) for f in os.listdir(path)
                            ]
                        except OSError:
                            pass

                test_args = [
                    "graalpython/com.oracle.graal.python.test/src/graalpytest.py",
                    "-v"
                ] + testfiles
                mx.run([
                    "./graalpython-svm",
                    "--python.CoreHome=graalpython/lib-graalpython",
                    "--python.StdLibHome=graalpython/lib-python/3", langhome
                ] + test_args,
                       nonZeroIsFatal=True)

    with Task('GraalPython downstream R tests',
              tasks,
              tags=[GraalPythonTags.downstream, GraalPythonTags.R]) as task:
        script_r2p = os.path.join(_suite.dir, "graalpython", "benchmarks",
                                  "src", "benchmarks", "interop",
                                  "r_python_image_demo.r")
        script_p2r = os.path.join(_suite.dir, "graalpython", "benchmarks",
                                  "src", "benchmarks", "interop",
                                  "python_r_image_demo.py")
        pythonjars = os.pathsep.join([
            os.path.join(_suite.dir, "mxbuild", "dists", "graalpython.jar"),
            os.path.join(_suite.dir, "mxbuild", "dists", "graalpython-env.jar")
        ])
        if task:
            rrepo = os.environ["FASTR_REPO_URL"]
            testdownstream(_suite, [
                rrepo,
                mx.suite("truffle").vc._remote_url(
                    mx.suite("truffle").dir, "origin")
            ], ".", [[
                "--dynamicimports", "graalpython",
                "--version-conflict-resolution", "latest_all", "build",
                "--force-deprecation-as-warning"
            ],
                     [
                         "--cp-sfx", pythonjars, "r", "--polyglot",
                         "--file=%s" % script_r2p
                     ]])
            testdownstream(_suite, [
                rrepo,
                mx.suite("truffle").vc._remote_url(
                    mx.suite("truffle").dir, "origin")
            ], ".", [[
                "--dynamicimports", "graalpython",
                "--version-conflict-resolution", "latest_all", "build",
                "--force-deprecation-as-warning"
            ],
                     [
                         "-v", "--cp-sfx", pythonjars, "r", "--jvm",
                         "--polyglot", "-e",
                         "eval.polyglot('python', path='%s')" % str(script_p2r)
                     ]])

    with Task('GraalPython apptests', tasks,
              tags=[GraalPythonTags.apptests]) as task:
        if task:
            apprepo = os.environ["GRAALPYTHON_APPTESTS_REPO_URL"]
            _apptest_suite = _suite.import_suite(
                "graalpython-apptests",
                urlinfos=[
                    mx.SuiteImportURLInfo(mx_urlrewrites.rewriteurl(apprepo),
                                          "git", mx.vc_system("git"))
                ])
            mx.run_mx(["-p", _apptest_suite.dir, "graalpython-apptests"])

    with Task('GraalPython license header update',
              tasks,
              tags=[GraalPythonTags.license]) as task:
        if task:
            python_checkcopyrights([])

    with Task('GraalPython GraalVM shared-library build',
              tasks,
              tags=[GraalPythonTags.downstream,
                    GraalPythonTags.graalvm]) as task:
        if task:
            run_shared_lib_test()

    with Task('GraalPython GraalVM build',
              tasks,
              tags=[GraalPythonTags.downstream,
                    GraalPythonTags.graalvm]) as task:
        if task:
            svm_image = python_svm(["--version"])
            benchmark = os.path.join("graalpython", "benchmarks", "src",
                                     "benchmarks", "image_magix.py")
            out = mx.OutputCapture()
            mx.run([svm_image, benchmark],
                   nonZeroIsFatal=True,
                   out=mx.TeeOutputCapture(out))
            success = "\n".join([
                "[0, 0, 0, 0, 0, 0, 20, 20, 20, 0, 0, 20, 20, 20, 0, 0, 20, 20, 20, 0, 0, 0, 0, 0, 0]",
                "[11, 12, 13, 14, 15, 21, 22, 23, 24, 25, 31, 32, 33, 34, 35, 41, 42, 43, 44, 45, 51, 52, 53, 54, 55]",
                "[11, 12, 13, 14, 15, 21, 22, 23, 24, 25, 31, 32, 36, 36, 35, 41, 41, 40, 40, 45, 51, 52, 53, 54, 55]"
            ])
            if success not in out.data:
                mx.abort('Output from generated SVM image "' + svm_image +
                         '" did not match success pattern:\n' + success)

    for name, iterations in sorted(python_test_benchmarks.iteritems()):
        with Task('PythonBenchmarksTest:' + name,
                  tasks,
                  tags=[GraalPythonTags.benchmarks]) as task:
            if task:
                _gate_python_benchmarks_tests(
                    "graalpython/benchmarks/src/benchmarks/" + name + ".py",
                    iterations)
Пример #10
0
def truffle_language_ensure(language_flag, version=None, native_image_root=None):
    """
    Ensures that we have a valid suite for the given language_flag, by downloading a binary if necessary
    and providing the suite distribution artifacts in the native-image directory hierachy (via symlinks).
    :param language_flag: native-image language_flag whose truffle-language we want to use
    :param version: if not specified and no TRUFFLE_<LANG>_VERSION set latest binary deployed master revision gets downloaded
    :param native_image_root: the native_image_root directory where the the artifacts get installed to
    :return: language suite for the given language_flag
    """
    if not native_image_root:
        native_image_root = suite_native_image_root()

    version_env_var = 'TRUFFLE_' + language_flag.upper() + '_VERSION'
    if not version and os.environ.has_key(version_env_var):
        version = os.environ[version_env_var]

    if language_flag not in flag_suitename_map:
        mx.abort('No truffle-language uses language_flag \'' + language_flag + '\'')

    language_entry = flag_suitename_map[language_flag]

    language_suite_name = language_entry[0]
    language_repo_name = language_entry[3] if len(language_entry) > 3 else None

    urlinfos = [
        mx.SuiteImportURLInfo(mx_urlrewrites.rewriteurl('https://curio.ssw.jku.at/nexus/content/repositories/snapshots'),
                              'binary',
                              mx.vc_system('binary'))
    ]

    if not version:
        # If no specific version requested use binary import of last recently deployed master version
        version = 'git-bref:binary'
        repo_suite_name = language_repo_name if language_repo_name else language_suite_name
        urlinfos.append(
            mx.SuiteImportURLInfo(
                mx_urlrewrites.rewriteurl('https://github.com/graalvm/{0}.git'.format(repo_suite_name)),
                'source',
                mx.vc_system('git')
            )
        )

    language_suite = suite.import_suite(
        language_suite_name,
        version=version,
        urlinfos=urlinfos,
        kind=None,
        in_subdir=bool(language_repo_name)
    )

    if not language_suite:
        mx.abort('Binary suite not found and no local copy of ' + language_suite_name + ' available.')

    language_dir = join('languages', language_flag)

    language_suite_depnames = language_entry[1]
    language_deps = [dep for dep in language_suite.dists + language_suite.libs if dep.name in language_suite_depnames]
    native_image_layout(language_deps, language_dir, native_image_root)

    language_suite_nativedistnames = language_entry[2]
    language_nativedists = [dist for dist in language_suite.dists if dist.name in language_suite_nativedistnames]
    native_image_extract(language_nativedists, language_dir, native_image_root)

    option_properties = join(language_suite.mxDir, 'native-image.properties')
    target_path = join(native_image_root, language_dir, 'native-image.properties')
    if islink(target_path):
        os.remove(target_path)
    if exists(option_properties):
        mx.logv('Add symlink to ' + str(option_properties))
        relsymlink(option_properties, target_path)
    else:
        native_image_option_properties('languages', language_flag, native_image_root)
    return language_suite
Пример #11
0
def download_binary_suite(args):
    """Download a binary suite at the given revision"""
    if len(args) == 1:
        name, version = args[0], ''
    else:
        name, version = args

    if len(version) == 0:
        version = None
    elif version == 'truffle':
        version = mx.suite('truffle').version()

    # Add to MX_BINARY_SUITES dynamically, make sure to not use a source suite
    if not mx._binary_suites:
        mx._binary_suites = []
    mx._binary_suites.append(name)

    # For Graal's mx_post_parse_cmd_line()
    mx.get_opts().vm_prefix = None
    # Do not check JAVA_HOME within Graal's suite yet
    os.environ['JVMCI_VERSION_CHECK'] = 'ignore'

    suite = _suite.import_suite(name=name, version=version, urlinfos=[
        mx.SuiteImportURLInfo('https://curio.ssw.jku.at/nexus/content/repositories/snapshots', 'binary', mx.vc_system('binary'))
    ], kind=None)