def _vc_tip(args):
parser = ArgumentParser(prog='mx mxt-vc-tip')
parser.add_argument('--dir', action='store', help='repo url', default=os.getcwd())
parser.add_argument('--kind', action='store', help='vc kind (hg, git, binary)', default='hg')
args = parser.parse_args(args)
vc = mx.vc_system(args.kind)
rc = vc.tip(args.dir)
print rc
def _vc_pull(args):
parser = ArgumentParser(prog='mx mxt-vc-pull')
parser.add_argument('--no-update', action='store_true', help='do not update')
parser.add_argument('--dir', action='store', help='repo url', default=os.getcwd())
parser.add_argument('--kind', action='store', help='vc kind (hg, git)', default='hg')
args = parser.parse_args(args)
vc = mx.vc_system(args.kind)
vc.pull(args.dir, update=not args.no_update)
def _vc_locate(args):
parser = ArgumentParser(prog='mx mxt-vc-locate')
parser.add_argument('--dir', action='store', help='repo url', default=os.getcwd())
parser.add_argument('--vind', action='store', help='vc kind (hg, git)', default='hg')
parser.add_argument('--patterns', action='store', help='patterns)')
args = parser.parse_args(args)
vc = mx.vc_system(args.kind)
lines = vc.locate(args.dir, patterns=args.patterns)
for line in lines:
print line
def _vc_clone(args):
parser = ArgumentParser(prog='mx mxt-vc-clone')
parser.add_argument('--url', action='store', help='repo url', required=True)
parser.add_argument('--target', action='store', help='target dir')
parser.add_argument('--rev', action='store', help='revision')
parser.add_argument('--kind', action='store', help='vc kind (hg, git)', default='hg')
parser.add_argument('--log', action='store', help='log command output', default='True')
args = parser.parse_args(args)
vc = mx.vc_system(args.kind)
rc = vc.clone(args.url, args.target, args.rev, args.log == 'True')
print rc
def load_optional_suite(name, rev, kind='hg', build=True, url=None):
if not url:
hg_base = mx.get_env('MX_' + kind.upper() + '_BASE')
if hg_base is None:
url = None
else:
url = join(hg_base, name)
urlinfos = None if url is None else [mx.SuiteImportURLInfo(url, kind, mx.vc_system(kind))]
opt_suite = _fastr_suite.import_suite(name, version=rev, urlinfos=urlinfos)
if opt_suite and build:
mx.build_suite(opt_suite)
return opt_suite
def load_optional_suite(name, rev):
hg_base = mx.get_env('MX_HG_BASE')
urlinfos = None if hg_base is None else [mx.SuiteImportURLInfo(join(hg_base, name), 'hg', mx.vc_system('hg'))]
opt_suite = _fastr_suite.import_suite(name, version=rev, urlinfos=urlinfos)
if opt_suite:
mx.build_suite(opt_suite)
return opt_suite
def truffle_language_ensure(language_flag, version=None, native_image_root=None, early_exit=False, extract=True, debug_gr_8964=False):
"""
Ensures that we have a valid suite for the given language_flag, by downloading a binary if necessary
and providing the suite distribution artifacts in the native-image directory hierachy (via symlinks).
:param language_flag: native-image language_flag whose truffle-language we want to use
:param version: if not specified and no TRUFFLE_<LANG>_VERSION set latest binary deployed master revision gets downloaded
:param native_image_root: the native_image_root directory where the the artifacts get installed to
:return: language suite for the given language_flag
"""
if not native_image_root:
native_image_root = suite_native_image_root()
version_env_var = 'TRUFFLE_' + language_flag.upper() + '_VERSION'
if not version and os.environ.has_key(version_env_var):
version = os.environ[version_env_var]
if language_flag not in flag_suitename_map:
mx.abort('No truffle-language uses language_flag \'' + language_flag + '\'')
language_dir = join('languages', language_flag)
if early_exit and exists(join(native_image_root, language_dir)):
mx.logv('Early exit mode: Language subdir \'' + language_flag + '\' exists. Skip suite.import_suite.')
return None
language_entry = flag_suitename_map[language_flag]
language_suite_name = language_entry[0]
language_repo_name = language_entry[3] if len(language_entry) > 3 else None
urlinfos = [
mx.SuiteImportURLInfo(mx_urlrewrites.rewriteurl('https://curio.ssw.jku.at/nexus/content/repositories/snapshots'),
'binary',
mx.vc_system('binary'))
]
failure_warning = None
if not version and not mx.suite(language_suite_name, fatalIfMissing=False):
# If no specific version requested use binary import of last recently deployed master version
repo_suite_name = language_repo_name if language_repo_name else language_suite_name
repo_url = mx_urlrewrites.rewriteurl('https://github.com/graalvm/{0}.git'.format(repo_suite_name))
version = mx.SuiteImport.resolve_git_branchref(repo_url, 'binary', abortOnError=False)
if not version:
failure_warning = 'Resolving \'binary\' against ' + repo_url + ' failed'
language_suite = suite.import_suite(
language_suite_name,
version=version,
urlinfos=urlinfos,
kind=None,
in_subdir=bool(language_repo_name)
)
if not language_suite:
if failure_warning:
mx.warn(failure_warning)
mx.abort('Binary suite not found and no local copy of ' + language_suite_name + ' available.')
if not extract:
if not exists(join(native_image_root, language_dir)):
mx.abort('Language subdir \'' + language_flag + '\' should already exist with extract=False')
return language_suite
language_suite_depnames = language_entry[1]
language_deps = language_suite.dists + language_suite.libs
language_deps = [dep for dep in language_deps if dep.name in language_suite_depnames]
native_image_layout(language_deps, language_dir, native_image_root, debug_gr_8964=debug_gr_8964)
language_suite_nativedistnames = language_entry[2]
language_nativedists = [dist for dist in language_suite.dists if dist.name in language_suite_nativedistnames]
native_image_extract(language_nativedists, language_dir, native_image_root)
option_properties = join(language_suite.mxDir, 'native-image.properties')
target_path = remove_existing_symlink(join(native_image_root, language_dir, 'native-image.properties'))
if exists(option_properties):
if not exists(target_path):
mx.logv('Add symlink to ' + str(option_properties))
symlink_or_copy(option_properties, target_path, debug_gr_8964=debug_gr_8964)
else:
native_image_option_properties('languages', language_flag, native_image_root)
return language_suite
def truffle_language_ensure(language_flag,
version=None,
native_image_root=None,
early_exit=False,
extract=True):
"""
Ensures that we have a valid suite for the given language_flag, by downloading a binary if necessary
and providing the suite distribution artifacts in the native-image directory hierachy (via symlinks).
:param language_flag: native-image language_flag whose truffle-language we want to use
:param version: if not specified and no TRUFFLE_<LANG>_VERSION set latest binary deployed master revision gets downloaded
:param native_image_root: the native_image_root directory where the the artifacts get installed to
:return: language suite for the given language_flag
"""
if not native_image_root:
native_image_root = suite_native_image_root()
version_env_var = 'TRUFFLE_' + language_flag.upper() + '_VERSION'
if not version and os.environ.has_key(version_env_var):
version = os.environ[version_env_var]
if language_flag not in flag_suitename_map:
mx.abort('No truffle-language uses language_flag \'' + language_flag +
'\'')
language_dir = join('languages', language_flag)
if early_exit and exists(join(native_image_root, language_dir)):
mx.logv('Early exit mode: Language subdir \'' + language_flag +
'\' exists. Skip suite.import_suite.')
return None
language_entry = flag_suitename_map[language_flag]
language_suite_name = language_entry[0]
language_repo_name = language_entry[3] if len(language_entry) > 3 else None
urlinfos = [
mx.SuiteImportURLInfo(
mx_urlrewrites.rewriteurl(
'https://curio.ssw.jku.at/nexus/content/repositories/snapshots'
), 'binary', mx.vc_system('binary'))
]
failure_warning = None
if not version and not mx.suite(language_suite_name, fatalIfMissing=False):
# If no specific version requested use binary import of last recently deployed master version
repo_suite_name = language_repo_name if language_repo_name else language_suite_name
repo_url = mx_urlrewrites.rewriteurl(
'https://github.com/graalvm/{0}.git'.format(repo_suite_name))
version = mx.SuiteImport.resolve_git_branchref(repo_url,
'binary',
abortOnError=False)
if not version:
failure_warning = 'Resolving \'binary\' against ' + repo_url + ' failed'
language_suite = suite.import_suite(language_suite_name,
version=version,
urlinfos=urlinfos,
kind=None,
in_subdir=bool(language_repo_name))
if not language_suite:
if failure_warning:
mx.warn(failure_warning)
mx.abort('Binary suite not found and no local copy of ' +
language_suite_name + ' available.')
if not extract:
if not exists(join(native_image_root, language_dir)):
mx.abort('Language subdir \'' + language_flag +
'\' should already exist with extract=False')
return language_suite
language_suite_depnames = language_entry[1]
language_deps = language_suite.dists + language_suite.libs
language_deps = [
dep for dep in language_deps if dep.name in language_suite_depnames
]
native_image_layout(language_deps, language_dir, native_image_root)
language_suite_nativedistnames = language_entry[2]
language_nativedists = [
dist for dist in language_suite.dists
if dist.name in language_suite_nativedistnames
]
native_image_extract(language_nativedists, language_dir, native_image_root)
option_properties = join(language_suite.mxDir, 'native-image.properties')
target_path = remove_existing_symlink(
join(native_image_root, language_dir, 'native-image.properties'))
if exists(option_properties):
if not exists(target_path):
mx.logv('Add symlink to ' + str(option_properties))
symlink_or_copy(option_properties, target_path)
else:
native_image_option_properties('languages', language_flag,
native_image_root)
return language_suite
def graalpython_gate_runner(args, tasks):
_graalpytest_driver = "graalpython/com.oracle.graal.python.test/src/graalpytest.py"
_test_project = "graalpython/com.oracle.graal.python.test/"
with Task('GraalPython JUnit', tasks,
tags=[GraalPythonTags.junit]) as task:
if task:
punittest(['--verbose'])
with Task('GraalPython Python tests',
tasks,
tags=[GraalPythonTags.unittest]) as task:
if task:
test_args = [
_graalpytest_driver, "-v", _test_project + "src/tests/"
]
mx.command_function(
"python")(["--python.CatchAllExceptions=true"] + test_args)
if platform.system() != 'Darwin':
# TODO: re-enable when python3 is available on darwin
mx.log("Running tests with CPython")
mx.run(["python3"] + test_args, nonZeroIsFatal=True)
with Task('GraalPython C extension tests',
tasks,
tags=[GraalPythonTags.cpyext]) as task:
if task:
test_args = [
_graalpytest_driver, "-v", _test_project + "src/tests/cpyext/"
]
mx.command_function("python")(test_args)
if platform.system() != 'Darwin':
# TODO: re-enable when python3 is available on darwin
mx.log("Running tests with CPython")
mx.run(["python3"] + test_args, nonZeroIsFatal=True)
with Task('GraalPython Python tests on SVM',
tasks,
tags=[GraalPythonTags.svmunit]) as task:
if task:
if not os.path.exists("./graalpython-svm"):
python_svm(["-h"])
if os.path.exists("./graalpython-svm"):
langhome = mx_subst.path_substitutions.substitute(
'--native.Dllvm.home=<path:SULONG_LIBS>')
# tests root directory
tests_folder = "graalpython/com.oracle.graal.python.test/src/tests/"
# list of excluded tests
excluded = ["test_interop.py"]
def is_included(path):
if path.endswith(".py"):
basename = path.rpartition("/")[2]
return basename.startswith(
"test_") and basename not in excluded
return False
# list all 1st-level tests and exclude the SVM-incompatible ones
testfiles = []
paths = [tests_folder]
while paths:
path = paths.pop()
if is_included(path):
testfiles.append(path)
else:
try:
paths += [
(path + f if path.endswith("/") else "%s/%s" %
(path, f)) for f in os.listdir(path)
]
except OSError:
pass
test_args = [
"graalpython/com.oracle.graal.python.test/src/graalpytest.py",
"-v"
] + testfiles
mx.run([
"./graalpython-svm",
"--python.CoreHome=graalpython/lib-graalpython",
"--python.StdLibHome=graalpython/lib-python/3", langhome
] + test_args,
nonZeroIsFatal=True)
with Task('GraalPython downstream R tests',
tasks,
tags=[GraalPythonTags.downstream, GraalPythonTags.R]) as task:
script_r2p = os.path.join(_suite.dir, "graalpython", "benchmarks",
"src", "benchmarks", "interop",
"r_python_image_demo.r")
script_p2r = os.path.join(_suite.dir, "graalpython", "benchmarks",
"src", "benchmarks", "interop",
"python_r_image_demo.py")
pythonjars = os.pathsep.join([
os.path.join(_suite.dir, "mxbuild", "dists", "graalpython.jar"),
os.path.join(_suite.dir, "mxbuild", "dists", "graalpython-env.jar")
])
if task:
rrepo = os.environ["FASTR_REPO_URL"]
testdownstream(_suite, [
rrepo,
mx.suite("truffle").vc._remote_url(
mx.suite("truffle").dir, "origin")
], ".", [[
"--dynamicimports", "graalpython",
"--version-conflict-resolution", "latest_all", "build",
"--force-deprecation-as-warning"
],
[
"--cp-sfx", pythonjars, "r", "--polyglot",
"--file=%s" % script_r2p
]])
testdownstream(_suite, [
rrepo,
mx.suite("truffle").vc._remote_url(
mx.suite("truffle").dir, "origin")
], ".", [[
"--dynamicimports", "graalpython",
"--version-conflict-resolution", "latest_all", "build",
"--force-deprecation-as-warning"
],
[
"-v", "--cp-sfx", pythonjars, "r", "--jvm",
"--polyglot", "-e",
"eval.polyglot('python', path='%s')" % str(script_p2r)
]])
with Task('GraalPython apptests', tasks,
tags=[GraalPythonTags.apptests]) as task:
if task:
apprepo = os.environ["GRAALPYTHON_APPTESTS_REPO_URL"]
_apptest_suite = _suite.import_suite(
"graalpython-apptests",
urlinfos=[
mx.SuiteImportURLInfo(mx_urlrewrites.rewriteurl(apprepo),
"git", mx.vc_system("git"))
])
mx.run_mx(["-p", _apptest_suite.dir, "graalpython-apptests"])
with Task('GraalPython license header update',
tasks,
tags=[GraalPythonTags.license]) as task:
if task:
python_checkcopyrights([])
with Task('GraalPython GraalVM shared-library build',
tasks,
tags=[GraalPythonTags.downstream,
GraalPythonTags.graalvm]) as task:
if task:
run_shared_lib_test()
with Task('GraalPython GraalVM build',
tasks,
tags=[GraalPythonTags.downstream,
GraalPythonTags.graalvm]) as task:
if task:
svm_image = python_svm(["--version"])
benchmark = os.path.join("graalpython", "benchmarks", "src",
"benchmarks", "image_magix.py")
out = mx.OutputCapture()
mx.run([svm_image, benchmark],
nonZeroIsFatal=True,
out=mx.TeeOutputCapture(out))
success = "\n".join([
"[0, 0, 0, 0, 0, 0, 20, 20, 20, 0, 0, 20, 20, 20, 0, 0, 20, 20, 20, 0, 0, 0, 0, 0, 0]",
"[11, 12, 13, 14, 15, 21, 22, 23, 24, 25, 31, 32, 33, 34, 35, 41, 42, 43, 44, 45, 51, 52, 53, 54, 55]",
"[11, 12, 13, 14, 15, 21, 22, 23, 24, 25, 31, 32, 36, 36, 35, 41, 41, 40, 40, 45, 51, 52, 53, 54, 55]"
])
if success not in out.data:
mx.abort('Output from generated SVM image "' + svm_image +
'" did not match success pattern:\n' + success)
for name, iterations in sorted(python_test_benchmarks.iteritems()):
with Task('PythonBenchmarksTest:' + name,
tasks,
tags=[GraalPythonTags.benchmarks]) as task:
if task:
_gate_python_benchmarks_tests(
"graalpython/benchmarks/src/benchmarks/" + name + ".py",
iterations)
def truffle_language_ensure(language_flag, version=None, native_image_root=None):
"""
Ensures that we have a valid suite for the given language_flag, by downloading a binary if necessary
and providing the suite distribution artifacts in the native-image directory hierachy (via symlinks).
:param language_flag: native-image language_flag whose truffle-language we want to use
:param version: if not specified and no TRUFFLE_<LANG>_VERSION set latest binary deployed master revision gets downloaded
:param native_image_root: the native_image_root directory where the the artifacts get installed to
:return: language suite for the given language_flag
"""
if not native_image_root:
native_image_root = suite_native_image_root()
version_env_var = 'TRUFFLE_' + language_flag.upper() + '_VERSION'
if not version and os.environ.has_key(version_env_var):
version = os.environ[version_env_var]
if language_flag not in flag_suitename_map:
mx.abort('No truffle-language uses language_flag \'' + language_flag + '\'')
language_entry = flag_suitename_map[language_flag]
language_suite_name = language_entry[0]
language_repo_name = language_entry[3] if len(language_entry) > 3 else None
urlinfos = [
mx.SuiteImportURLInfo(mx_urlrewrites.rewriteurl('https://curio.ssw.jku.at/nexus/content/repositories/snapshots'),
'binary',
mx.vc_system('binary'))
]
if not version:
# If no specific version requested use binary import of last recently deployed master version
version = 'git-bref:binary'
repo_suite_name = language_repo_name if language_repo_name else language_suite_name
urlinfos.append(
mx.SuiteImportURLInfo(
mx_urlrewrites.rewriteurl('https://github.com/graalvm/{0}.git'.format(repo_suite_name)),
'source',
mx.vc_system('git')
)
)
language_suite = suite.import_suite(
language_suite_name,
version=version,
urlinfos=urlinfos,
kind=None,
in_subdir=bool(language_repo_name)
)
if not language_suite:
mx.abort('Binary suite not found and no local copy of ' + language_suite_name + ' available.')
language_dir = join('languages', language_flag)
language_suite_depnames = language_entry[1]
language_deps = [dep for dep in language_suite.dists + language_suite.libs if dep.name in language_suite_depnames]
native_image_layout(language_deps, language_dir, native_image_root)
language_suite_nativedistnames = language_entry[2]
language_nativedists = [dist for dist in language_suite.dists if dist.name in language_suite_nativedistnames]
native_image_extract(language_nativedists, language_dir, native_image_root)
option_properties = join(language_suite.mxDir, 'native-image.properties')
target_path = join(native_image_root, language_dir, 'native-image.properties')
if islink(target_path):
os.remove(target_path)
if exists(option_properties):
mx.logv('Add symlink to ' + str(option_properties))
relsymlink(option_properties, target_path)
else:
native_image_option_properties('languages', language_flag, native_image_root)
return language_suite
def download_binary_suite(args):
"""Download a binary suite at the given revision"""
if len(args) == 1:
name, version = args[0], ''
else:
name, version = args
if len(version) == 0:
version = None
elif version == 'truffle':
version = mx.suite('truffle').version()
# Add to MX_BINARY_SUITES dynamically, make sure to not use a source suite
if not mx._binary_suites:
mx._binary_suites = []
mx._binary_suites.append(name)
# For Graal's mx_post_parse_cmd_line()
mx.get_opts().vm_prefix = None
# Do not check JAVA_HOME within Graal's suite yet
os.environ['JVMCI_VERSION_CHECK'] = 'ignore'
suite = _suite.import_suite(name=name, version=version, urlinfos=[
mx.SuiteImportURLInfo('https://curio.ssw.jku.at/nexus/content/repositories/snapshots', 'binary', mx.vc_system('binary'))
], kind=None)