def python_svm(args):
mx.run_mx(_SVM_ARGS + ["build"])
out = mx.OutputCapture()
mx.run_mx(_SVM_ARGS + ["graalvm-home"], out=mx.TeeOutputCapture(out))
svm_image = os.path.join(out.data.strip(), "bin", "graalpython")
mx.run([svm_image] + args)
return svm_image
def python_build_svm(args):
mx.run_mx([
"--dynamicimports", "/substratevm,/vm", "build",
"--force-deprecation-as-warning", "--dependencies",
"GRAAL_MANAGEMENT,graalpython.image"
],
nonZeroIsFatal=True)
shutil.copy(__get_svm_binary_from_graalvm(), _get_svm_binary())
def build(self):
if not self.subject.build:
mx.log("...skip build of {}".format(self.subject))
return
mx.log('...perform build of {}'.format(self.subject))
rubyDir = _suite.dir
mavenDir = os.path.join(rubyDir, 'mxbuild', 'mvn')
# HACK: since the maven executable plugin does not configure the
# java executable that is used we unfortunately need to append it to the PATH
javaHome = os.getenv('JAVA_HOME')
if javaHome:
os.environ["PATH"] = os.environ[
"JAVA_HOME"] + '/bin' + os.pathsep + os.environ["PATH"]
mx.logv('Setting PATH to {}'.format(os.environ["PATH"]))
mx.logv('Calling java -version')
mx.run(['java', '-version'])
# Truffle version
truffle = mx.suite('truffle')
truffle_commit = truffle.vc.parent(truffle.dir)
maven_version_arg = '-Dtruffle.version=' + truffle_commit
maven_repo_arg = '-Dmaven.repo.local=' + mavenDir
mx.run_mx(['maven-install', '--repo', mavenDir], suite=truffle)
open(os.path.join(rubyDir, 'VERSION'), 'w').write('graal-vm\n')
# Build jruby-truffle
mx.run_maven(['--version', maven_repo_arg],
nonZeroIsFatal=False,
cwd=rubyDir)
mx.log('Building without tests')
mx.run_maven(['-DskipTests', maven_version_arg, maven_repo_arg],
cwd=rubyDir)
mx.log('Building complete version')
mx.run_maven(
['-Pcomplete', '-DskipTests', maven_version_arg, maven_repo_arg],
cwd=rubyDir)
mx.run([
'zip', '-d',
'maven/jruby-complete/target/jruby-complete-graal-vm.jar',
'META-INF/jruby.home/lib/*'
],
cwd=rubyDir)
mx.run(['bin/jruby', 'bin/gem', 'install', 'bundler', '-v', '1.10.6'],
cwd=rubyDir)
mx.log('...finished build of {}'.format(self.subject))
def _squeak_graalvm_launcher(args):
"""Build and run a GraalVM graalsqueak launcher"""
dy = ['--dynamicimports', '/vm']
mx.run_mx(dy + ['--env', 'ce-graalsqueak', 'build'])
out = mx.OutputCapture()
mx.run_mx(dy + ["graalvm-home"], out=mx.TeeOutputCapture(out))
launcher = os.path.join(out.data.strip(), "bin", "graalsqueak").split("\n")[-1].strip()
mx.log(launcher)
if args:
mx.run([launcher] + args)
return launcher
def benchmark_callback(suite, commit):
suite.vc.update_to_branch(suite.vc_dir, commit)
mx.run_mx(['sforceimports'], suite=suite)
mx.run_mx(['--env', 'ce', 'sforceimports'], suite=get_suite('/vm'))
if args.enterprise and suite.name != 'vm-enterprise':
checkout_args = [
'--dynamicimports', '/vm-enterprise', 'checkout-downstream',
'vm', 'vm-enterprise'
]
if fetched_enterprise[0]:
checkout_args.append('--no-fetch')
mx.run_mx(checkout_args, out=mx.OutputCapture())
# Make sure vm is imported before vm-enterprise
get_suite('/vm')
mx.run_mx(['--env', 'ee', 'sforceimports'],
suite=get_suite('/vm-enterprise'))
fetched_enterprise[0] = True
suite.vc.update_to_branch(suite.vc_dir, commit)
mx.run_mx(['sforceimports'], suite=suite)
debug_str = "debug: graalpython={} graal={}".format(
get_commit(get_suite('graalpython')), get_commit(get_suite('/vm')))
if args.enterprise:
debug_str += " graal-enterprise={}".format(
get_commit(get_suite('/vm-enterprise')))
print(debug_str)
build_command = shlex.split(args.build_command)
if not args.no_clean:
try:
clean_command = build_command[:build_command.
index('build')] + ['clean']
retcode = mx.run(clean_command, nonZeroIsFatal=False)
if retcode:
print("Warning: clean command failed")
except ValueError:
pass
retcode = mx.run(build_command, nonZeroIsFatal=False)
if retcode:
raise RuntimeError(
"Failed to execute the build command for {}".format(commit))
output = mx.OutputCapture()
retcode = mx.run(shlex.split(args.benchmark_command),
out=mx.TeeOutputCapture(output),
nonZeroIsFatal=False)
if retcode:
raise RuntimeError(
"Failed to execute benchmark for {}".format(commit))
match = re.search(
r'{}.*duration: ([\d.]+)'.format(
re.escape(args.benchmark_criterion)), output.data)
if not match:
raise RuntimeError("Failed to get result from the benchmark")
return float(match.group(1))
def python_svm(args):
mx.run_mx([
"--dynamicimports", "/substratevm,/vm", "build",
"--force-deprecation-as-warning", "--dependencies",
"GRAAL_MANAGEMENT,graalpython.image"
],
nonZeroIsFatal=True)
vmdir = os.path.join(mx.suite("truffle").dir, "..", "vm")
svm_image = os.path.join(vmdir, "mxbuild",
"-".join([mx.get_os(), mx.get_arch()]),
"graalpython.image", "graalpython")
shutil.copy(svm_image, os.path.join(_suite.dir, "graalpython-svm"))
mx.run([svm_image] + args)
return svm_image
def _python_graalvm_launcher(args):
dy = "/vm,/tools,/substratevm"
if "sandboxed" in args:
args.remove("sandboxed")
dy += ",/sulong-managed"
dy = ["--dynamicimports", dy]
mx.run_mx(dy + ["build"])
out = mx.OutputCapture()
mx.run_mx(dy + ["graalvm-home"], out=mx.TeeOutputCapture(out))
launcher = os.path.join(out.data.strip(), "bin", "graalpython").split("\n")[-1].strip()
mx.log(launcher)
if args:
mx.run([launcher] + args)
return launcher
def build(self):
if not self.subject.build:
mx.log("...skip build of {}".format(self.subject))
return
mx.log('...perform build of {}'.format(self.subject))
rubyDir = _suite.dir
mavenDir = os.path.join(rubyDir, 'mxbuild', 'mvn')
# HACK: since the maven executable plugin does not configure the
# java executable that is used we unfortunately need to append it to the PATH
javaHome = os.getenv('JAVA_HOME')
if javaHome:
os.environ["PATH"] = os.environ["JAVA_HOME"] + '/bin' + os.pathsep + os.environ["PATH"]
mx.logv('Setting PATH to {}'.format(os.environ["PATH"]))
mx.logv('Calling java -version')
mx.run(['java', '-version'])
# Truffle version
truffle = mx.suite('truffle')
truffle_commit = truffle.vc.parent(truffle.dir)
maven_version_arg = '-Dtruffle.version=' + truffle_commit
maven_repo_arg = '-Dmaven.repo.local=' + mavenDir
mx.run_mx(['maven-install', '--repo', mavenDir, '--only', 'TRUFFLE_API,TRUFFLE_DEBUG,TRUFFLE_DSL_PROCESSOR,TRUFFLE_TCK'], suite=truffle)
open(os.path.join(rubyDir, 'VERSION'), 'w').write('graal-vm\n')
# Build jruby-truffle
env = os.environ.copy()
env['JRUBY_BUILD_MORE_QUIET'] = 'true'
mx.run_maven(['-q', '--version', maven_repo_arg], nonZeroIsFatal=False, cwd=rubyDir, env=env)
mx.log('Building without tests')
mx.run_maven(['-q', '-DskipTests', maven_version_arg, maven_repo_arg], cwd=rubyDir, env=env)
mx.log('Building complete version')
mx.run_maven(['-q', '-Pcomplete', '-DskipTests', maven_version_arg, maven_repo_arg], cwd=rubyDir, env=env)
mx.run(['zip', '-d', 'maven/jruby-complete/target/jruby-complete-graal-vm.jar', 'META-INF/jruby.home/lib/*'], cwd=rubyDir)
mx.run(['bin/jruby', 'bin/gem', 'install', 'bundler', '-v', '1.10.6'], cwd=rubyDir)
mx.log('...finished build of {}'.format(self.subject))
def testgraal(args):
cloneFrom = mx.get_env("GRAAL_URL")
if not cloneFrom:
cloneFrom = "http://github.com/graalvm/graal-core"
graalSuiteSubDir = mx.get_env("GRAAL_SUITE_SUBDIR")
suite = mx.suite('truffle')
suiteDir = suite.dir
workDir = join(suite.get_output_root(), 'sanitycheck')
mx.ensure_dir_exists(join(workDir, suite.name))
for f in os.listdir(suiteDir):
subDir = os.path.join(suiteDir, f)
if subDir == suite.get_output_root():
continue
src = join(suiteDir, f)
tgt = join(workDir, suite.name, f)
if isdir(src):
if exists(tgt):
shutil.rmtree(tgt)
shutil.copytree(src, tgt)
else:
shutil.copy(src, tgt)
sanityDir = join(workDir, 'sanity')
git = mx.GitConfig()
if exists(sanityDir):
git.pull(sanityDir)
else:
git.clone(cloneFrom, sanityDir)
sanitySuiteDir = sanityDir if graalSuiteSubDir is None else join(sanityDir, graalSuiteSubDir)
return mx.run_mx(['--java-home=' + mx.get_jdk().home, 'gate', '-B--force-deprecation-as-warning', '--tags', 'build,test'], sanitySuiteDir)
def benchmark_callback(suite, commit):
suite.vc.update_to_branch(suite.vc_dir, commit)
mx.run_mx(['sforceimports'], suite=suite)
mx.run_mx(['--env', 'ce', 'sforceimports'], suite=get_suite('/vm'))
if args.enterprise and suite.name != 'vm-enterprise':
checkout_args = [
'--dynamicimports', '/vm-enterprise', 'checkout-downstream',
'vm', 'vm-enterprise'
]
if fetched_enterprise[0]:
checkout_args.append('--no-fetch')
mx.run_mx(checkout_args, out=mx.OutputCapture())
# Make sure vm is imported before vm-enterprise
get_suite('/vm')
mx.run_mx(['--env', 'ee', 'sforceimports'],
suite=get_suite('/vm-enterprise'))
fetched_enterprise[0] = True
suite.vc.update_to_branch(suite.vc_dir, commit)
mx.run_mx(['sforceimports'], suite=suite)
debug_str = "debug: graalpython={} graal={}".format(
get_commit(get_suite('graalpython')), get_commit(get_suite('/vm')))
if args.enterprise:
debug_str += " graal-enterprise={}".format(
get_commit(get_suite('/vm-enterprise')))
print(debug_str)
env = os.environ.copy()
env['MX_ALT_OUTPUT_ROOT'] = 'mxbuild-{}'.format(commit)
retcode = mx.run(shlex.split(args.build_command),
env=env,
nonZeroIsFatal=False)
if retcode:
raise RuntimeError(
"Failed to execute the build command for {}".format(commit))
output = mx.OutputCapture()
retcode = mx.run(shlex.split(args.benchmark_command),
env=env,
out=mx.TeeOutputCapture(output),
nonZeroIsFatal=False)
if retcode:
raise RuntimeError(
"Failed to execute benchmark for {}".format(commit))
match = re.search(
r'{}.*duration: ([\d.]+)'.format(
re.escape(args.benchmark_criterion)), output.data)
if not match:
raise RuntimeError("Failed to get result from the benchmark")
return float(match.group(1))
def verify_graalvm_configs(suites=None):
"""
Check the consistency of registered GraalVM configs.
:param suites: optionally restrict the check to the configs registered by this list of suites.
:type suites: list[str] or None
"""
import mx_sdk_vm_impl
child_env = os.environ.copy()
for env_var in ['DYNAMIC_IMPORTS', 'DEFAULT_DYNAMIC_IMPORTS', 'COMPONENTS', 'EXCLUDE_COMPONENTS', 'SKIP_LIBRARIES', 'NATIVE_IMAGES', 'FORCE_BASH_LAUNCHERS', 'DISABLE_POLYGLOT', 'DISABLE_LIBPOLYGLOT']:
if env_var in child_env:
del child_env[env_var]
for dist_name, _, components, suite, env_file in _vm_configs:
if env_file is not False and (suites is None or suite.name in suites):
_env_file = env_file or dist_name
graalvm_dist_name = '{base_name}_{dist_name}_JAVA{jdk_version}'.format(base_name=mx_sdk_vm_impl._graalvm_base_name, dist_name=dist_name, jdk_version=mx_sdk_vm_impl._src_jdk_version).upper().replace('-', '_')
mx.log("Checking that the env file '{}' in suite '{}' produces a GraalVM distribution named '{}'".format(_env_file, suite.name, graalvm_dist_name))
out = mx.LinesOutputCapture()
err = mx.LinesOutputCapture()
retcode = mx.run_mx(['--quiet', '--no-warning', '--env', _env_file, 'graalvm-dist-name'], suite, out=out, err=err, env=child_env, nonZeroIsFatal=False)
if retcode != 0:
mx.abort("Unexpected return code '{}' for 'graalvm-dist-name' for env file '{}' in suite '{}'. Output:\n{}\nError:\n{}".format(retcode, _env_file, suite.name, '\n'.join(out.lines), '\n'.join(err.lines)))
if len(out.lines) != 1 or out.lines[0] != graalvm_dist_name:
out2 = mx.LinesOutputCapture()
retcode2 = mx.run_mx(['--no-warning', '--env', _env_file, 'graalvm-components'], suite, out=out2, err=out2, env=child_env, nonZeroIsFatal=False)
if retcode2 or len(out2.lines) != 1:
got_components = '<error>'
diff = ''
else:
got_components = out2.lines[0] # example string: "['bpolyglot', 'cmp']"
got_components_set = set(got_components[1:-1].replace('\'', '').split(', '))
components_set = set(components)
added = list(got_components_set - components_set)
removed = list(components_set - got_components_set)
diff = ('Added:\n{}\n'.format(added) if added else '') + ('Removed:\n{}\n'.format(removed) if removed else '')
mx.abort("""\
Unexpected GraalVM dist name for env file '{}' in suite '{}'.
Expected dist name: '{}'
Actual dist name: '{}'.
Expected component list:
{}
Actual component list:
{}
{}Did you forget to update the registration of the GraalVM config?""".format(_env_file, suite.name, graalvm_dist_name, '\n'.join(out.lines + err.lines), sorted(components), got_components, diff))
def build(self):
if not self.subject.build:
mx.log("...skip build of {}".format(self.subject))
return
mx.log('...perform build of {}'.format(self.subject))
rubyDir = _suite.dir
# HACK: since the maven executable plugin does not configure the
# java executable that is used we unfortunately need to append it to the PATH
javaHome = os.getenv('JAVA_HOME')
if javaHome:
os.environ["PATH"] = os.environ["JAVA_HOME"] + '/bin' + os.pathsep + os.environ["PATH"]
mx.logv('Setting PATH to {}'.format(os.environ["PATH"]))
mx.logv('Calling java -version')
mx.run(['java', '-version'])
# Truffle version
truffle = mx.suite('truffle')
truffle_commit = truffle.vc.parent(truffle.dir)
mx.run_mx(['maven-install'], suite=truffle)
open(join(rubyDir, 'VERSION'), 'w').write('graal-vm\n')
# Build jruby-truffle and
mx.run(['find', '.'], nonZeroIsFatal=False, cwd=rubyDir)
mx.run_maven(['--version'], nonZeroIsFatal=False, cwd=rubyDir)
mx.log('Building without tests')
mx.run_maven(['-DskipTests', '-Dtruffle.version=' + truffle_commit], cwd=rubyDir)
mx.log('Building complete version')
mx.run_maven(['-Pcomplete', '-DskipTests', '-Dtruffle.version=' + truffle_commit], cwd=rubyDir)
mx.run(['zip', '-d', 'maven/jruby-complete/target/jruby-complete-graal-vm.jar', 'META-INF/jruby.home/lib/*'], cwd=rubyDir)
mx.run(['bin/jruby', 'bin/gem', 'install', 'bundler', '-v', '1.10.6'], cwd=rubyDir)
def checkout_suite(suite, commit):
suite.vc.update_to_branch(suite.vc_dir, commit)
mx.run_mx(['sforceimports'], suite=suite)
mx.run_mx(['--env', 'ce', 'sforceimports'], suite=get_suite('/vm'))
if args.enterprise and suite.name != 'vm-enterprise':
checkout_enterprise()
# Make sure vm is imported before vm-enterprise
get_suite('/vm')
mx.run_mx(['--env', 'ee', 'sforceimports'], suite=get_suite('/vm-enterprise'))
suite.vc.update_to_branch(suite.vc_dir, commit)
mx.run_mx(['sforceimports'], suite=suite)
debug_str = "debug: graalpython={} graal={}".format(
get_commit(get_suite('graalpython')), get_commit(get_suite('/vm')))
if args.enterprise:
debug_str += " graal-enterprise={}".format(get_commit(get_suite('/vm-enterprise')))
print(debug_str)
def testdownstream(args):
"""test downstream users of GraalCore"""
parser = ArgumentParser(prog='mx testdownstream')
parser.add_argument('--target', action='store', help='URL of client repo to clone', required=True, metavar='<url>')
parser.add_argument('--suitedir', action='store', help='directory of target suite in client repo', default='.', metavar='<path>')
parser.add_argument('-C', dest='clientMxCmd', action='append', help='arg to mx command run on client (e.g., -C-v -C--strict-compliance -Cgate)', default=[], metavar='<arg>')
args = parser.parse_args(args)
workDir = join(_suite.get_output_root(), 'testdownstream')
mirror = join(workDir, _suite.name)
if exists(mirror):
shutil.rmtree(mirror)
mx.ensure_dir_exists(mirror)
for f in os.listdir(_suite.dir):
subDir = join(_suite.dir, f)
if subDir == _suite.get_output_root():
continue
src = join(_suite.dir, f)
dst = join(mirror, f)
mx.logv('[Creating symlink from {} to {}]'.format(dst, src))
os.symlink(src, dst)
# Deduce a target name from the target URL
url = urlparse(args.target)
targetName = url.path
if targetName.rfind('/') != -1:
targetName = targetName[targetName.rfind('/') + 1:]
if targetName.endswith('.git'):
targetName = targetName[0:-len('.git')]
targetDir = join(workDir, targetName)
git = mx.GitConfig()
if exists(targetDir):
git.pull(targetDir)
else:
git.clone(args.target, targetDir)
# See if there's a matching (non-master) branch downstream and use it if there is
branch = git.git_command(_suite.dir, ['rev-parse', '--abbrev-ref', 'HEAD']).strip()
if branch != 'master':
git.git_command(targetDir, ['checkout', branch], abortOnError=False)
targetSuiteDir = join(targetDir, args.suitedir)
cmd = ['--java-home=' + mx.get_jdk().home] + args.clientMxCmd
mx.logv('[running "mx ' + ' '.join(cmd) + '" in ' + targetSuiteDir + ']')
return mx.run_mx(cmd, targetSuiteDir)
def testgraal(args):
cloneFrom = mx.get_env("GRAAL_URL")
if not cloneFrom:
cloneFrom = "http://github.com/graalvm/graal-core"
graalSuiteSubDir = mx.get_env("GRAAL_SUITE_SUBDIR")
suite = mx.suite('truffle')
suiteDir = suite.dir
workDir = join(suite.get_output_root(), 'sanitycheck')
mx.ensure_dir_exists(join(workDir, suite.name))
for f in os.listdir(suiteDir):
subDir = os.path.join(suiteDir, f)
if subDir == suite.get_output_root():
continue
src = join(suiteDir, f)
tgt = join(workDir, suite.name, f)
if isdir(src):
if exists(tgt):
shutil.rmtree(tgt)
shutil.copytree(src, tgt)
else:
shutil.copy(src, tgt)
sanityDir = join(workDir, 'sanity')
git = mx.GitConfig()
if exists(sanityDir):
git.pull(sanityDir)
else:
git.clone(cloneFrom, sanityDir)
sanitySuiteDir = sanityDir if graalSuiteSubDir is None else join(
sanityDir, graalSuiteSubDir)
return mx.run_mx([
'-v', '--java-home=' + mx.get_jdk().home, 'gate',
'-B--force-deprecation-as-warning', '--tags', 'build,test'
], sanitySuiteDir)
def graalpython_gate_runner(args, tasks):
with Task('GraalPython JUnit', tasks,
tags=[GraalPythonTags.junit]) as task:
if task:
punittest(['--verbose'])
with Task('GraalPython Python tests',
tasks,
tags=[GraalPythonTags.unittest]) as task:
if task:
gate_unittests()
with Task('GraalPython C extension tests',
tasks,
tags=[GraalPythonTags.cpyext]) as task:
if task:
gate_unittests(subdir="cpyext/")
with Task('GraalPython C extension managed tests',
tasks,
tags=[GraalPythonTags.cpyext_managed]) as task:
if task:
mx.run_mx([
"--dynamicimports", "sulong-managed", "python-gate-unittests",
"--llvm.configuration=managed", "--subdir=cpyext", "--"
])
with Task('GraalPython C extension sandboxed tests',
tasks,
tags=[GraalPythonTags.cpyext_sandboxed]) as task:
if task:
mx.run_mx([
"--dynamicimports", "sulong-managed", "python-gate-unittests",
"--llvm.configuration=sandboxed", "--subdir=cpyext", "--"
])
with Task('GraalPython Python tests on SVM',
tasks,
tags=[GraalPythonTags.svmunit]) as task:
if task:
svm_image_name = "./graalpython-svm"
if not os.path.exists(svm_image_name):
python_svm(["-h"])
else:
llvm_home = mx_subst.path_substitutions.substitute(
'--native.Dllvm.home=<path:SULONG_LIBS>')
args = [
"--python.CoreHome=%s" %
os.path.join(_suite.dir, "graalpython", "lib-graalpython"),
"--python.StdLibHome=%s" %
os.path.join(_suite.dir, "graalpython", "lib-python/3"),
llvm_home
]
run_python_unittests(svm_image_name, args)
with Task('GraalPython apptests', tasks,
tags=[GraalPythonTags.apptests]) as task:
if task:
apprepo = os.environ["GRAALPYTHON_APPTESTS_REPO_URL"]
_apptest_suite = _suite.import_suite(
"graalpython-apptests",
version="f40fcf3af008d30a67e0dbc325a0d90f1e68f0c0",
urlinfos=[
mx.SuiteImportURLInfo(mx_urlrewrites.rewriteurl(apprepo),
"git", mx.vc_system("git"))
])
mx.run_mx(["-p", _apptest_suite.dir, "graalpython-apptests"])
with Task('GraalPython license header update',
tasks,
tags=[GraalPythonTags.license]) as task:
if task:
python_checkcopyrights([])
with Task('GraalPython GraalVM shared-library build',
tasks,
tags=[GraalPythonTags.downstream,
GraalPythonTags.graalvm]) as task:
if task:
run_shared_lib_test()
with Task('GraalPython GraalVM build',
tasks,
tags=[GraalPythonTags.downstream,
GraalPythonTags.graalvm]) as task:
if task:
svm_image = python_svm(["--version"])
benchmark = os.path.join(PATH_MESO, "image-magix.py")
out = mx.OutputCapture()
mx.run([svm_image, benchmark],
nonZeroIsFatal=True,
out=mx.TeeOutputCapture(out))
success = "\n".join([
"[0, 0, 0, 0, 0, 0, 10, 10, 10, 0, 0, 10, 3, 10, 0, 0, 10, 10, 10, 0, 0, 0, 0, 0, 0]",
])
if success not in out.data:
mx.abort('Output from generated SVM image "' + svm_image +
'" did not match success pattern:\n' + success)
def testdownstream(suite, repoUrls, relTargetSuiteDir, mxCommands, branch=None):
"""
Tests a downstream repo against the current working directory state of `suite`.
:param mx.Suite suite: the suite to test against the downstream repo
:param list repoUrls: URLs of downstream repos to clone, the first of which is the repo being tested
:param str relTargetSuiteDir: directory of the downstream suite to test relative to the top level
directory of the downstream repo being tested
:param list mxCommands: argument lists for the mx commands run in downstream suite being tested
:param str branch: name of branch to look for in downstream repo(s)
"""
assert len(repoUrls) > 0
workDir = join(suite.get_output_root(), 'testdownstream')
# A mirror of each suites in the same repo as `suite` is created via copying
rel_mirror = os.path.relpath(suite.dir, mx.SuiteModel.siblings_dir(suite.dir))
in_subdir = os.sep in rel_mirror
suites_in_repo = [suite]
if in_subdir:
base = os.path.dirname(suite.dir)
for e in os.listdir(base):
candidate = join(base, e)
if candidate != suite.dir:
mxDir = mx._is_suite_dir(candidate)
if mxDir:
matches = [s for s in mx.suites() if s.dir == candidate]
if len(matches) == 0:
suites_in_repo.append(mx.SourceSuite(mxDir, primary=False, load=False))
else:
suites_in_repo.append(matches[0])
for suite_in_repo in suites_in_repo:
if suite_in_repo.vc_dir and suite_in_repo.dir != suite_in_repo.vc_dir:
mirror = join(workDir, basename(suite_in_repo.vc_dir), suite_in_repo.name)
else:
mirror = join(workDir, suite_in_repo.name)
if exists(mirror):
shutil.rmtree(mirror)
output_root = suite_in_repo.get_output_root()
def ignore_output_root(d, names):
mx.log('Copying ' + d)
if d == os.path.dirname(output_root):
mx.log('Omitting ' + output_root)
return [os.path.basename(output_root)]
return []
shutil.copytree(suite_in_repo.dir, mirror, ignore=ignore_output_root)
targetDir = None
for repoUrl in repoUrls:
# Deduce a target name from the target URL
url = urlparse(repoUrl)
targetName = url.path
if targetName.rfind('/') != -1:
targetName = targetName[targetName.rfind('/') + 1:]
if targetName.endswith('.git'):
targetName = targetName[0:-len('.git')]
repoWorkDir = join(workDir, targetName)
git = mx.GitConfig()
if exists(repoWorkDir):
git.pull(repoWorkDir)
else:
git.clone(repoUrl, repoWorkDir)
# See if there's a matching (non-master) branch and use it if there is
if not branch:
branch = git.git_command(suite.dir, ['rev-parse', '--abbrev-ref', 'HEAD']).strip()
if branch != 'master':
git.git_command(repoWorkDir, ['checkout', branch], abortOnError=False)
if not targetDir:
targetDir = repoWorkDir
assert not isabs(relTargetSuiteDir)
targetSuiteDir = join(targetDir, relTargetSuiteDir)
assert targetSuiteDir.startswith(targetDir)
mxpy = None if suite != mx._mx_suite else join(mirror, 'mx.py')
for command in mxCommands:
mx.logv('[running "mx ' + ' '.join(command) + '" in ' + targetSuiteDir + ']')
mx.run_mx(command, targetSuiteDir, mxpy=mxpy)
def graalpython_gate_runner(args, tasks):
_graalpytest_driver = "graalpython/com.oracle.graal.python.test/src/graalpytest.py"
_test_project = "graalpython/com.oracle.graal.python.test/"
with Task('GraalPython JUnit', tasks,
tags=[GraalPythonTags.junit]) as task:
if task:
punittest(['--verbose'])
with Task('GraalPython Python tests',
tasks,
tags=[GraalPythonTags.unittest]) as task:
if task:
test_args = [
_graalpytest_driver, "-v", _test_project + "src/tests/"
]
mx.command_function(
"python")(["--python.CatchAllExceptions=true"] + test_args)
if platform.system() != 'Darwin':
# TODO: re-enable when python3 is available on darwin
mx.log("Running tests with CPython")
mx.run(["python3"] + test_args, nonZeroIsFatal=True)
with Task('GraalPython C extension tests',
tasks,
tags=[GraalPythonTags.cpyext]) as task:
if task:
test_args = [
_graalpytest_driver, "-v", _test_project + "src/tests/cpyext/"
]
mx.command_function("python")(test_args)
if platform.system() != 'Darwin':
# TODO: re-enable when python3 is available on darwin
mx.log("Running tests with CPython")
mx.run(["python3"] + test_args, nonZeroIsFatal=True)
with Task('GraalPython Python tests on SVM',
tasks,
tags=[GraalPythonTags.svmunit]) as task:
if task:
if not os.path.exists("./graalpython-svm"):
python_svm(["-h"])
if os.path.exists("./graalpython-svm"):
langhome = mx_subst.path_substitutions.substitute(
'--native.Dllvm.home=<path:SULONG_LIBS>')
# tests root directory
tests_folder = "graalpython/com.oracle.graal.python.test/src/tests/"
# list of excluded tests
excluded = ["test_interop.py"]
def is_included(path):
if path.endswith(".py"):
basename = path.rpartition("/")[2]
return basename.startswith(
"test_") and basename not in excluded
return False
# list all 1st-level tests and exclude the SVM-incompatible ones
testfiles = []
paths = [tests_folder]
while paths:
path = paths.pop()
if is_included(path):
testfiles.append(path)
else:
try:
paths += [
(path + f if path.endswith("/") else "%s/%s" %
(path, f)) for f in os.listdir(path)
]
except OSError:
pass
test_args = [
"graalpython/com.oracle.graal.python.test/src/graalpytest.py",
"-v"
] + testfiles
mx.run([
"./graalpython-svm",
"--python.CoreHome=graalpython/lib-graalpython",
"--python.StdLibHome=graalpython/lib-python/3", langhome
] + test_args,
nonZeroIsFatal=True)
with Task('GraalPython downstream R tests',
tasks,
tags=[GraalPythonTags.downstream, GraalPythonTags.R]) as task:
script_r2p = os.path.join(_suite.dir, "graalpython", "benchmarks",
"src", "benchmarks", "interop",
"r_python_image_demo.r")
script_p2r = os.path.join(_suite.dir, "graalpython", "benchmarks",
"src", "benchmarks", "interop",
"python_r_image_demo.py")
pythonjars = os.pathsep.join([
os.path.join(_suite.dir, "mxbuild", "dists", "graalpython.jar"),
os.path.join(_suite.dir, "mxbuild", "dists", "graalpython-env.jar")
])
if task:
rrepo = os.environ["FASTR_REPO_URL"]
testdownstream(_suite, [
rrepo,
mx.suite("truffle").vc._remote_url(
mx.suite("truffle").dir, "origin")
], ".", [[
"--dynamicimports", "graalpython",
"--version-conflict-resolution", "latest_all", "build",
"--force-deprecation-as-warning"
],
[
"--cp-sfx", pythonjars, "r", "--polyglot",
"--file=%s" % script_r2p
]])
testdownstream(_suite, [
rrepo,
mx.suite("truffle").vc._remote_url(
mx.suite("truffle").dir, "origin")
], ".", [[
"--dynamicimports", "graalpython",
"--version-conflict-resolution", "latest_all", "build",
"--force-deprecation-as-warning"
],
[
"-v", "--cp-sfx", pythonjars, "r", "--jvm",
"--polyglot", "-e",
"eval.polyglot('python', path='%s')" % str(script_p2r)
]])
with Task('GraalPython apptests', tasks,
tags=[GraalPythonTags.apptests]) as task:
if task:
apprepo = os.environ["GRAALPYTHON_APPTESTS_REPO_URL"]
_apptest_suite = _suite.import_suite(
"graalpython-apptests",
urlinfos=[
mx.SuiteImportURLInfo(mx_urlrewrites.rewriteurl(apprepo),
"git", mx.vc_system("git"))
])
mx.run_mx(["-p", _apptest_suite.dir, "graalpython-apptests"])
with Task('GraalPython license header update',
tasks,
tags=[GraalPythonTags.license]) as task:
if task:
python_checkcopyrights([])
with Task('GraalPython GraalVM shared-library build',
tasks,
tags=[GraalPythonTags.downstream,
GraalPythonTags.graalvm]) as task:
if task:
run_shared_lib_test()
with Task('GraalPython GraalVM build',
tasks,
tags=[GraalPythonTags.downstream,
GraalPythonTags.graalvm]) as task:
if task:
svm_image = python_svm(["--version"])
benchmark = os.path.join("graalpython", "benchmarks", "src",
"benchmarks", "image_magix.py")
out = mx.OutputCapture()
mx.run([svm_image, benchmark],
nonZeroIsFatal=True,
out=mx.TeeOutputCapture(out))
success = "\n".join([
"[0, 0, 0, 0, 0, 0, 20, 20, 20, 0, 0, 20, 20, 20, 0, 0, 20, 20, 20, 0, 0, 0, 0, 0, 0]",
"[11, 12, 13, 14, 15, 21, 22, 23, 24, 25, 31, 32, 33, 34, 35, 41, 42, 43, 44, 45, 51, 52, 53, 54, 55]",
"[11, 12, 13, 14, 15, 21, 22, 23, 24, 25, 31, 32, 36, 36, 35, 41, 41, 40, 40, 45, 51, 52, 53, 54, 55]"
])
if success not in out.data:
mx.abort('Output from generated SVM image "' + svm_image +
'" did not match success pattern:\n' + success)
for name, iterations in sorted(python_test_benchmarks.iteritems()):
with Task('PythonBenchmarksTest:' + name,
tasks,
tags=[GraalPythonTags.benchmarks]) as task:
if task:
_gate_python_benchmarks_tests(
"graalpython/benchmarks/src/benchmarks/" + name + ".py",
iterations)
def testdownstream(suite,
repoUrls,
relTargetSuiteDir,
mxCommands,
branch=None):
"""
Tests a downstream repo against the current working directory state of `suite`.
:param mx.Suite suite: the suite to test against the downstream repo
:param list repoUrls: URLs of downstream repos to clone, the first of which is the repo being tested
:param str relTargetSuiteDir: directory of the downstream suite to test relative to the top level
directory of the downstream repo being tested
:param list mxCommands: argument lists for the mx commands run in downstream suite being tested
:param str branch: name of branch to look for in downstream repo(s)
"""
assert len(repoUrls) > 0
workDir = join(suite.get_output_root(), 'testdownstream')
# A mirror of each suites in the same repo as `suite` is created with symlinks
rel_mirror = os.path.relpath(suite.dir,
mx.SuiteModel.siblings_dir(suite.dir))
in_subdir = os.sep in rel_mirror
suites_in_repo = [suite]
if in_subdir:
base = os.path.dirname(suite.dir)
for e in os.listdir(base):
candidate = join(base, e)
if candidate != suite.dir:
mxDir = mx._is_suite_dir(candidate)
if mxDir:
matches = [s for s in mx.suites() if s.dir == candidate]
if len(matches) == 0:
suites_in_repo.append(
mx.SourceSuite(mxDir, primary=False, load=False))
else:
suites_in_repo.append(matches[0])
for suite_in_repo in suites_in_repo:
rel_mirror = os.path.relpath(
suite_in_repo.dir, mx.SuiteModel.siblings_dir(suite_in_repo.dir))
mirror = join(workDir, rel_mirror)
if exists(mirror):
shutil.rmtree(mirror)
mx.ensure_dir_exists(mirror)
for f in os.listdir(suite_in_repo.dir):
subDir = join(suite_in_repo.dir, f)
if subDir == suite_in_repo.get_output_root():
continue
src = join(suite_in_repo.dir, f)
dst = join(mirror, f)
mx.logv('[Creating symlink from {} to {}]'.format(dst, src))
relsrc = os.path.relpath(src, os.path.dirname(dst))
os.symlink(relsrc, dst)
targetDir = None
for repoUrl in repoUrls:
# Deduce a target name from the target URL
url = urlparse(repoUrl)
targetName = url.path
if targetName.rfind('/') != -1:
targetName = targetName[targetName.rfind('/') + 1:]
if targetName.endswith('.git'):
targetName = targetName[0:-len('.git')]
repoWorkDir = join(workDir, targetName)
git = mx.GitConfig()
if exists(repoWorkDir):
git.pull(repoWorkDir)
else:
git.clone(repoUrl, repoWorkDir)
# See if there's a matching (non-master) branch and use it if there is
if not branch:
branch = git.git_command(
suite.dir, ['rev-parse', '--abbrev-ref', 'HEAD']).strip()
if branch != 'master':
git.git_command(repoWorkDir, ['checkout', branch],
abortOnError=False)
if not targetDir:
targetDir = repoWorkDir
assert not isabs(relTargetSuiteDir)
targetSuiteDir = join(targetDir, relTargetSuiteDir)
assert targetSuiteDir.startswith(targetDir)
mxpy = None if suite != mx._mx_suite else join(mirror, 'mx.py')
for command in mxCommands:
mx.logv('[running "mx ' + ' '.join(command) + '" in ' +
targetSuiteDir + ']')
mx.run_mx(command, targetSuiteDir, mxpy=mxpy)
def _mx_vm(args, config, nonZeroIsFatal=True, out=None, err=None, timeout=None, env=None, quiet=False):
return mx.run_mx(config.mx_args() + args, suite=_vm_suite_dir(), nonZeroIsFatal=nonZeroIsFatal, out=out, err=err, timeout=timeout, env=env, quiet=quiet)
def python_coverage(args):
"Generate coverage report running args"
mx.run_mx(
['--jacoco=on', '--jacoco-whitelist-package=com.oracle.graal.python'] +
args)
mx.command_function("jacocoreport")(["--omit-excluded", "--format=html"])
def testdownstream(suite,
repoUrls,
relTargetSuiteDir,
mxCommands,
branch=None):
"""
Tests a downstream repo against the current working directory state of `suite`.
:param mx.Suite suite: the suite to test against the downstream repo
:param list repoUrls: URLs of downstream repos to clone, the first of which is the repo being tested
:param str relTargetSuiteDir: directory of the downstream suite to test relative to the top level
directory of the downstream repo being tested
:param list mxCommands: argument lists for the mx commands run in downstream suite being tested
:param str branch: name(s) of branch to look for in downstream repo(s)
"""
assert len(repoUrls) > 0
repoUrls = [mx_urlrewrites.rewriteurl(url) for url in repoUrls]
workDir = join(suite.get_output_root(), 'testdownstream')
# A mirror of each suites in the same repo as `suite` is created via copying
rel_mirror = os.path.relpath(suite.dir,
mx.SuiteModel.siblings_dir(suite.dir))
in_subdir = os.sep in rel_mirror
suites_in_repo = [suite]
if in_subdir:
base = os.path.dirname(suite.dir)
for e in os.listdir(base):
candidate = join(base, e)
if candidate != suite.dir:
mxDir = mx._is_suite_dir(candidate)
if mxDir:
matches = [s for s in mx.suites() if s.dir == candidate]
if len(matches) == 0:
suites_in_repo.append(
mx.SourceSuite(mxDir, primary=False, load=False))
else:
suites_in_repo.append(matches[0])
if suite.vc:
vc_metadir = mx._safe_path(mx.VC.get_vc(suite.vc_dir).metadir())
blacklist = {suite.vc_dir: [join(suite.vc_dir, vc_metadir)]}
else:
blacklist = {}
for suite_in_repo in suites_in_repo:
output_root = mx._safe_path(suite_in_repo.get_output_root())
blacklist.setdefault(dirname(output_root), []).append(output_root)
def omitted_dirs(d, names):
mx.log('Copying ' + d)
to_omit = []
for blacklisted_dir in blacklist.get(d, []):
mx.log('Omitting ' + blacklisted_dir)
to_omit.append(basename(blacklisted_dir))
return to_omit
if suite.vc_dir and suite.dir != suite.vc_dir:
mirror = join(workDir, basename(suite.vc_dir))
else:
mirror = join(workDir, suite.name)
if exists(mirror):
mx.rmtree(mirror)
mx.copytree(suite.vc_dir, mirror, ignore=omitted_dirs, symlinks=True)
targetDir = None
for repoUrl in repoUrls:
# Deduce a target name from the target URL
url = _urllib_parse.urlparse(repoUrl)
targetName = url.path
if targetName.rfind('/') != -1:
targetName = targetName[targetName.rfind('/') + 1:]
if targetName.endswith('.git'):
targetName = targetName[0:-len('.git')]
repoWorkDir = join(workDir, targetName)
git = mx.GitConfig()
if exists(repoWorkDir):
git.pull(repoWorkDir)
else:
git.clone(repoUrl, repoWorkDir)
if branch is None:
branch = []
elif isinstance(branch, str):
branch = [branch]
else:
assert isinstance(branch, list)
# fall back to the branch of the main repo
active_branch = git.active_branch(suite.dir, abortOnError=False)
if active_branch:
branch.append(active_branch)
updated = False
for branch_name in branch:
if git.update_to_branch(repoWorkDir,
branch_name,
abortOnError=False):
updated = True
break
if not updated:
mx.warn('Could not update {} to any of the following branches: {}'.
format(repoWorkDir, ', '.join(branch)))
if not targetDir:
targetDir = repoWorkDir
assert not isabs(relTargetSuiteDir)
targetSuiteDir = join(targetDir, relTargetSuiteDir)
assert targetSuiteDir.startswith(targetDir)
mxpy = None if suite != mx._mx_suite else join(mirror, 'mx.py')
for command in mxCommands:
mx.logv('[running "mx ' + ' '.join(command) + '" in ' +
targetSuiteDir + ']')
mx.run_mx(command, targetSuiteDir, mxpy=mxpy)
def graalpython_gate_runner(args, tasks):
with Task('GraalPython JUnit', tasks,
tags=[GraalPythonTags.junit]) as task:
if task:
punittest(['--verbose'])
with Task('GraalPython Python tests',
tasks,
tags=[GraalPythonTags.unittest]) as task:
if task:
gate_unittests()
with Task('GraalPython C extension tests',
tasks,
tags=[GraalPythonTags.cpyext]) as task:
if task:
# we deliberately added this to test the combination of Sulong and 'mx_unittest'
unittest([
'--regex',
re.escape(
'com.oracle.graal.python.test.module.MemoryviewTest'),
"-Dgraal.TraceTruffleCompilation=true"
])
gate_unittests(subdir="cpyext/")
with Task('GraalPython C extension managed tests',
tasks,
tags=[GraalPythonTags.cpyext_managed]) as task:
if task:
mx.run_mx([
"--dynamicimports", "sulong-managed", "python-gate-unittests",
"--llvm.configuration=managed", "--subdir=cpyext", "--"
])
with Task('GraalPython C extension sandboxed tests',
tasks,
tags=[GraalPythonTags.cpyext_sandboxed]) as task:
if task:
mx.run_mx([
"--dynamicimports", "sulong-managed", "python-gate-unittests",
"--llvm.configuration=sandboxed", "--subdir=cpyext", "--"
])
with Task('GraalPython Python tests on SVM',
tasks,
tags=[GraalPythonTags.svmunit]) as task:
if task:
svm_image_name = "./graalpython-svm"
if not os.path.exists(svm_image_name):
svm_image_name = python_svm(["-h"])
llvm_home = mx_subst.path_substitutions.substitute(
'--native.Dllvm.home=<path:SULONG_LIBS>')
args = [
"--python.CoreHome=%s" %
os.path.join(SUITE.dir, "graalpython", "lib-graalpython"),
"--python.StdLibHome=%s" %
os.path.join(SUITE.dir, "graalpython", "lib-python/3"),
llvm_home
]
run_python_unittests(svm_image_name, args)
with Task('GraalPython apptests', tasks,
tags=[GraalPythonTags.apptests]) as task:
if task:
apprepo = os.environ["GRAALPYTHON_APPTESTS_REPO_URL"]
_apptest_suite = SUITE.import_suite(
"graalpython-apptests",
version="1fc0e86a54cbe090d36f262c062d8f4eee8f2e6d",
urlinfos=[
mx.SuiteImportURLInfo(mx_urlrewrites.rewriteurl(apprepo),
"git", mx.vc_system("git"))
])
mx.log(
" ".join(["Running", "mx"] +
["-p", _apptest_suite.dir, "graalpython-apptests"]))
mx.run_mx(["-p", _apptest_suite.dir, "graalpython-apptests"])
with Task('GraalPython license header update',
tasks,
tags=[GraalPythonTags.license]) as task:
if task:
python_checkcopyrights([])
with Task('GraalPython GraalVM shared-library build',
tasks,
tags=[GraalPythonTags.downstream,
GraalPythonTags.graalvm]) as task:
if task:
run_shared_lib_test()
with Task('GraalPython GraalVM build',
tasks,
tags=[GraalPythonTags.downstream,
GraalPythonTags.graalvm]) as task:
if task:
svm_image = python_svm(["--version"])
benchmark = os.path.join(PATH_MESO, "image-magix.py")
out = mx.OutputCapture()
mx.run([svm_image, benchmark],
nonZeroIsFatal=True,
out=mx.TeeOutputCapture(out))
success = "\n".join([
"[0, 0, 0, 0, 0, 0, 10, 10, 10, 0, 0, 10, 3, 10, 0, 0, 10, 10, 10, 0, 0, 0, 0, 0, 0]",
])
if success not in out.data:
mx.abort('Output from generated SVM image "' + svm_image +
'" did not match success pattern:\n' + success)
llvm_home = mx_subst.path_substitutions.substitute(
'--native.Dllvm.home=<path:SULONG_LIBS>')
args = [
"--python.CoreHome=%s" %
os.path.join(SUITE.dir, "graalpython", "lib-graalpython"),
"--python.StdLibHome=%s" %
os.path.join(SUITE.dir, "graalpython", "lib-python/3"),
llvm_home
]
run_python_unittests(svm_image, args)
def testdownstream(suite, repoUrls, relTargetSuiteDir, mxCommands, branch=None):
"""
Tests a downstream repo against the current working directory state of `suite`.
:param mx.Suite suite: the suite to test against the downstream repo
:param list repoUrls: URLs of downstream repos to clone, the first of which is the repo being tested
:param str relTargetSuiteDir: directory of the downstream suite to test relative to the top level
directory of the downstream repo being tested
:param list mxCommands: argument lists for the mx commands run in downstream suite being tested
:param str branch: name of branch to look for in downstream repo(s)
"""
assert len(repoUrls) > 0
workDir = join(suite.get_output_root(), 'testdownstream')
# A mirror of the current suite is created with symlinks
mirror = join(workDir, suite.name)
if exists(mirror):
shutil.rmtree(mirror)
mx.ensure_dir_exists(mirror)
for f in os.listdir(suite.dir):
subDir = join(suite.dir, f)
if subDir == suite.get_output_root():
continue
src = join(suite.dir, f)
dst = join(mirror, f)
mx.logv('[Creating symlink from {} to {}]'.format(dst, src))
relsrc = os.path.relpath(src, os.path.dirname(dst))
os.symlink(relsrc, dst)
targetDir = None
for repoUrl in repoUrls:
# Deduce a target name from the target URL
url = urlparse(repoUrl)
targetName = url.path
if targetName.rfind('/') != -1:
targetName = targetName[targetName.rfind('/') + 1:]
if targetName.endswith('.git'):
targetName = targetName[0:-len('.git')]
repoWorkDir = join(workDir, targetName)
git = mx.GitConfig()
if exists(repoWorkDir):
git.pull(repoWorkDir)
else:
git.clone(repoUrl, repoWorkDir)
# See if there's a matching (non-master) branch downstream and use it if there is
if not branch:
branch = git.git_command(suite.dir, ['rev-parse', '--abbrev-ref', 'HEAD']).strip()
if branch != 'master':
git.git_command(repoWorkDir, ['checkout', branch], abortOnError=False)
if not targetDir:
targetDir = repoWorkDir
assert not isabs(relTargetSuiteDir)
targetSuiteDir = join(targetDir, relTargetSuiteDir)
assert targetSuiteDir.startswith(targetDir)
mxpy = None if suite != mx._mx_suite else join(mirror, 'mx.py')
for command in mxCommands:
mx.logv('[running "mx ' + ' '.join(command) + '" in ' + targetSuiteDir + ']')
mx.run_mx(command, targetSuiteDir, mxpy=mxpy)
def graalpython_gate_runner(args, tasks):
with Task('GraalPython JUnit', tasks,
tags=[GraalPythonTags.junit]) as task:
if task:
punittest(['--verbose'])
with Task('GraalPython Python tests',
tasks,
tags=[GraalPythonTags.unittest]) as task:
if task:
gate_unittests()
with Task('GraalPython C extension tests',
tasks,
tags=[GraalPythonTags.cpyext]) as task:
if task:
# we deliberately added this to test the combination of Sulong and 'mx_unittest'
unittest([
'--regex',
re.escape(
'com.oracle.graal.python.test.module.MemoryviewTest'),
"-Dgraal.TraceTruffleCompilation=true"
])
gate_unittests(subdir="cpyext/")
with Task('GraalPython C extension managed tests',
tasks,
tags=[GraalPythonTags.cpyext_managed]) as task:
if task:
mx.run_mx([
"--dynamicimports", "sulong-managed", "python-gate-unittests",
"--llvm.configuration=managed", "--subdir=cpyext", "--"
])
with Task('GraalPython C extension sandboxed tests',
tasks,
tags=[GraalPythonTags.cpyext_sandboxed]) as task:
if task:
mx.run_mx([
"--dynamicimports", "sulong-managed", "python-gate-unittests",
"--llvm.configuration=sandboxed", "--subdir=cpyext", "--"
])
with Task('GraalPython Python tests on SVM',
tasks,
tags=[GraalPythonTags.svmunit]) as task:
if task:
svm_image_name = python_svm(["-h"])
run_python_unittests(svm_image_name)
with Task('GraalPython license header update',
tasks,
tags=[GraalPythonTags.license]) as task:
if task:
python_checkcopyrights([])
with Task('GraalPython GraalVM shared-library build',
tasks,
tags=[GraalPythonTags.downstream,
GraalPythonTags.graalvm]) as task:
if task:
run_shared_lib_test()
with Task('GraalPython GraalVM build',
tasks,
tags=[GraalPythonTags.downstream,
GraalPythonTags.graalvm]) as task:
if task:
svm_image = python_svm(["--version"])
benchmark = os.path.join(PATH_MESO, "image-magix.py")
out = mx.OutputCapture()
mx.run([svm_image, benchmark],
nonZeroIsFatal=True,
out=mx.TeeOutputCapture(out))
success = "\n".join([
"[0, 0, 0, 0, 0, 0, 10, 10, 10, 0, 0, 10, 3, 10, 0, 0, 10, 10, 10, 0, 0, 0, 0, 0, 0]",
])
if success not in out.data:
mx.abort('Output from generated SVM image "' + svm_image +
'" did not match success pattern:\n' + success)
def run_shared_lib_test(args=None):
mx.run_mx([
"--dynamicimports", "/substratevm,/vm", "build",
"--force-deprecation-as-warning", "--dependencies",
"GRAAL_MANAGEMENT,POLYGLOT_NATIVE_API_HEADERS,libpolyglot.so.image"
],
nonZeroIsFatal=True)
vmdir = os.path.join(mx.suite("truffle").dir, "..", "vm")
svm_lib_path = os.path.join(vmdir, "mxbuild",
"-".join([mx.get_os(),
mx.get_arch()]),
"libpolyglot.so.image")
fd = name = progname = None
try:
fd, name = tempfile.mkstemp(suffix='.c')
os.write(
fd, """
#include "stdio.h"
#include "polyglot_api.h"
#define assert_ok(msg, f) { if (!(f)) { \\
const poly_extended_error_info* error_info; \\
poly_get_last_error_info(isolate_thread, &error_info); \\
fprintf(stderr, "%s\\n", error_info->error_message); \\
return fprintf(stderr, "%s\\n", msg); } } while (0)
poly_isolate global_isolate;
poly_thread isolate_thread;
poly_engine engine;
poly_context context;
static poly_status create_context() {
poly_status status;
if (poly_attach_thread(global_isolate, &isolate_thread)) {
return poly_generic_failure;
}
poly_engine_builder engine_builder;
status = poly_create_engine_builder(isolate_thread, &engine_builder);
if (status != poly_ok) {
return status;
}
status = poly_engine_builder_build(isolate_thread, engine_builder, &engine);
if (status != poly_ok) {
return status;
}
poly_context_builder builder;
status = poly_create_context_builder(isolate_thread, NULL, 0, &builder);
if (status != poly_ok) {
return status;
}
status = poly_context_builder_engine(isolate_thread, builder, engine);
if (status != poly_ok) {
return status;
}
status = poly_context_builder_option(isolate_thread, builder, "python.VerboseFlag", "true");
if (status != poly_ok) {
return status;
}
status = poly_context_builder_allow_io(isolate_thread, builder, true);
if (status != poly_ok) {
return status;
}
status = poly_context_builder_build(isolate_thread, builder, &context);
if (status != poly_ok) {
return status;
}
poly_destroy_handle(isolate_thread, engine_builder);
poly_destroy_handle(isolate_thread, builder);
return poly_ok;
}
static poly_status tear_down_context() {
poly_status status = poly_context_close(isolate_thread, context, true);
if (status != poly_ok) {
return status;
}
status = poly_destroy_handle(isolate_thread, context);
if (status != poly_ok) {
return status;
}
status = poly_engine_close(isolate_thread, engine, true);
if (status != poly_ok) {
return status;
}
status = poly_destroy_handle(isolate_thread, engine);
if (status != poly_ok) {
return status;
}
if (poly_detach_thread(isolate_thread)) {
return poly_ok;
}
return poly_ok;
}
static int test_basic_python_function() {
assert_ok("Context creation failed.", create_context() == poly_ok);
poly_value func;
assert_ok("function eval failed", poly_context_eval(isolate_thread, context, "python", "test_func", "def test_func(x):\\n return x * x\\ntest_func", &func) == poly_ok);
int32_t arg_value = 42;
poly_value primitive_object;
assert_ok("create argument failed", poly_create_int32(isolate_thread, context, arg_value, &primitive_object) == poly_ok);
poly_value arg[1] = {primitive_object};
poly_value value;
assert_ok("invocation was unsuccessful", poly_value_execute(isolate_thread, func, arg, 1, &value) == poly_ok);
int32_t result_value;
poly_value_as_int32(isolate_thread, value, &result_value);
assert_ok("primitive free failed", poly_destroy_handle(isolate_thread, primitive_object) == poly_ok);
assert_ok("value free failed", poly_destroy_handle(isolate_thread, value) == poly_ok);
assert_ok("value computation was incorrect", result_value == 42 * 42);
assert_ok("func free failed", poly_destroy_handle(isolate_thread, func) == poly_ok);
assert_ok("Context tear down failed.", tear_down_context() == poly_ok);
return 0;
}
int32_t main(int32_t argc, char **argv) {
poly_isolate_params isolate_params = {};
if (poly_create_isolate(&isolate_params, &global_isolate)) {
return 1;
}
return test_basic_python_function();
}
""")
os.close(fd)
progname = os.path.join(_suite.dir, "graalpython-embedded-tool")
mx.log("".join([
"Running ", "'clang",
"-I%s" % svm_lib_path,
"-L%s" % svm_lib_path, name, "-o", progname, "-lpolyglot"
]))
mx.run([
"clang",
"-I%s" % svm_lib_path,
"-L%s" % svm_lib_path, name,
"-o%s" % progname, "-lpolyglot"
],
nonZeroIsFatal=True)
mx.log("Running " + progname + " with LD_LIBRARY_PATH " + svm_lib_path)
mx.run(["ls", "-l", progname])
mx.run(["ls", "-l", svm_lib_path])
run_env = {
"LD_LIBRARY_PATH": svm_lib_path,
"GRAAL_PYTHONHOME": os.environ["GRAAL_PYTHONHOME"]
}
print(run_env)
mx.run([progname], env=run_env)
finally:
try:
os.unlink(progname)
except:
pass
try:
os.close(fd)
except:
pass
try:
os.unlink(name)
except:
pass
def gate_body(args, tasks):
with Task('Vm: Basic GraalVM Tests', tasks,
tags=[VmGateTasks.compiler]) as t:
if t and mx_vm.has_component('GraalVM compiler'):
# 1. the build must be a GraalVM
# 2. the build must be JVMCI-enabled since the 'GraalVM compiler' component is registered
mx_vm.check_versions(
mx_vm.graalvm_output(),
graalvm_version_regex=mx_vm.graalvm_version_regex,
expect_graalvm=True,
check_jvmci=True)
with Task('Vm: GraalVM dist names', tasks,
tags=[VmGateTasks.integration]) as t:
if t:
for suite, env_file_name, graalvm_dist_name in env_tests:
out = mx.LinesOutputCapture()
mx.run_mx([
'--no-warning', '--env', env_file_name, 'graalvm-dist-name'
],
suite,
out=out,
err=out,
env={})
mx.log(
"Checking that the env file '{}' in suite '{}' produces a GraalVM distribution named '{}'"
.format(env_file_name, suite.name, graalvm_dist_name))
if len(out.lines) != 1 or out.lines[0] != graalvm_dist_name:
mx.abort(
"Unexpected GraalVM dist name for env file '{}' in suite '{}'.\nExpected: '{}', actual: '{}'.\nDid you forget to update the registration of the GraalVM config?"
.format(env_file_name, suite.name, graalvm_dist_name,
'\n'.join(out.lines)))
if mx_vm.has_component('LibGraal'):
libgraal_location = mx_vm.get_native_image_locations(
'LibGraal', 'jvmcicompiler')
if libgraal_location is None:
mx.warn(
"Skipping libgraal tests: no library enabled in the LibGraal component"
)
else:
extra_vm_arguments = [
'-XX:+UseJVMCICompiler', '-XX:+UseJVMCINativeLibrary',
'-XX:JVMCILibPath=' + dirname(libgraal_location)
]
if args.extra_vm_argument:
extra_vm_arguments += args.extra_vm_argument
import mx_compiler
# run avrora on the GraalVM binary itself
with Task('LibGraal Compiler:GraalVM DaCapo-avrora',
tasks,
tags=[VmGateTasks.libgraal]) as t:
if t:
mx.run([
join(mx_vm.graalvm_home(), 'bin',
'java'), '-XX:+UseJVMCICompiler',
'-XX:+UseJVMCINativeLibrary', '-jar',
mx.library('DACAPO').get_path(True), 'avrora'
])
with Task('LibGraal Compiler:CTW',
tasks,
tags=[VmGateTasks.libgraal]) as t:
if t:
mx_compiler.ctw([
'-DCompileTheWorld.Config=Inline=false CompilationFailureAction=ExitVM',
'-esa',
'-XX:+EnableJVMCI',
'-DCompileTheWorld.MultiThreaded=true',
'-Dgraal.InlineDuringParsing=false',
'-Dgraal.TrackNodeSourcePosition=true',
'-DCompileTheWorld.Verbose=false',
'-XX:ReservedCodeCacheSize=300m',
], extra_vm_arguments)
mx_compiler.compiler_gate_benchmark_runner(
tasks, extra_vm_arguments, prefix='LibGraal Compiler:')
with Task('LibGraal Truffle:unittest',
tasks,
tags=[VmGateTasks.libgraal]) as t:
if t:
def _unittest_config_participant(config):
vmArgs, mainClass, mainClassArgs = config
def is_truffle_fallback(arg):
fallback_args = [
"-Dtruffle.TruffleRuntime=com.oracle.truffle.api.impl.DefaultTruffleRuntime",
"-Dgraalvm.ForcePolyglotInvalid=true"
]
return arg in fallback_args
newVmArgs = [
arg for arg in vmArgs
if not is_truffle_fallback(arg)
]
return (newVmArgs, mainClass, mainClassArgs)
mx_unittest.add_config_participant(
_unittest_config_participant)
excluded_tests = environ.get("TEST_LIBGRAAL_EXCLUDE")
if excluded_tests:
with NamedTemporaryFile(prefix='blacklist.',
mode='w',
delete=False) as fp:
fp.file.writelines(
[l + '\n' for l in excluded_tests.split()])
unittest_args = ["--blacklist", fp.name]
else:
unittest_args = []
unittest_args = unittest_args + [
"--enable-timing", "--verbose"
]
mx_unittest.unittest(unittest_args + extra_vm_arguments + [
"-Dgraal.TruffleCompileImmediately=true",
"-Dgraal.TruffleBackgroundCompilation=false", "truffle"
])
else:
mx.warn("Skipping libgraal tests: component not enabled")
gate_substratevm(tasks)
gate_sulong(tasks)
gate_ruby(tasks)
gate_python(tasks)
gate_svm_truffle_tck_js(tasks)
def testdownstream(suite,
repoUrls,
relTargetSuiteDir,
mxCommands,
branch=None):
"""
Tests a downstream repo against the current working directory state of `suite`.
:param mx.Suite suite: the suite to test against the downstream repo
:param list repoUrls: URLs of downstream repos to clone, the first of which is the repo being tested
:param str relTargetSuiteDir: directory of the downstream suite to test relative to the top level
directory of the downstream repo being tested
:param list mxCommands: argument lists for the mx commands run in downstream suite being tested
:param str branch: name of branch to look for in downstream repo(s)
"""
assert len(repoUrls) > 0
repoUrls = [mx_urlrewrites.rewriteurl(url) for url in repoUrls]
workDir = join(suite.get_output_root(), 'testdownstream')
# A mirror of each suites in the same repo as `suite` is created via copying
rel_mirror = os.path.relpath(suite.dir,
mx.SuiteModel.siblings_dir(suite.dir))
in_subdir = os.sep in rel_mirror
suites_in_repo = [suite]
if in_subdir:
base = os.path.dirname(suite.dir)
for e in os.listdir(base):
candidate = join(base, e)
if candidate != suite.dir:
mxDir = mx._is_suite_dir(candidate)
if mxDir:
matches = [s for s in mx.suites() if s.dir == candidate]
if len(matches) == 0:
suites_in_repo.append(
mx.SourceSuite(mxDir, primary=False, load=False))
else:
suites_in_repo.append(matches[0])
for suite_in_repo in suites_in_repo:
if suite_in_repo.vc_dir and suite_in_repo.dir != suite_in_repo.vc_dir:
mirror = join(workDir, basename(suite_in_repo.vc_dir),
suite_in_repo.name)
else:
mirror = join(workDir, suite_in_repo.name)
if exists(mirror):
mx.rmtree(mirror)
output_root = mx._safe_path(suite_in_repo.get_output_root())
def ignore_output_root(d, names):
mx.log('Copying ' + d)
if d == os.path.dirname(output_root):
mx.log('Omitting ' + output_root)
return [os.path.basename(output_root)]
return []
mx.copytree(suite_in_repo.dir,
mirror,
ignore=ignore_output_root,
symlinks=True)
targetDir = None
for repoUrl in repoUrls:
# Deduce a target name from the target URL
url = _urllib_parse.urlparse(repoUrl)
targetName = url.path
if targetName.rfind('/') != -1:
targetName = targetName[targetName.rfind('/') + 1:]
if targetName.endswith('.git'):
targetName = targetName[0:-len('.git')]
repoWorkDir = join(workDir, targetName)
git = mx.GitConfig()
if exists(repoWorkDir):
git.pull(repoWorkDir)
else:
git.clone(repoUrl, repoWorkDir)
# See if there's a matching (non-master) branch and use it if there is
if not branch:
branch = git.git_command(
suite.dir, ['rev-parse', '--abbrev-ref', 'HEAD']).strip()
if branch != 'master':
git.git_command(repoWorkDir, ['checkout', branch],
abortOnError=False)
if not targetDir:
targetDir = repoWorkDir
assert not isabs(relTargetSuiteDir)
targetSuiteDir = join(targetDir, relTargetSuiteDir)
assert targetSuiteDir.startswith(targetDir)
mxpy = None if suite != mx._mx_suite else join(mirror, 'mx.py')
for command in mxCommands:
mx.logv('[running "mx ' + ' '.join(command) + '" in ' +
targetSuiteDir + ']')
mx.run_mx(command, targetSuiteDir, mxpy=mxpy)
