def run_java(self, args, out=None, err=None, cwd=None, nonZeroIsFatal=False):
tag = mx.get_jdk_option().tag
if tag and tag != mx_graal_core._JVMCI_JDK_TAG:
mx.abort("The '{0}/{1}' VM requires '--jdk={2}'".format(
self.name(), self.config_name(), mx_graal_core._JVMCI_JDK_TAG))
mx.get_jdk(tag=mx_graal_core._JVMCI_JDK_TAG).run_java(
args, out=out, err=out, cwd=cwd, nonZeroIsFatal=False)
def jdkartifactstats(args):
"""show stats about JDK deployed Graal artifacts"""
artifacts = {}
jdkDir = get_jvmci_jdk().home
def _getDeployedJars():
if JVMCI_VERSION < 9:
for root, _, filenames in os.walk(join(jdkDir, 'jre', 'lib')):
for f in filenames:
if f.endswith('.jar') and not f.endswith('.stripped.jar'):
yield join(root, f)
else:
for jdkDist in jdkDeployedDists:
dist = jdkDist.dist()
if isinstance(jdkDist, JvmciJDKDeployedDist):
yield dist.path
for jar in _getDeployedJars():
f = basename(jar)
if 'truffle' in f:
if 'enterprise' in f:
artifacts.setdefault('GraalEnterpriseTruffle', []).append(jar)
else:
artifacts.setdefault('GraalTruffle', []).append(jar)
elif 'enterprise' in f:
artifacts.setdefault('GraalEnterprise', []).append(jar)
elif 'jvmci' in f:
artifacts.setdefault('JVMCI', []).append(jar)
elif 'graal' in f:
artifacts.setdefault('Graal', []).append(jar)
else:
mx.logv('ignored: ' + jar)
print '{:>10} {:>10} {:>10} {}'.format('All', 'NoVars', 'None', 'Jar')
for category in sorted(artifacts.viewkeys()):
jars = artifacts[category]
if jars:
totals = (0, 0, 0)
print
for j in jars:
gSize = os.path.getsize(j)
stripped = j[:-len('.jar')] + '.stripped.jar'
mx.run([mx.get_jdk().pack200, '--repack', '--quiet', '-J-Djava.util.logging.config.file=', '-DLocalVariableTypeTable=strip', '-DLocalVariableTable=strip', stripped, j])
gLinesSourceSize = os.path.getsize(stripped)
mx.run([mx.get_jdk().pack200, '--repack', '--quiet', '-J-Djava.util.logging.config.file=', '-G', stripped, j])
gNoneSize = os.path.getsize(stripped)
os.remove(stripped)
print '{:10,} {:10,} {:10,} {}:{}'.format(gSize, gLinesSourceSize, gNoneSize, category, basename(j))
t1, t2, t3 = totals
totals = (t1 + gSize, t2 + gLinesSourceSize, t3 + gNoneSize)
t1, t2, t3 = totals
print '{:10,} {:10,} {:10,} {}'.format(t1, t2, t3, category)
jvmLib = join(jdkDir, relativeVmLibDirInJdk(), get_vm(), mx.add_lib_suffix(mx.add_lib_prefix('jvm')))
print
if exists(jvmLib):
print '{:10,} {}'.format(os.path.getsize(jvmLib), jvmLib)
else:
print '{:>10} {}'.format('<missing>', jvmLib)
def _sigtest_check(checktype, args, suite=None, projects=None):
"""run sigtest against Java projects with API"""
sigtestlib = mx.library('SIGTEST').get_path(resolve=True)
nonTestProjects = [p for p in mx.projects() if _should_test_project(p)]
if not nonTestProjects:
return 1
javaCompliance = max([p.javaCompliance for p in nonTestProjects])
class OutputCapture:
def __init__(self):
self.data = ""
def __call__(self, data):
self.data += data
failed = None
for p in nonTestProjects:
sigtestResults = p.dir + os.sep + 'snapshot.sigtest'
if not os.path.exists(sigtestResults):
continue
jdk = mx.get_jdk(javaCompliance)
cmd = ['-cp', mx._cygpathU2W(sigtestlib), 'com.sun.tdk.signaturetest.SignatureTest',
'-Static', '-Mode', 'bin', '-FileName', sigtestResults,
'-ClassPath', mx.classpath(p, jdk=jdk) + os.pathsep + jdk.bootclasspath(),
]
if checktype != 'all':
cmd.append('-b')
for pkg in mx._find_packages(p):
cmd = cmd + ['-PackageWithoutSubpackages', pkg]
out = OutputCapture()
print 'Checking ' + checktype + ' signature changes against ' + sigtestResults
exitcode = mx.run_java(cmd, nonZeroIsFatal=False, jdk=mx.get_jdk(javaCompliance), out=out, err=out)
mx.ensure_dir_exists(p.get_output_root())
with open(p.get_output_root() + os.path.sep + 'sigtest-junit.xml', 'w') as f:
f.write('<?xml version="1.0" encoding="UTF-8" ?>\n')
f.write('<testsuite tests="1" name="' + p.name + '.sigtest.' + checktype + '">\n')
f.write('<testcase classname="' + p.name + '" name="sigtest.' + checktype + '">\n')
if exitcode != 95:
print out.data
failed = sigtestResults
f.write('<failure type="SignatureCheck"><![CDATA[\n')
f.write(out.data)
f.write(']]></failure>')
else:
f.write('<system-err><![CDATA[\n')
f.write(out.data)
f.write(']]></system-err>')
f.write('</testcase>\n')
f.write('</testsuite>\n')
if failed:
mx.abort('Signature error in ' + failed)
else:
print 'OK.'
return 0
def _update_JDK9_STUBS_library():
"""
Sets the "path" and "sha1" attributes of the "JDK9_STUBS" library.
"""
jdk9InternalLib = _suite.suiteDict['libraries']['JDK9_STUBS']
jarInputDir = join(_suite.get_output_root(), 'jdk9-stubs')
jarPath = join(_suite.get_output_root(), 'jdk9-stubs.jar')
stubs = [
('jdk.internal.misc', 'VM', """package jdk.internal.misc;
public class VM {
public static String getSavedProperty(String key) {
throw new InternalError("should not reach here");
}
}
""")
]
if not exists(jarPath):
sourceFiles = []
for (package, className, source) in stubs:
sourceFile = join(jarInputDir, package.replace('.', os.sep), className + '.java')
mx.ensure_dir_exists(os.path.dirname(sourceFile))
with open(sourceFile, 'w') as fp:
fp.write(source)
sourceFiles.append(sourceFile)
jdk = mx.get_jdk(tag='default')
mx.run([jdk.javac, '-d', jarInputDir] + sourceFiles)
mx.run([jdk.jar, 'cf', jarPath, '.'], cwd=jarInputDir)
jdk9InternalLib['path'] = jarPath
jdk9InternalLib['sha1'] = mx.sha1OfFile(jarPath)
def _igvJdk():
v8u20 = mx.VersionSpec("1.8.0_20")
v8u40 = mx.VersionSpec("1.8.0_40")
v8 = mx.VersionSpec("1.8")
def _igvJdkVersionCheck(version):
return version >= v8 and (version < v8u20 or version >= v8u40)
return mx.get_jdk(_igvJdkVersionCheck, versionDescription='>= 1.8 and < 1.8.0u20 or >= 1.8.0u40', purpose="running IGV").home
def _runmake(args):
"""run the JDK make process
To build hotspot and import it into the JDK: "mx make hotspot import-hotspot"
{0}"""
jdkBuildDir = _get_jdk_build_dir()
if not exists(jdkBuildDir):
# JDK9 must be bootstrapped with a JDK8
compliance = mx.JavaCompliance('8')
jdk8 = mx.get_jdk(compliance.exactMatch, versionDescription=compliance.value)
cmd = ['sh', 'configure', '--with-debug-level=' + _vm.debugLevel, '--with-native-debug-symbols=external', '--disable-precompiled-headers', '--with-jvm-features=graal',
'--with-jvm-variants=' + _vm.jvmVariant, '--disable-warnings-as-errors', '--with-boot-jdk=' + jdk8.home, '--with-jvm-features=graal']
mx.run(cmd, cwd=_jdkSourceRoot)
cmd = [mx.gmake_cmd(), 'CONF=' + _vm.debugLevel]
if mx.get_opts().verbose:
cmd.append('LOG=debug')
cmd.extend(args)
if mx.get_opts().use_jdk_image and 'images' not in args:
cmd.append('images')
if not mx.get_opts().verbose:
mx.log('--------------- make execution ----------------------')
mx.log('Working directory: ' + _jdkSourceRoot)
mx.log('Command line: ' + ' '.join(cmd))
mx.log('-----------------------------------------------------')
mx.run(cmd, cwd=_jdkSourceRoot)
def run_r(args, command, parser=None, extraVmArgs=None, jdk=None, nonZeroIsFatal=True):
'''
Common function for running either R, Rscript (or rrepl).
args are a list of strings that came after 'command' on the command line
'''
parser = parser if parser is not None else ArgumentParser(prog='mx ' + command)
parser.add_argument('--J', dest='extraVmArgsList', action='append', help='extra Java VM arguments', metavar='@<args>')
parser.add_argument('--jdk', action='store', help='jdk to use')
ns, rargs = parser.parse_known_args(args)
if ns.extraVmArgsList:
j_extraVmArgsList = mx.split_j_args(ns.extraVmArgsList)
if extraVmArgs is None:
extraVmArgs = []
extraVmArgs += j_extraVmArgsList
if not jdk and ns.jdk:
jdk = mx.get_jdk(tag=ns.jdk)
# special cases normally handled in shell script startup
if command == 'r' and len(rargs) > 0:
if rargs[0] == 'RHOME':
print _fastr_suite.dir
sys.exit(0)
elif rargs[0] == 'CMD':
print 'CMD not implemented via mx, use: bin/R CMD ...'
sys.exit(1)
return do_run_r(rargs, command, extraVmArgs=extraVmArgs, jdk=jdk, nonZeroIsFatal=nonZeroIsFatal)
def testgraal(args):
cloneFrom = mx.get_env("GRAAL_URL")
if not cloneFrom:
cloneFrom = "http://github.com/graalvm/graal-core"
graalSuiteSubDir = mx.get_env("GRAAL_SUITE_SUBDIR")
suite = mx.suite('truffle')
suiteDir = suite.dir
workDir = join(suite.get_output_root(), 'sanitycheck')
mx.ensure_dir_exists(join(workDir, suite.name))
for f in os.listdir(suiteDir):
subDir = os.path.join(suiteDir, f)
if subDir == suite.get_output_root():
continue
src = join(suiteDir, f)
tgt = join(workDir, suite.name, f)
if isdir(src):
if exists(tgt):
shutil.rmtree(tgt)
shutil.copytree(src, tgt)
else:
shutil.copy(src, tgt)
sanityDir = join(workDir, 'sanity')
git = mx.GitConfig()
if exists(sanityDir):
git.pull(sanityDir)
else:
git.clone(cloneFrom, sanityDir)
sanitySuiteDir = sanityDir if graalSuiteSubDir is None else join(sanityDir, graalSuiteSubDir)
return mx.run_mx(['--java-home=' + mx.get_jdk().home, 'gate', '-B--force-deprecation-as-warning', '--tags', 'build,test'], sanitySuiteDir)
def _path_args(depNames=None):
"""
Gets the VM args for putting the dependencies named in `depNames` on the
class path and module path (if running on JDK9 or later).
:param names: a Dependency, str or list containing Dependency/str objects. If None,
then all registered dependencies are used.
"""
jdk = mx.get_jdk()
if jdk.javaCompliance >= '1.9':
modules = [as_java_module(dist, jdk) for dist in _suite.dists if get_java_module_info(dist)]
if modules:
# Partition resources between the class path and module path
modulepath = []
classpath = []
cpEntryToModule = {m.dist.path : m for m in modules}
for e in mx.classpath(depNames).split(os.pathsep):
if cpEntryToModule.has_key(e):
modulepath.append(cpEntryToModule[e].jarpath)
else:
classpath.append(e)
# The Truffle modules must be eagerly loaded as they could be referenced from
# the main class hence the --add-modules argument
return ['--add-modules=' + ','.join([m.name for m in modules]), '--module-path=' + os.pathsep.join(modulepath), '-cp', os.pathsep.join(classpath)]
return ['-cp', mx.classpath(depNames)]
def _tck(args):
"""runs TCK tests"""
parser = ArgumentParser(prog="mx tck", description="run the TCK tests", formatter_class=RawDescriptionHelpFormatter, epilog=_debuggertestHelpSuffix)
parser.add_argument("--tck-configuration", help="TCK configuration", choices=["compile", "debugger", "default"], default="default")
parsed_args, args = parser.parse_known_args(args)
tckConfiguration = parsed_args.tck_configuration
index = len(args)
for arg in reversed(args):
if arg.startswith("-"):
break
index = index - 1
args_no_tests = args[0:index]
tests = args[index:len(args)]
if len(tests) == 0:
tests = ["com.oracle.truffle.tck.tests"]
index = len(args_no_tests)
for arg in reversed(args_no_tests):
if arg.startswith("--"):
break
index = index - 1
unitTestOptions = args_no_tests[0:max(index-1, 0)]
jvmOptions = args_no_tests[index:len(args_no_tests)]
if tckConfiguration == "default":
unittest(unitTestOptions + ["--"] + jvmOptions + tests)
elif tckConfiguration == "debugger":
with mx.SafeFileCreation(os.path.join(tempfile.gettempdir(), "debugalot")) as sfc:
_execute_debugger_test(tests, sfc.tmpPath, False, unitTestOptions, jvmOptions)
elif tckConfiguration == "compile":
if not _is_graalvm(mx.get_jdk()):
mx.abort("The 'compile' TCK configuration requires graalvm execution, run with --java-home=<path_to_graalvm>.")
unittest(unitTestOptions + ["--"] + jvmOptions + ["-Dgraal.TruffleCompileImmediately=true", "-Dgraal.TruffleCompilationExceptionsAreThrown=true"] + tests)
def genInlineAssemblyParser(args=None, out=None):
"""generate inline assembly parser and scanner if corresponding grammer is new"""
generatedParserDir = _inlineAssemblySrcDir + _inlineAssemblyPackageName.replace(".", "/") + "/"
generatedFiles = [generatedParserDir + "Parser.java", generatedParserDir + "Scanner.java"]
configFiles = [
_inlineAssemblySrcDir + "InlineAssembly.atg",
_inlineAssemblySrcDir + "Parser.frame",
_inlineAssemblySrcDir + "Scanner.frame",
_inlineAssemblySrcDir + "Copyright.frame",
]
isAllGeneratedFilesExists = all([isfile(fileName) for fileName in generatedFiles])
latestGeneratedFile = (
sorted(generatedFiles, key=os.path.getmtime, reverse=True)[0] if isAllGeneratedFilesExists else ""
)
latestConfigFile = sorted(configFiles, key=os.path.getmtime, reverse=True)[0]
# If any auto-generated file is missing or any config file is updated after last auto-generation then regenerate the files
if not isAllGeneratedFilesExists or os.path.getmtime(latestConfigFile) >= os.path.getmtime(latestGeneratedFile):
localCocoJarFile = _suite.dir + "/lib/Coco.jar"
if not isfile(localCocoJarFile):
jarFileUrls = ["https://lafo.ssw.uni-linz.ac.at/pub/sulong-deps/Coco.jar"]
mx.download(localCocoJarFile, jarFileUrls)
command = [
mx.get_jdk(tag="jvmci").java,
"-jar",
localCocoJarFile,
"-package",
_inlineAssemblyPackageName,
"-o",
generatedParserDir,
_inlineAssemblyGrammer,
]
mx.run(command)
# Files get generated in Windows file format. Convert them to avoid style check failure during regression testing
dos2unix(generatedFiles)
def testgen(args):
'''generate the expected output for unit tests, and All/Failing test classes'''
parser = ArgumentParser(prog='r testgen')
parser.add_argument('--tests', action='store', default=_all_unit_tests(), help='pattern to match test classes')
args = parser.parse_args(args)
# check we are in the home directory
if os.getcwd() != _fastr_suite.dir:
mx.abort('must run rtestgen from FastR home directory')
# check the version of GnuR against FastR
try:
fastr_version = subprocess.check_output([mx.get_jdk().java, '-cp', mx.classpath('com.oracle.truffle.r.runtime'), 'com.oracle.truffle.r.runtime.RVersionNumber'])
gnur_version = subprocess.check_output(['R', '--version'])
if not gnur_version.startswith(fastr_version):
mx.abort('R version is incompatible with FastR, please update to ' + fastr_version)
except subprocess.CalledProcessError:
mx.abort('RVersionNumber.main failed')
# clean the test project to invoke the test analyzer AP
testOnly = ['--projects', 'com.oracle.truffle.r.test']
mx.clean(['--no-dist', ] + testOnly)
mx.build(testOnly)
# now just invoke junit with the appropriate options
mx.log("generating expected output for packages: ")
for pkg in args.tests.split(','):
mx.log(" " + str(pkg))
junit(['--tests', args.tests, '--gen-expected-output', '--gen-expected-quiet'])
def _unittest(args, annotations, prefixCp="", blacklist=None, whitelist=None, verbose=False, fail_fast=False, enable_timing=False, regex=None, color=False, eager_stacktrace=False, gc_after_test=False, suite=None):
testfile = os.environ.get('MX_TESTFILE', None)
if testfile is None:
(_, testfile) = tempfile.mkstemp(".testclasses", "mxtool")
os.close(_)
mainClass = 'com.oracle.mxtool.junit.MxJUnitWrapper'
if not exists(join(mx.project('com.oracle.mxtool.junit').output_dir(), mainClass.replace('.', os.sep) + '.class')):
mx.build(['--only', 'com.oracle.mxtool.junit'])
coreCp = mx.classpath(['com.oracle.mxtool.junit'])
coreArgs = []
if verbose:
coreArgs.append('-JUnitVerbose')
if fail_fast:
coreArgs.append('-JUnitFailFast')
if enable_timing:
coreArgs.append('-JUnitEnableTiming')
if color:
coreArgs.append('-JUnitColor')
if eager_stacktrace:
coreArgs.append('-JUnitEagerStackTrace')
if gc_after_test:
coreArgs.append('-JUnitGCAfterTest')
def harness(unittestCp, vmLauncher, vmArgs):
prefixArgs = ['-esa', '-ea']
if gc_after_test:
prefixArgs.append('-XX:-DisableExplicitGC')
with open(testfile) as fp:
testclasses = [l.rstrip() for l in fp.readlines()]
cp = prefixCp + coreCp + os.pathsep + unittestCp
# suppress menubar and dock when running on Mac
vmArgs = prefixArgs + ['-Djava.awt.headless=true'] + vmArgs + ['-cp', mx._separatedCygpathU2W(cp)]
# Execute Junit directly when one test is being run. This simplifies
# replaying the VM execution in a native debugger (e.g., gdb).
mainClassArgs = coreArgs + (testclasses if len(testclasses) == 1 else ['@' + mx._cygpathU2W(testfile)])
config = (vmArgs, mainClass, mainClassArgs)
for p in _config_participants:
config = p(config)
vmLauncher.launcher(*config)
vmLauncher = _vm_launcher
if vmLauncher is None:
jdk = mx.get_jdk()
def _run_vm(vmArgs, mainClass, mainClassArgs):
mx.run_java(vmArgs + [mainClass] + mainClassArgs, jdk=jdk)
vmLauncher = _VMLauncher('default VM launcher', _run_vm, jdk)
try:
_run_tests(args, harness, vmLauncher, annotations, testfile, blacklist, whitelist, regex, mx.suite(suite) if suite else None)
finally:
if os.environ.get('MX_TESTFILE') is None:
os.remove(testfile)
def microbench(self, args):
"""run JMH microbenchmark projects"""
parser = ArgumentParser(prog='mx microbench', description=microbench.__doc__,
usage="%(prog)s [command options|VM options] [-- [JMH options]]")
parser.add_argument('--jar', help='Explicitly specify micro-benchmark location')
self.add_arguments(parser)
mx.warn("`mx microbench` is deprecated! Consider moving to `mx_benchmark.JMHRunnerBenchmarkSuite`")
known_args, args = parser.parse_known_args(args)
vmArgs, jmhArgs = mx.extract_VM_args(args, useDoubleDash=True)
vmArgs = self.parseVmArgs(vmArgs)
# look for -f in JMH arguments
forking = True
for i in range(len(jmhArgs)):
arg = jmhArgs[i]
if arg.startswith('-f'):
if arg == '-f' and (i+1) < len(jmhArgs):
arg += jmhArgs[i+1]
try:
if int(arg[2:]) == 0:
forking = False
except ValueError:
pass
if known_args.jar:
# use the specified jar
args = ['-jar', known_args.jar]
if not forking:
args += vmArgs
# we do not know the compliance level of the jar - assuming 1.8
self.javaCompliance = mx.JavaCompliance('1.8')
else:
# find all projects with a direct JMH dependency
projects_dict = mx_benchmark.JMHRunnerBenchmarkSuite.get_jmh_projects_dict()
jmhProjects = projects_dict['JMH'] if 'JMH' in projects_dict else []
# get java compliance - 1.8 is minimum since we build jmh-runner with java 8
self.javaCompliance = max([p.javaCompliance for p in jmhProjects] + [mx.JavaCompliance('1.8')])
cpArgs = mx.get_runtime_jvm_args([p.name for p in jmhProjects], jdk=mx.get_jdk(self.javaCompliance))
# execute JMH runner
if forking:
args, cpVmArgs = self.filterVmArgs(cpArgs)
vmArgs += cpVmArgs
else:
args = cpArgs + vmArgs
args += ['org.openjdk.jmh.Main']
if forking:
def quoteSpace(s):
if " " in s:
return '"' + s + '"'
return s
forkedVmArgs = map(quoteSpace, self.parseForkedVmArgs(vmArgs))
args += ['--jvmArgsPrepend', ' '.join(forkedVmArgs)]
self.run_java(args + jmhArgs)
def _igvJdk():
v8u20 = mx.VersionSpec("1.8.0_20")
v8u40 = mx.VersionSpec("1.8.0_40")
v9 = mx.VersionSpec("1.9")
def _igvJdkVersionCheck(version):
# Remove v9 check once GR-3187 is resolved
return version < v9 and (version < v8u20 or version >= v8u40)
return mx.get_jdk(_igvJdkVersionCheck, versionDescription='>= 1.8 and < 1.8.0u20 or >= 1.8.0u40', purpose="running IGV").home
def _unittest_config_participant(config):
"""modifies the classpath to use the Sulong distribution jars instead of the classfiles to enable the use of Java's ServiceLoader"""
(vmArgs, mainClass, mainClassArgs) = config
cpIndex, _ = mx.find_classpath_arg(vmArgs)
junitCp = mx.classpath("com.oracle.mxtool.junit")
sulongCp = ':'.join([mx.classpath(mx.distribution(distr), jdk=mx.get_jdk(tag='jvmci')) for distr in sulongDistributions])
vmArgs[cpIndex] = junitCp + ":" + sulongCp
return (vmArgs, mainClass, mainClassArgs)
def microbench(self, args):
"""run JMH microbenchmark projects"""
parser = ArgumentParser(prog='mx microbench', description=microbench.__doc__,
usage="%(prog)s [command options|VM options] [-- [JMH options]]")
parser.add_argument('--jar', help='Explicitly specify micro-benchmark location')
self.add_arguments(parser)
known_args, args = parser.parse_known_args(args)
vmArgs, jmhArgs = mx.extract_VM_args(args, useDoubleDash=True)
vmArgs = self.parseVmArgs(vmArgs)
# look for -f in JMH arguments
forking = True
for i in range(len(jmhArgs)):
arg = jmhArgs[i]
if arg.startswith('-f'):
if arg == '-f' and (i+1) < len(jmhArgs):
arg += jmhArgs[i+1]
try:
if int(arg[2:]) == 0:
forking = False
except ValueError:
pass
if known_args.jar:
# use the specified jar
args = ['-jar', known_args.jar]
if not forking:
args += vmArgs
# we do not know the compliance level of the jar - assuming 1.8
self.javaCompliance = mx.JavaCompliance('1.8')
else:
# find all projects with a direct JMH dependency
jmhProjects = []
for p in mx.projects_opt_limit_to_suites():
if 'JMH' in [x.name for x in p.deps]:
jmhProjects.append(p)
# get java compliance - 1.8 is minimum since we build jmh-runner with java 8
self.javaCompliance = max([p.javaCompliance for p in jmhProjects] + [mx.JavaCompliance('1.8')])
cp = mx.classpath([p.name for p in jmhProjects], jdk=mx.get_jdk(self.javaCompliance))
# execute JMH runner
args = ['-cp', cp]
if not forking:
args += vmArgs
args += ['org.openjdk.jmh.Main']
if forking:
def quoteSpace(s):
if " " in s:
return '"' + s + '"'
return s
forkedVmArgs = map(quoteSpace, self.parseForkedVmArgs(vmArgs))
args += ['--jvmArgsPrepend', ' '.join(forkedVmArgs)]
self.run_java(args + jmhArgs)
def extraVmArgs(self):
# find all projects with a direct JMH dependency
jmhProjects = []
for p in mx.projects_opt_limit_to_suites():
if 'JMH' in [x.name for x in p.deps]:
jmhProjects.append(p)
cp = mx.classpath([p.name for p in jmhProjects], jdk=mx.get_jdk())
return ['-cp', cp]
def _truffle_gate_runner(args, tasks):
jdk = mx.get_jdk(tag=mx.DEFAULT_JDK_TAG)
if jdk.javaCompliance < '9':
with Task('Truffle Javadoc', tasks) as t:
if t: mx.javadoc(['--unified'])
with Task('Truffle UnitTests', tasks) as t:
if t: unittest(['--suite', 'truffle', '--enable-timing', '--verbose', '--fail-fast'])
with Task('Truffle Signature Tests', tasks) as t:
if t: sigtest(['--check', 'binary'])
def _runmake(args):
"""run the JDK make process
To build hotspot and import it into the JDK: "mx make hotspot import-hotspot"
{0}"""
jdkBuildDir = _get_jdk_build_dir()
if not exists(jdkBuildDir):
# JDK9 must be bootstrapped with a JDK8
compliance = mx.JavaCompliance('8')
jdk8 = mx.get_jdk(compliance.exactMatch, versionDescription=compliance.value)
cmd = ['sh', 'configure', '--with-debug-level=' + _vm.debugLevel, '--with-native-debug-symbols=external', '--disable-precompiled-headers',
'--with-jvm-variants=' + _vm.jvmVariant, '--disable-warnings-as-errors', '--with-boot-jdk=' + jdk8.home]
mx.run(cmd, cwd=_jdkSourceRoot)
cmd = [mx.gmake_cmd(), 'CONF=' + _vm.debugLevel]
if mx.get_opts().verbose:
cmd.append('LOG=debug')
cmd.extend(args)
if mx.get_opts().use_jdk_image and 'images' not in args:
cmd.append('images')
if not mx.get_opts().verbose:
mx.log('--------------- make execution ----------------------')
mx.log('Working directory: ' + _jdkSourceRoot)
mx.log('Command line: ' + ' '.join(cmd))
mx.log('-----------------------------------------------------')
mx.run(cmd, cwd=_jdkSourceRoot)
if 'images' in cmd:
jdkImageDir = join(jdkBuildDir, 'images', 'jdk')
# The OpenJDK build creates an empty cacerts file so copy one from
# the default JDK (which is assumed to be an OracleJDK)
srcCerts = join(mx.get_jdk(tag='default').home, 'lib', 'security', 'cacerts')
if not exists(srcCerts):
# Might be building with JDK8 which has cacerts under jre/
srcCerts = join(mx.get_jdk(tag='default').home, 'jre', 'lib', 'security', 'cacerts')
dstCerts = join(jdkImageDir, 'lib', 'security', 'cacerts')
if srcCerts != dstCerts:
shutil.copyfile(srcCerts, dstCerts)
_create_jdk_bundle(jdkBuildDir, _vm.debugLevel, jdkImageDir)
def _unittest_config_participant(config):
vmArgs, mainClass, mainClassArgs = config
if mx.get_jdk(tag='default').javaCompliance > '1.8':
# This is required to access jdk.internal.module.Modules which
# in turn allows us to dynamically open fields/methods to reflection.
vmArgs = vmArgs + ['--add-exports=java.base/jdk.internal.module=ALL-UNNAMED']
# This is required for the call to setAccessible in
# TruffleTCK.testValueWithSource to work.
vmArgs = vmArgs + ['--add-opens=com.oracle.truffle.truffle_api/com.oracle.truffle.api.vm=ALL-UNNAMED', '--add-modules=ALL-MODULE-PATH']
return (vmArgs, mainClass, mainClassArgs)
def jol(args):
"""Java Object Layout"""
joljar = mx.library('JOL_INTERNALS').get_path(resolve=True)
candidates = mx.findclass(args, logToConsole=False, matcher=lambda s, classname: s == classname or classname.endswith('.' + s) or classname.endswith('$' + s))
if len(candidates) > 0:
candidates = mx.select_items(sorted(candidates))
else:
# mx.findclass can be mistaken, don't give up yet
candidates = args
mx.run_java(['-javaagent:' + joljar, '-cp', os.pathsep.join([mx.classpath(jdk=mx.get_jdk()), joljar]), "org.openjdk.jol.MainObjectInternals"] + candidates)
def _truffle_gate_runner(args, tasks):
jdk = mx.get_jdk(tag=mx.DEFAULT_JDK_TAG)
with Task('Jackpot check', tasks) as t:
if t: jackpot(['--fail-on-warnings'], suite=None, nonZeroIsFatal=True)
if jdk.javaCompliance < '9':
with Task('Truffle Javadoc', tasks) as t:
if t: javadoc([])
with Task('Truffle UnitTests', tasks) as t:
if t: unittest(['--suite', 'truffle', '--enable-timing', '--verbose', '--fail-fast'])
with Task('Truffle Signature Tests', tasks) as t:
if t: sigtest(['--check', 'binary'])
with Task('File name length check', tasks) as t:
if t: check_filename_length([])
def _sigtest_generate(args, suite=None, projects=None):
"""run sigtest generator for Java projects with API"""
sigtestlib = mx.library('SIGTEST').get_path(resolve=True)
nonTestProjects = [p for p in mx.projects() if _should_test_project(p)]
if not nonTestProjects:
return 0
javaCompliance = max([p.javaCompliance for p in nonTestProjects])
for p in nonTestProjects:
sigtestResults = p.dir + os.sep + 'snapshot.sigtest'
cmd = ['-cp', mx._cygpathU2W(sigtestlib), 'com.sun.tdk.signaturetest.Setup',
'-Static', '-FileName', sigtestResults,
'-ClassPath', mx.classpath(p) + os.pathsep + mx.get_jdk(javaCompliance).bootclasspath(),
]
for pkg in mx.find_packages(p):
cmd = cmd + ['-PackageWithoutSubpackages', pkg]
exitcode = mx.run_java(cmd, nonZeroIsFatal=False, jdk=mx.get_jdk(javaCompliance))
if exitcode != 95:
mx.abort('Exit code was ' + str(exitcode) + ' while generating ' + sigtestResults)
if not exists(sigtestResults):
mx.abort('Cannot generate ' + sigtestResults)
mx.log("Sigtest snapshot generated to " + sigtestResults)
return 0
def _unittest_config_participant(config):
vmArgs, mainClass, mainClassArgs = config
if mx.get_jdk(tag='default').javaCompliance > '1.8':
# This is required to access jdk.internal.module.Modules which
# in turn allows us to dynamically open fields/methods to reflection.
vmArgs = vmArgs + [
'--add-exports=java.base/jdk.internal.module=ALL-UNNAMED'
]
# This is required for the call to setAccessible in
# TruffleTCK.testValueWithSource to work.
vmArgs = vmArgs + [
'--add-opens=com.oracle.truffle.truffle_api/com.oracle.truffle.api.vm=ALL-UNNAMED',
'--add-modules=ALL-MODULE-PATH'
]
return (vmArgs, mainClass, mainClassArgs)
def _truffle_gate_runner(args, tasks):
jdk = mx.get_jdk(tag=mx.DEFAULT_JDK_TAG)
with Task('Jackpot check', tasks) as t:
if t: jackpot(['--fail-on-warnings'], suite=None, nonZeroIsFatal=True)
if jdk.javaCompliance < '9':
with Task('Truffle Javadoc', tasks) as t:
if t: javadoc([])
with Task('File name length check', tasks) as t:
if t: check_filename_length([])
with Task('Truffle Signature Tests', tasks) as t:
if t: sigtest(['--check', 'binary'])
with Task('Truffle UnitTests', tasks) as t:
if t: unittest(['--suite', 'truffle', '--enable-timing', '--verbose', '--fail-fast'])
with Task('Truffle DSL max state bit tests', tasks) as t:
if t:
_truffle_gate_state_bitwidth_tests()
def runLispScript(args=None,
out=None,
get_classpath_options=getClasspathOptions):
dists = ["TruffleCourse"]
return mx.run_java(
get_classpath_options(dists) + [
"-XX:+EnableJVMCI",
"-XX:+UseJVMCICompiler",
"-Dgraal.TraceTruffleCompilation=true",
# "-Dgraal.TruffleCompileImmediately=true",
"-Dgraal.TruffleBackgroundCompilation=false",
"-Dgraal.PrintGraph=Network",
"-Dgraal.Dump=Truffle:9"
] + args + ["org.truffle.cs.lisp.main.LispRuntime", "sample.lisp"],
out=out,
jdk=mx.get_jdk())
def contents(self, tool, exe):
# platform support
all_params = '"%*"' if mx.is_windows() else '"$@"'
_quote = _quote_windows if mx.is_windows() else pipes.quote
# build command line
java = mx.get_jdk().java
classpath_deps = [dep for dep in self.subject.buildDependencies if isinstance(dep, mx.ClasspathDependency)]
extra_props = ['-Dorg.graalvm.launcher.executablename="{}"'.format(exe)]
main_class = self.subject.suite.toolchain._tool_to_main(tool)
jvm_args = [_quote(arg) for arg in mx.get_runtime_jvm_args(classpath_deps)]
command = [java] + jvm_args + extra_props + [main_class, all_params]
# create script
if mx.is_windows():
return "@echo off\n" + " ".join(command) + "\n"
else:
return "#!/usr/bin/env bash\n" + "exec " + " ".join(command) + "\n"
def testdownstream(args):
"""test downstream users of GraalCore"""
parser = ArgumentParser(prog='mx testdownstream')
parser.add_argument('--target', action='store', help='URL of client repo to clone', required=True, metavar='<url>')
parser.add_argument('--suitedir', action='store', help='directory of target suite in client repo', default='.', metavar='<path>')
parser.add_argument('-C', dest='clientMxCmd', action='append', help='arg to mx command run on client (e.g., -C-v -C--strict-compliance -Cgate)', default=[], metavar='<arg>')
args = parser.parse_args(args)
workDir = join(_suite.get_output_root(), 'testdownstream')
mirror = join(workDir, _suite.name)
if exists(mirror):
shutil.rmtree(mirror)
mx.ensure_dir_exists(mirror)
for f in os.listdir(_suite.dir):
subDir = join(_suite.dir, f)
if subDir == _suite.get_output_root():
continue
src = join(_suite.dir, f)
dst = join(mirror, f)
mx.logv('[Creating symlink from {} to {}]'.format(dst, src))
os.symlink(src, dst)
# Deduce a target name from the target URL
url = urlparse(args.target)
targetName = url.path
if targetName.rfind('/') != -1:
targetName = targetName[targetName.rfind('/') + 1:]
if targetName.endswith('.git'):
targetName = targetName[0:-len('.git')]
targetDir = join(workDir, targetName)
git = mx.GitConfig()
if exists(targetDir):
git.pull(targetDir)
else:
git.clone(args.target, targetDir)
# See if there's a matching (non-master) branch downstream and use it if there is
branch = git.git_command(_suite.dir, ['rev-parse', '--abbrev-ref', 'HEAD']).strip()
if branch != 'master':
git.git_command(targetDir, ['checkout', branch], abortOnError=False)
targetSuiteDir = join(targetDir, args.suitedir)
cmd = ['--java-home=' + mx.get_jdk().home] + args.clientMxCmd
mx.logv('[running "mx ' + ' '.join(cmd) + '" in ' + targetSuiteDir + ']')
return mx.run_mx(cmd, targetSuiteDir)
def suite_collector(suite, predicate, collector, javaProperties, seenSuites):
if suite.name in seenSuites:
return
seenSuites.add(suite.name)
suite.visit_imports(import_visitor, predicate=predicate, collector=collector, javaProperties=javaProperties, seenSuites=seenSuites)
for dist in suite.dists:
if predicate(dist):
for distCpEntry in mx.classpath_entries(dist):
if hasattr(distCpEntry, "getJavaProperties"):
for key, value in dist.getJavaProperties().items():
javaProperties[key] = value
if distCpEntry.isJdkLibrary() or distCpEntry.isJreLibrary():
cpPath = distCpEntry.classpath_repr(mx.get_jdk(), resolve=True)
else:
cpPath = distCpEntry.classpath_repr(resolve=True)
if cpPath:
collector[cpPath] = None
def suite_collector(suite, predicate, collector, javaProperties, seenSuites):
if suite.name in seenSuites:
return
seenSuites.add(suite.name)
suite.visit_imports(import_visitor, predicate=predicate, collector=collector, javaProperties=javaProperties, seenSuites=seenSuites)
for dist in suite.dists:
if predicate(dist):
for distCpEntry in mx.classpath_entries(dist):
if hasattr(distCpEntry, "getJavaProperties"):
for key, value in distCpEntry.getJavaProperties().items():
javaProperties[key] = value
if distCpEntry.isJdkLibrary() or distCpEntry.isJreLibrary():
cpPath = distCpEntry.classpath_repr(mx.get_jdk(), resolve=True)
else:
cpPath = distCpEntry.classpath_repr(resolve=True)
if cpPath:
collector[cpPath] = None
def _getJdkHome():
global _jdkHome
if not _jdkHome:
if mx.suite('compiler', fatalIfMissing=False):
import mx_compiler
if hasattr(mx_compiler, 'get_graaljdk'):
_jdkHome = mx_compiler.get_graaljdk().home
elif hasattr(mx_compiler, '_update_graaljdk'):
jdk = mx.get_jdk(tag='default')
_jdkHome, _ = mx_compiler._update_graaljdk(jdk)
else:
mx.abort(
'Cannot find a GraalJDK.\nAre you running with an old version of Graal?'
)
else:
_jdkHome = mx.get_env('JAVA_HOME')
return _jdkHome
def squeak(args, extra_vm_args=None, env=None, jdk=None, **kwargs):
if not env:
env = os.environ
check_vm_env = env.get('GRAALPYTHON_MUST_USE_GRAAL', False)
if check_vm_env:
if check_vm_env == '1':
check_vm(must_be_jvmci=True)
elif check_vm_env == '0':
check_vm()
vm_args, squeak_args = mx.extract_VM_args(args,
useDoubleDash=False,
defaultAllVMArgs=False)
classpath = ["de.hpi.swa.trufflesqueak"]
if mx.suite("tools-enterprise", fatalIfMissing=False):
classpath.append("tools-enterprise:CHROMEINSPECTOR")
vm_args = ['-cp', mx.classpath(classpath)]
if not jdk:
jdk = mx.get_jdk()
vm_args += [
# '-XX:+UseJVMCICompiler',
# '-Djvmci.Compiler=graal',
# '-Dgraal.TraceTruffleCompilation=true',
# '-Dgraal.Dump=',
# '-Dgraal.MethodFilter=Truffle.*',
# '-XX:CompileCommand=print,*OptimizedCallTarget.callRoot',
# '-XX:CompileCommand=exclude,*OptimizedCallTarget.callRoot',
# '-Dgraal.TruffleBackgroundCompilation=false',
# '-Dgraal.TruffleCompileImmediately=true',
# '-Dgraal.TraceTrufflePerformanceWarnings=true',
# '-Dgraal.TruffleCompilationExceptionsArePrinted=true',
]
# default: assertion checking is enabled
if extra_vm_args is None or '-da' not in extra_vm_args:
vm_args += ['-ea', '-esa']
if extra_vm_args:
vm_args += extra_vm_args
vm_args.append("de.hpi.swa.trufflesqueak.TruffleSqueakMain")
return mx.run_java(vm_args + squeak_args, jdk=jdk, **kwargs)
def runJMeterInBackground(jmeterBenchmarkSuite, benchmarkName):
if not JMeterBenchmarkSuite.findApplication(
jmeterBenchmarkSuite.applicationPort()):
mx.abort(
"Failed to find server application in JMeterBenchmarkSuite")
jmeterCmd = [mx.get_jdk().java, "-jar", jmeterBenchmarkSuite.jmeterPath(), "-n", "-t", jmeterBenchmarkSuite.workloadPath(benchmarkName), "-j", "/dev/stdout"] # pylint: disable=line-too-long
mx.log("Running JMeter: {0}".format(jmeterCmd))
jmeterBenchmarkSuite.jmeterOutput = mx.TeeOutputCapture(
mx.OutputCapture())
mx.run(jmeterCmd,
out=jmeterBenchmarkSuite.jmeterOutput,
err=subprocess.PIPE)
if not jmeterBenchmarkSuite.terminateApplication(
jmeterBenchmarkSuite.applicationPort()):
mx.abort(
"Failed to terminate server application in JMeterBenchmarkSuite"
)
def run_java(self,
args,
out=None,
err=None,
cwd=None,
nonZeroIsFatal=False):
tag = mx.get_jdk_option().tag
if tag and tag != mx_compiler._JVMCI_JDK_TAG:
mx.abort("The '{0}/{1}' VM requires '--jdk={2}'".format(
self.name(), self.config_name(), mx_compiler._JVMCI_JDK_TAG))
return mx.get_jdk(tag=mx_compiler._JVMCI_JDK_TAG).run_java(
args,
out=out,
err=out,
cwd=cwd,
nonZeroIsFatal=False,
command_mapper_hooks=self.command_mapper_hooks)
def layout_native_image_root(native_image_root):
def names_to_dists(dist_names):
return [mx.dependency(dist_name) for dist_name in dist_names]
def native_image_layout_dists(subdir, dist_names):
native_image_layout(names_to_dists(dist_names), subdir, native_image_root)
def native_image_extract_dists(subdir, dist_names):
native_image_extract(names_to_dists(dist_names), subdir, native_image_root)
# Create native-image layout for sdk parts
native_image_layout_dists(join('lib', 'boot'), ['sdk:GRAAL_SDK'])
native_image_layout_dists(join('lib', 'graalvm'), ['substratevm:SVM_DRIVER', 'sdk:LAUNCHER_COMMON'])
# Create native-image layout for compiler & jvmci parts
native_image_layout_dists(join('lib', 'jvmci'), graalDistribution)
jdk_config = mx.get_jdk()
jvmci_path = join(jdk_config.home, 'jre', 'lib', 'jvmci')
if os.path.isdir(jvmci_path):
for symlink_name in os.listdir(jvmci_path):
symlink_or_copy(join(jvmci_path, symlink_name), join(native_image_root, 'lib', 'jvmci', symlink_name))
# Create native-image layout for truffle parts
native_image_layout_dists(join('lib', 'truffle'), ['truffle:TRUFFLE_API', 'truffle:TRUFFLE_NFI'])
# Create native-image layout for tools parts
for tool_name in tools_map:
tool_descriptor = tools_map[tool_name]
native_image_layout_dists(join('tools', tool_name, 'builder'), tool_descriptor.builder_deps)
native_image_layout_dists(join('tools', tool_name), tool_descriptor.image_deps)
native_image_extract_dists(join('tools', tool_name), tool_descriptor.native_deps)
native_image_option_properties('tools', tool_name, native_image_root)
# Create native-image layout for svm parts
svm_subdir = join('lib', 'svm')
native_image_layout_dists(svm_subdir, librarySupportDistribution)
native_image_layout_dists(join(svm_subdir, 'builder'), svmDistribution + ['substratevm:POINTSTO', 'substratevm:OBJECTFILE'])
for clibrary_path in clibrary_paths():
from distutils.errors import DistutilsFileError # pylint: disable=no-name-in-module
try:
copy_tree(clibrary_path, join(native_image_root, join(svm_subdir, 'clibraries')))
except DistutilsFileError:
# ignore until GR-7932 is resolved
pass
def findbugs(args, fbArgs=None, suite=None, projects=None):
"""run FindBugs against non-test Java projects"""
findBugsHome = mx.get_env('FINDBUGS_HOME', None)
if suite is None:
suite = mx._primary_suite
if findBugsHome:
findbugsJar = join(findBugsHome, 'lib', 'findbugs.jar')
else:
findbugsLib = join(mx._mx_suite.get_output_root(), 'findbugs-3.0.0')
if not exists(findbugsLib):
tmp = tempfile.mkdtemp(prefix='findbugs-download-tmp', dir=mx._mx_suite.dir)
try:
findbugsDist = mx.library('FINDBUGS_DIST').get_path(resolve=True)
with zipfile.ZipFile(findbugsDist) as zf:
candidates = [e for e in zf.namelist() if e.endswith('/lib/findbugs.jar')]
assert len(candidates) == 1, candidates
libDirInZip = os.path.dirname(candidates[0])
zf.extractall(tmp)
shutil.copytree(join(tmp, libDirInZip), findbugsLib)
finally:
shutil.rmtree(tmp)
findbugsJar = join(findbugsLib, 'findbugs.jar')
assert exists(findbugsJar)
nonTestProjects = [p for p in mx.projects() if _should_test_project(p)]
if not nonTestProjects:
return 0
outputDirs = map(mx._cygpathU2W, [p.output_dir() for p in nonTestProjects])
javaCompliance = max([p.javaCompliance for p in nonTestProjects])
jdk = mx.get_jdk(javaCompliance)
if jdk.javaCompliance >= "1.9":
mx.log('FindBugs does not yet support JDK9 - skipping')
return 0
findbugsResults = join(suite.dir, 'findbugs.results')
if fbArgs is None:
fbArgs = defaultFindbugsArgs()
cmd = ['-jar', mx._cygpathU2W(findbugsJar)] + fbArgs
cmd = cmd + ['-auxclasspath', mx._separatedCygpathU2W(mx.classpath([p.name for p in nonTestProjects], jdk=jdk)), '-output', mx._cygpathU2W(findbugsResults), '-exitcode'] + args + outputDirs
exitcode = mx.run_java(cmd, nonZeroIsFatal=False, jdk=jdk)
if exitcode != 0:
with open(findbugsResults) as fp:
mx.log(fp.read())
os.unlink(findbugsResults)
return exitcode
def _tck(args):
"""runs TCK tests"""
parser = ArgumentParser(prog="mx tck",
description="run the TCK tests",
formatter_class=RawDescriptionHelpFormatter,
epilog=_debuggertestHelpSuffix)
parser.add_argument("--tck-configuration",
help="TCK configuration",
choices=["compile", "debugger", "default"],
default="default")
parsed_args, args = parser.parse_known_args(args)
tckConfiguration = parsed_args.tck_configuration
index = len(args)
for arg in reversed(args):
if arg.startswith("-"):
break
index = index - 1
args_no_tests = args[0:index]
tests = args[index:len(args)]
if len(tests) == 0:
tests = ["com.oracle.truffle.tck.tests"]
index = len(args_no_tests)
for arg in reversed(args_no_tests):
if arg.startswith("--"):
break
index = index - 1
unitTestOptions = args_no_tests[0:max(index - 1, 0)]
jvmOptions = args_no_tests[index:len(args_no_tests)]
if tckConfiguration == "default":
unittest(unitTestOptions + ["--"] + jvmOptions + tests)
elif tckConfiguration == "debugger":
with mx.SafeFileCreation(
os.path.join(tempfile.gettempdir(), "debugalot")) as sfc:
_execute_debugger_test(tests, sfc.tmpPath, False, unitTestOptions,
jvmOptions)
elif tckConfiguration == "compile":
if not _is_graalvm(mx.get_jdk()):
mx.abort(
"The 'compile' TCK configuration requires graalvm execution, run with --java-home=<path_to_graalvm>."
)
unittest(unitTestOptions + ["--"] + jvmOptions + [
"-Dgraal.TruffleCompileImmediately=true",
"-Dgraal.TruffleCompilationExceptionsAreThrown=true"
] + tests)
def layout_native_image_root(native_image_root):
def names_to_dists(dist_names):
return [mx.dependency(dist_name) for dist_name in dist_names]
def native_image_layout_dists(subdir, dist_names):
native_image_layout(names_to_dists(dist_names), subdir, native_image_root)
def native_image_extract_dists(subdir, dist_names):
native_image_extract(names_to_dists(dist_names), subdir, native_image_root)
# Create native-image layout for sdk parts
native_image_layout_dists(join('lib', 'boot'), ['sdk:GRAAL_SDK'])
native_image_layout_dists(join('lib', 'graalvm'), ['substratevm:SVM_DRIVER', 'sdk:LAUNCHER_COMMON'])
# Create native-image layout for compiler & jvmci parts
native_image_layout_dists(join('lib', 'jvmci'), graalDistribution)
jdk_config = mx.get_jdk()
jvmci_path = join(jdk_config.home, 'jre', 'lib', 'jvmci')
if os.path.isdir(jvmci_path):
for symlink_name in os.listdir(jvmci_path):
symlink_or_copy(join(jvmci_path, symlink_name), join(native_image_root, 'lib', 'jvmci', symlink_name))
# Create native-image layout for truffle parts
native_image_layout_dists(join('lib', 'truffle'), ['truffle:TRUFFLE_API', 'truffle:TRUFFLE_NFI'])
# Create native-image layout for tools parts
for tool_name in tools_map:
tool_descriptor = tools_map[tool_name]
native_image_layout_dists(join('tools', tool_name, 'builder'), tool_descriptor.builder_deps)
native_image_layout_dists(join('tools', tool_name), tool_descriptor.image_deps)
native_image_extract_dists(join('tools', tool_name), tool_descriptor.native_deps)
native_image_option_properties('tools', tool_name, native_image_root)
# Create native-image layout for svm parts
svm_subdir = join('lib', 'svm')
native_image_layout_dists(svm_subdir, librarySupportDistribution)
native_image_layout_dists(join(svm_subdir, 'builder'), svmDistribution + ['substratevm:POINTSTO', 'substratevm:OBJECTFILE'])
for clibrary_path in clibrary_paths():
from distutils.errors import DistutilsFileError # pylint: disable=no-name-in-module
try:
copy_tree(clibrary_path, join(native_image_root, join(svm_subdir, 'clibraries')))
except DistutilsFileError:
# ignore until GR-7932 is resolved
pass
def testgen(args):
'''generate the expected output for unit tests'''
# check we are in the home directory
if os.getcwd() != _fastr_suite.dir:
mx.abort('must run rtestgen from FastR home directory')
def need_version_check():
vardef = os.environ.has_key('FASTR_TESTGEN_GNUR')
varval = os.environ['FASTR_TESTGEN_GNUR'] if vardef else None
version_check = vardef and varval != 'internal'
if version_check:
rpath = join(varval, 'bin', 'R')
else:
rpath = None
return version_check, rpath
version_check, rpath = need_version_check()
if version_check:
# check the version of GnuR against FastR
try:
fastr_version = subprocess.check_output([
mx.get_jdk().java,
mx.get_runtime_jvm_args('com.oracle.truffle.r.runtime'),
'com.oracle.truffle.r.runtime.RVersionNumber'
])
gnur_version = subprocess.check_output([rpath, '--version'])
if not gnur_version.startswith(fastr_version):
mx.abort(
'R version is incompatible with FastR, please update to ' +
fastr_version)
except subprocess.CalledProcessError:
mx.abort('RVersionNumber.main failed')
tests = _all_generated_unit_tests()
# now just invoke unittst with the appropriate options
mx.log("generating expected output for packages: ")
for pkg in tests:
mx.log(" " + str(pkg))
os.environ["TZDIR"] = "/usr/share/zoneinfo/"
_unset_conflicting_envs()
mx_unittest.unittest([
'-Dfastr.test.gen.expected=' + _test_srcdir(),
'-Dfastr.test.gen.expected.quiet', '-Dfastr.test.project.output.dir=' +
mx.project('com.oracle.truffle.r.test').output_dir()
] + tests)
def webserver(args):
"""start or bundle a simple webserver with a web site"""
parser = ArgumentParser(
prog='mx webserver',
description='Start a webserver to serve the site at root or '
'bundle the site along with the server into an executable jar.')
parser.add_argument(
'-a',
'--archive',
help=
'instead of starting the server, create an executable jar in <path> '
'containing the server and the site at root such that `java -jar <path>` will start a server '
'for the site and open a browser at its root.',
metavar='<path>')
parser.add_argument('-p',
'--port',
help='local port on which the server should listen',
metavar='<num>')
parser.add_argument(
'--no-browse',
help='do not open default web browser on the served site',
action='store_true')
parser.add_argument('root', metavar='root')
args = parser.parse_args(args)
project = 'com.oracle.mxtool.webserver'
mainClass = project + '.WebServer'
mx.build(['--no-daemon', '--dependencies', project])
coreCp = mx.classpath([project])
jdk = mx.get_jdk(tag='default')
java_args = mx.get_runtime_jvm_args(project, coreCp, jdk=jdk)
java_args += [mainClass]
if args.archive:
java_args.append('--archive=' + args.archive)
if args.port:
java_args.append('--port=' + args.port)
if args.no_browse:
java_args.append('--no-browse')
java_args.append(args.root)
mx.run_java(java_args, jdk=jdk)
def _unittest_config_participant(config):
vmArgs, mainClass, mainClassArgs = config
if mx.get_jdk(tag='default').javaCompliance > '1.8':
# This is required to access jdk.internal.module.Modules which
# in turn allows us to dynamically open fields/methods to reflection.
vmArgs = vmArgs + ['--add-exports=java.base/jdk.internal.module=ALL-UNNAMED']
# Needed for om.oracle.truffle.api.dsl.test.TestHelper#instrumentSlowPath
vmArgs = vmArgs + ['--add-opens=org.graalvm.truffle/com.oracle.truffle.api.nodes=ALL-UNNAMED']
# This is required for the call to setAccessible in
# TruffleTCK.testValueWithSource to work.
vmArgs = vmArgs + ['--add-opens=org.graalvm.truffle/com.oracle.truffle.polyglot=ALL-UNNAMED', '--add-modules=ALL-MODULE-PATH']
config = (vmArgs, mainClass, mainClassArgs)
if _shouldRunTCKParticipant:
config = _unittest_config_participant_tck(config)
return config
def _open_module_exports_args():
"""
Gets the VM args for exporting all Truffle API packages on JDK9 or later.
"""
assert mx.get_jdk().javaCompliance >= '1.9'
truffle_api_dist = mx.distribution('TRUFFLE_API')
truffle_api_module_name = truffle_api_dist.moduleInfo['name']
module_info_open_exports = getattr(truffle_api_dist, 'moduleInfo:open')['exports']
args = []
for export in module_info_open_exports:
if ' to ' in export: # Qualified exports
package, targets = export.split(' to ')
targets = targets.replace(' ', '')
else: # Unqualified exports
package = export
targets = 'ALL-UNNAMED'
args.append('--add-exports=' + truffle_api_module_name + '/' + package + '=' + targets)
return args
def _configs():
class Configs:
def __init__(self):
self.configs = dict()
def eat(self, line):
(k, v) = line.split('#')
self.configs[k] = v.rstrip()
c = Configs()
mx.run([
mx.get_jdk().java, '-client', '-Xmx40m', '-Xms40m', '-XX:NewSize=30m',
'-cp',
mx.classpath(resolve=False),
'com.oracle.max.vm.tests.vm.MaxineTesterConfiguration'
],
out=c.eat)
return c.configs
def jackpot(args, suite=None, nonZeroIsFatal=False):
"""run Jackpot 3.0 against non-test Java projects"""
jackpotHome = mx.get_env('JACKPOT_HOME', None)
if jackpotHome:
jackpotJar = join(jackpotHome, 'jackpot.jar')
else:
jackpotJar = mx.library('JACKPOT').get_path(resolve=True)
assert exists(jackpotJar)
if suite is None:
suite = mx._primary_suite
nonTestProjects = [p for p in mx.projects() if _should_test_project(p)]
if not nonTestProjects:
return 0
groups = []
for p in nonTestProjects:
javacClasspath = []
deps = []
p.walk_deps(visit=lambda dep, edge: deps.append(dep) if dep.isLibrary() or dep.isJavaProject() else None)
annotationProcessorOnlyDeps = []
if len(p.annotation_processors()) > 0:
for apDep in p.annotation_processors():
if not apDep in deps:
deps.append(apDep)
annotationProcessorOnlyDeps.append(apDep)
for dep in deps:
if dep == p:
continue
if dep in annotationProcessorOnlyDeps:
continue
javacClasspath.append(dep.classpath_repr(resolve=True))
javaCompliance = p.javaCompliance
groups = groups + ['--group', "--classpath " + mx._separatedCygpathU2W(_escape_string(os.pathsep.join(javacClasspath))) + " --source " + str(p.javaCompliance) + " " + " ".join([_escape_string(d) for d in p.source_dirs()])]
cmd = ['-classpath', mx._cygpathU2W(jackpotJar), 'org.netbeans.modules.jackpot30.cmdline.Main']
cmd = cmd + ['--fail-on-warnings', '--progress'] + args + groups
return mx.run_java(cmd, nonZeroIsFatal=nonZeroIsFatal, jdk=mx.get_jdk(javaCompliance))
def jacocoreport(args):
"""create a JaCoCo coverage report
Creates the report from the 'jacoco.exec' file in the current directory.
Default output directory is 'coverage', but an alternative can be provided as an argument."""
dist_name = "MX_JACOCO_REPORT"
mx.command_function("build")(['--dependencies', dist_name])
dist = mx.distribution(dist_name)
jdk = mx.get_jdk(dist.javaCompliance)
parser = ArgumentParser(prog='mx jacocoreport')
parser.add_argument('--format', help='Export format (HTML or XML)', default='html', choices=['html', 'xml'])
parser.add_argument('--omit-excluded', action='store_true', help='omit excluded files from report')
parser.add_argument('output_directory', help='Output directory', default='coverage', nargs='?')
args = parser.parse_args(args)
# list of strings of the form "project-dir:binary-dir"
includedirs = []
for p in mx.projects():
projsetting = getattr(p, 'jacoco', '')
if (projsetting == 'include' or projsetting == '') and _jacoco_is_package_whitelisted(p.name):
if isinstance(p, mx.ClasspathDependency):
source_dirs = []
if p.isJavaProject():
source_dirs += p.source_dirs() + [p.source_gen_dir()]
includedirs.append(":".join([p.dir, p.classpath_repr(jdk)] + source_dirs))
def _run_reporter(extra_args=None):
mx.run_java(['-cp', mx.classpath([dist_name], jdk=jdk), '-jar', dist.path, '--in', 'jacoco.exec', '--out',
args.output_directory, '--format', args.format] +
(extra_args or []) +
sorted(includedirs),
jdk=jdk, addDefaultArgs=False)
if not args.omit_excluded:
_run_reporter()
else:
with tempfile.NamedTemporaryFile(suffix="jacoco-report-exclude", mode="w") as fp:
excludes, _ = _jacoco_excludes_includes()
fp.writelines((e + "\n" for e in excludes))
fp.flush()
_run_reporter(['--exclude-file', fp.name])
def runLLVM(args=None, out=None):
"""uses Sulong to execute a LLVM IR file"""
vmArgs, sulongArgsWithLibs = truffle_extract_VM_args(args)
sulongArgs = []
libNames = []
for arg in sulongArgsWithLibs:
if arg.startswith("-l"):
libNames.append(arg)
else:
sulongArgs.append(arg)
return mx.run_java(
getCommonOptions(libNames)
+ vmArgs
+ getClasspathOptions()
+ ["-XX:-UseJVMCIClassLoader", "com.oracle.truffle.llvm.LLVM"]
+ sulongArgs,
out=out,
jdk=mx.get_jdk(tag="jvmci"),
)
def _truffle_gate_runner(args, tasks):
jdk = mx.get_jdk(tag=mx.DEFAULT_JDK_TAG)
with Task('Jackpot check', tasks) as t:
if t: jackpot(['--fail-on-warnings'], suite=None, nonZeroIsFatal=True)
if jdk.javaCompliance < '9':
with Task('Truffle Javadoc', tasks) as t:
if t: javadoc([])
with Task('File name length check', tasks) as t:
if t: check_filename_length([])
with Task('Truffle Signature Tests', tasks) as t:
if t: sigtest(['--check', 'binary'])
with Task('Truffle UnitTests', tasks) as t:
if t: unittest(list(['--suite', 'truffle', '--enable-timing', '--verbose', '--fail-fast']))
if os.getenv('DISABLE_DSL_STATE_BITS_TESTS', 'false').lower() != 'true':
with Task('Truffle DSL max state bit tests', tasks) as t:
if t:
_truffle_gate_state_bitwidth_tests()
with Task('Validate parsers', tasks) as t:
if t: validate_parsers()
def _tck(args):
"""runs TCK tests"""
parser = ArgumentParser(prog="mx tck", description="run the TCK tests", formatter_class=RawDescriptionHelpFormatter, epilog=_tckHelpSuffix)
parser.add_argument("--tck-configuration", help="TCK configuration", choices=["compile", "debugger", "default"], default="default")
parsed_args, args = parser.parse_known_args(args)
tckConfiguration = parsed_args.tck_configuration
index = len(args)
for arg in reversed(args):
if arg.startswith("-"):
break
index = index - 1
args_no_tests = args[0:index]
tests = args[index:len(args)]
if len(tests) == 0:
tests = ["com.oracle.truffle.tck.tests"]
index = len(args_no_tests)
has_separator_arg = False
for arg in reversed(args_no_tests):
if arg.startswith("--"):
if arg == "--":
has_separator_arg = True
break
index = index - 1
unitTestOptions = args_no_tests[0:max(index - (1 if has_separator_arg else 0), 0)]
jvmOptions = args_no_tests[index:len(args_no_tests)]
if tckConfiguration == "default":
unittest(unitTestOptions + ["--"] + jvmOptions + tests)
elif tckConfiguration == "debugger":
with mx.SafeFileCreation(os.path.join(tempfile.gettempdir(), "debugalot")) as sfc:
_execute_debugger_test(tests, sfc.tmpPath, False, unitTestOptions, jvmOptions)
elif tckConfiguration == "compile":
if not _is_graalvm(mx.get_jdk()):
mx.abort("The 'compile' TCK configuration requires graalvm execution, run with --java-home=<path_to_graalvm>.")
compileOptions = [
"-Dpolyglot.engine.AllowExperimentalOptions=true",
"-Dpolyglot.engine.Mode=latency",
"-Dpolyglot.engine.CompilationFailureAction=Throw",
"-Dpolyglot.engine.CompileImmediately=true",
"-Dpolyglot.engine.BackgroundCompilation=false",
]
unittest(unitTestOptions + ["--"] + jvmOptions + compileOptions + tests)
def jol(args):
"""Java Object Layout"""
joljar = mx.library('JOL_INTERNALS').get_path(resolve=True)
candidates = mx.findclass(
args,
logToConsole=False,
matcher=lambda s, classname: s == classname or classname.endswith(
'.' + s) or classname.endswith('$' + s))
if len(candidates) > 0:
candidates = mx.select_items(sorted(candidates))
else:
# mx.findclass can be mistaken, don't give up yet
candidates = args
mx.run_java([
'-javaagent:' + joljar, '-cp',
os.pathsep.join([mx.classpath(
jdk=mx.get_jdk()), joljar]), "org.openjdk.jol.MainObjectInternals"
] + candidates)
def cflags(self):
default_cflags = []
if mx.is_windows() and mx.get_jdk(
tag=mx.DEFAULT_JDK_TAG).javaCompliance >= '11':
default_cflags += ['-FS'] # necessary until GR-16572 is resolved
if self._kind == self._kinds['shared_lib']:
default_cflags += dict(windows=['-MD'
], ).get(mx.get_os(), ['-fPIC'])
if mx.is_linux() or mx.is_darwin():
# Do not leak host paths via dwarf debuginfo
def add_debug_prefix(prefix_dir):
return '-fdebug-prefix-map={}={}'.format(
prefix_dir, mx.basename(prefix_dir))
default_cflags += [add_debug_prefix(self.suite.vc_dir)]
default_cflags += [add_debug_prefix(NinjaProject.get_jdk().home)]
default_cflags += ['-gno-record-gcc-switches']
return default_cflags + super(DefaultNativeProject, self).cflags
def jol(args):
"""Java Object Layout"""
joljar = mx.library('JOL_CLI').get_path(resolve=True)
commands = ['estimates', 'externals', 'footprint', 'heapdump', 'heapdumpstats', 'idealpack', 'internals', 'shapes', 'string-compress', 'help']
command = 'internals'
if len(args) == 0:
command = 'help'
elif args[0] in commands:
command, args = args[0], args[1:]
# classpath operations
if command in ['estimates', 'externals', 'footprint', 'internals']:
candidates = mx.findclass(args, logToConsole=False, matcher=lambda s, classname: s == classname or classname.endswith('.' + s) or classname.endswith('$' + s))
if len(candidates) > 0:
args = mx.select_items(sorted(candidates))
if len(args) > 0:
args = ['-cp', mx.classpath(jdk=mx.get_jdk())] + args
mx.run_java(['-javaagent:' + joljar, '-cp', joljar, 'org.openjdk.jol.Main', command] + args)
def contents(self, result):
java = mx.get_jdk().java
classpath_deps = [
dep for dep in self.subject.buildDependencies
if isinstance(dep, mx.ClasspathDependency)
]
jvm_args = [
pipes.quote(arg) for arg in mx.get_runtime_jvm_args(classpath_deps)
]
jvm_args.append('-Dorg.graalvm.language.ruby.home=' + root)
main_class = 'org.truffleruby.launcher.RubyLauncher'
ruby_options = [
'--experimental-options',
'--building-core-cexts',
'--launcher=' + result,
'--disable-gems',
'--disable-rubyopt',
]
command = [java] + jvm_args + [main_class] + ruby_options + ['"$@"']
return "#!/usr/bin/env bash\n" + "exec " + " ".join(command) + "\n"
def _unittest_config_participant(config):
vmArgs, mainClass, mainClassArgs = config
# Disable DefaultRuntime warning
vmArgs = vmArgs + ['-Dpolyglot.engine.WarnInterpreterOnly=false']
jdk = mx.get_jdk(tag='default')
if jdk.javaCompliance > '1.8':
# This is required to access jdk.internal.module.Modules which
# in turn allows us to dynamically open fields/methods to reflection.
vmArgs = vmArgs + [
'--add-exports=java.base/jdk.internal.module=ALL-UNNAMED'
]
# The arguments below are only actually needed if Truffle is deployed as a
# module. However, that's determined by the compiler suite which may not
# be present. In that case, adding these options results in annoying
# but harmless messages from the VM:
#
# WARNING: Unknown module: org.graalvm.truffle specified to --add-opens
#
# Needed for com.oracle.truffle.api.dsl.test.TestHelper#instrumentSlowPath
vmArgs = vmArgs + [
'--add-opens=org.graalvm.truffle/com.oracle.truffle.api.nodes=ALL-UNNAMED'
]
# This is required for the call to setAccessible in
# TruffleTCK.testValueWithSource to work.
vmArgs = vmArgs + [
'--add-opens=org.graalvm.truffle/com.oracle.truffle.polyglot=ALL-UNNAMED',
'--add-modules=ALL-MODULE-PATH'
]
# Needed for object model tests.
vmArgs = vmArgs + [
'--add-opens=org.graalvm.truffle/com.oracle.truffle.object=ALL-UNNAMED'
]
config = (vmArgs, mainClass, mainClassArgs)
if _shouldRunTCKParticipant:
config = _unittest_config_participant_tck(config)
return config
def testme(args):
"""run some or all of the Maxine tests
The Maxine sources include a variety of tests that can be run by a
special launcher. These include JUnit tests, VM micro tests, certain
benchmark suites and output comparison tests, amongst others.
Use "mx test -help" to see what other options this command accepts."""
maxineTesterDir = join(_maxine_home, 'maxine-tester')
if isdir(maxineTesterDir):
for root, _, files in os.walk(maxineTesterDir):
for name in files:
if name.rsplit(', ', 1) in [
'stdout', 'stderr', 'passed', 'failed', 'command'
]:
os.remove(join(root, name))
else:
os.mkdir(maxineTesterDir)
class Tee:
def __init__(self, f):
self.f = f
def eat(self, line):
mx.log(line.rstrip())
self.f.write(line)
console = join(maxineTesterDir, 'console')
with open(console, 'w', 0) as f:
tee = Tee(f)
jdk = mx.get_jdk()
mx.run_java([
'-cp',
suite_classpath(), 'com.oracle.max.vm.tests.vm.MaxineTester',
'-output-dir=maxine-tester',
'-graal-jar=' + mx.distribution('GRAAL').path,
'-refvm=' + jdk.java, '-refvm-args=' + ' '.join(jdk.java_args)
] + args,
out=tee.eat,
err=subprocess.STDOUT)
def contents(self, tool, exe):
# platform support
all_params = '"%*"' if mx.is_windows() else '"$@"'
_quote = _quote_windows if mx.is_windows() else pipes.quote
# build command line
java = mx.get_jdk().java
classpath_deps = [dep for dep in self.subject.buildDependencies if isinstance(dep, mx.ClasspathDependency)]
extra_props = ['-Dorg.graalvm.launcher.executablename="{}"'.format(exe)]
main_class = self.subject.suite.toolchain._tool_to_main(tool)
# add jvm args from dependencies
jvm_args = [_quote(arg) for arg in mx.get_runtime_jvm_args(classpath_deps)]
# add properties from the project
if hasattr(self.subject, "getJavaProperties"):
for key, value in sorted(self.subject.getJavaProperties().items()):
jvm_args.append("-D" + key + "=" + value)
command = [java] + jvm_args + extra_props + [main_class, all_params]
# create script
if mx.is_windows():
return "@echo off\n" + " ".join(command) + "\n"
else:
return "#!/usr/bin/env bash\n" + "exec " + " ".join(command) + "\n"
def _unittest_config_participant(config):
vmArgs, mainClass, mainClassArgs = config
jdk = mx.get_jdk(tag='default')
runtime_args = _get_runtime_jvm_args(jdk)
# Remove the cp argument from the runtime args
cp = None
for i, cp in enumerate(runtime_args[:]):
if cp == "-cp":
cp = runtime_args[i + 1]
runtime_args.remove("-cp")
runtime_args.remove(cp)
break
# Attach remaining runtime args
vmArgs += runtime_args
# Merge the classpaths
if cp:
for i, arg in enumerate(vmArgs):
if arg == "-cp":
vmArgs[i + 1] += ":" + cp
config = (vmArgs, mainClass, mainClassArgs)
return config
def _create_jdk_bundle(jdkBuildDir):
"""
Creates a tar.gz JDK archive, an accompanying tar.gz.sha1 file with its
SHA1 signature plus symlinks to the archive for non-canonical architecture names.
"""
jdkImageDir = join(jdkBuildDir, 'images', 'jdk')
arches = _get_jdk_bundle_arches()
jdkTgzPath = join(_suite.get_output_root(), 'jdk-bundles',
'jdk9-{}-{}.tar.gz'.format(_get_openjdk_os(), arches[0]))
with mx.Archiver(jdkTgzPath, kind='tgz') as arc:
mx.log('Creating ' + jdkTgzPath)
for root, _, filenames in os.walk(jdkImageDir):
for name in filenames:
f = join(root, name)
arcname = 'jdk1.9.0/' + os.path.relpath(f, jdkImageDir)
arc.zf.add(name=f, arcname=arcname, recursive=False)
# The OpenJDK build creates an empty cacerts file so grab one from
# the default JDK which is assumed to be an OracleJDK
cacerts = join(
mx.get_jdk(tag='default').home, 'jre', 'lib', 'security',
'cacerts')
arc.zf.add(name=cacerts, arcname='jdk1.9.0/lib/security/cacerts')
with open(jdkTgzPath + '.sha1', 'w') as fp:
mx.log('Creating ' + jdkTgzPath + '.sha1')
fp.write(mx.sha1OfFile(jdkTgzPath))
def _create_link(source, link_name):
if exists(link_name):
os.remove(link_name)
mx.log('Creating ' + link_name + ' -> ' + source)
os.symlink(source, link_name)
for arch in arches[1:]:
link_name = join(_suite.get_output_root(), 'jdk-bundles',
'jdk9-{}-{}.tar.gz'.format(_get_openjdk_os(), arch))
jdkTgzName = os.path.basename(jdkTgzPath)
_create_link(jdkTgzName, link_name)
_create_link(jdkTgzName + '.sha1', link_name + '.sha1')
def createCommandLineArgs(self, benchmarks, bmSuiteArgs):
vmArgs = self.vmArgs(bmSuiteArgs)
dists = ["GRAALPYTHON", "TRUFFLE_NFI", "GRAALPYTHON-LAUNCHER"]
if mx.suite("tools", fatalIfMissing=False):
dists.extend(('CHROMEINSPECTOR', 'TRUFFLE_PROFILER'))
if mx.suite("sulong", fatalIfMissing=False):
dists.append('SULONG')
if mx.suite("sulong-managed", fatalIfMissing=False):
dists.append('SULONG_MANAGED')
vmArgs += [
"-Dorg.graalvm.language.python.home=%s" %
join(SUITE.dir, "graalpython"),
]
vmArgs += mx.get_runtime_jvm_args(
dists + ['com.oracle.graal.python.benchmarks'], jdk=mx.get_jdk())
jmh_entry = ["com.oracle.graal.python.benchmarks.interop.BenchRunner"]
runArgs = self.runArgs(bmSuiteArgs)
bench_name = benchmarks[0]
bench_args = self._benchmarks[bench_name]
return vmArgs + jmh_entry + runArgs + [bench_name] + bench_args