def suite_native_image_root(suite=None):
if not suite:
suite = svm_suite()
root_dir = join(svmbuild_dir(suite), 'native-image-root')
rev_file_name = join(root_dir, 'rev')
rev_value = suite.vc.parent(suite.vc_dir)
def write_rev_file():
mkpath(root_dir)
with open(rev_file_name, 'w') as rev_file:
rev_file.write(rev_value)
if exists(root_dir):
try:
with open(rev_file_name, 'r') as rev_file:
prev_rev_value = rev_file.readline()
except:
prev_rev_value = 'nothing'
if prev_rev_value != rev_value:
mx.warn('Rebuilding native-image-root as working directory revision changed from ' + prev_rev_value + ' to ' + rev_value)
remove_tree(root_dir)
layout_native_image_root(root_dir)
write_rev_file()
else:
layout_native_image_root(root_dir)
write_rev_file()
return root_dir
def microbench(self, args):
"""run JMH microbenchmark projects"""
parser = ArgumentParser(prog='mx microbench', description=microbench.__doc__,
usage="%(prog)s [command options|VM options] [-- [JMH options]]")
parser.add_argument('--jar', help='Explicitly specify micro-benchmark location')
self.add_arguments(parser)
mx.warn("`mx microbench` is deprecated! Consider moving to `mx_benchmark.JMHRunnerBenchmarkSuite`")
known_args, args = parser.parse_known_args(args)
vmArgs, jmhArgs = mx.extract_VM_args(args, useDoubleDash=True)
vmArgs = self.parseVmArgs(vmArgs)
# look for -f in JMH arguments
forking = True
for i in range(len(jmhArgs)):
arg = jmhArgs[i]
if arg.startswith('-f'):
if arg == '-f' and (i+1) < len(jmhArgs):
arg += jmhArgs[i+1]
try:
if int(arg[2:]) == 0:
forking = False
except ValueError:
pass
if known_args.jar:
# use the specified jar
args = ['-jar', known_args.jar]
if not forking:
args += vmArgs
# we do not know the compliance level of the jar - assuming 1.8
self.javaCompliance = mx.JavaCompliance('1.8')
else:
# find all projects with a direct JMH dependency
projects_dict = mx_benchmark.JMHRunnerBenchmarkSuite.get_jmh_projects_dict()
jmhProjects = projects_dict['JMH'] if 'JMH' in projects_dict else []
# get java compliance - 1.8 is minimum since we build jmh-runner with java 8
self.javaCompliance = max([p.javaCompliance for p in jmhProjects] + [mx.JavaCompliance('1.8')])
cpArgs = mx.get_runtime_jvm_args([p.name for p in jmhProjects], jdk=mx.get_jdk(self.javaCompliance))
# execute JMH runner
if forking:
args, cpVmArgs = self.filterVmArgs(cpArgs)
vmArgs += cpVmArgs
else:
args = cpArgs + vmArgs
args += ['org.openjdk.jmh.Main']
if forking:
def quoteSpace(s):
if " " in s:
return '"' + s + '"'
return s
forkedVmArgs = map(quoteSpace, self.parseForkedVmArgs(vmArgs))
args += ['--jvmArgsPrepend', ' '.join(forkedVmArgs)]
self.run_java(args + jmhArgs)
def repairDatapointsAndFail(self, benchmarks, bmSuiteArgs, partialResults, message):
try:
super(BaseDaCapoBenchmarkSuite, self).repairDatapointsAndFail(benchmarks, bmSuiteArgs, partialResults, message)
finally:
if self.workdir:
# keep old workdir for investigation, create a new one for further benchmarking
mx.warn("Keeping scratch directory after failed benchmark: {0}".format(self.workdir))
self._create_tmp_workdir()
def getVariants(self):
if not hasattr(self, '_variants'):
self._variants = []
for v in self.variants:
if 'gcc' in v and not SulongTestSuite.haveDragonegg():
mx.warn('Could not find dragonegg, not building test variant "%s"' % v)
continue
self._variants.append(v)
return self._variants
def before(self, bmSuiteArgs):
parser = mx_benchmark.parsers["dacapo_benchmark_suite"].parser
bmArgs, _ = parser.parse_known_args(bmSuiteArgs)
self.keepScratchDir = bmArgs.keep_scratch
if not bmArgs.no_scratch:
self._create_tmp_workdir()
else:
mx.warn("NO scratch directory created! (--no-scratch)")
self.workdir = None
def run_vm(*positionalargs, **kwargs):
"""run a Java program by executing the java executable in a Graal JDK"""
# convert positional args to a list so the first element can be updated
positionalargs = list(positionalargs)
args = positionalargs[0]
if '-G:+PrintFlags' in args and '-Xcomp' not in args:
mx.warn('Using -G:+PrintFlags may have no effect without -Xcomp as Graal initialization is lazy')
positionalargs[0] = _buildGOptionsArgs(args)
return _jvmci_run_vm(*positionalargs, **kwargs)
def _parseVmArgs(jdk, args, addDefaultArgs=True):
args = mx.expand_project_in_args(args, insitu=False)
jacocoArgs = mx_gate.get_jacoco_agent_args()
if jacocoArgs:
args = jacocoArgs + args
# Support for -G: options
def translateGOption(arg):
if arg.startswith("-G:+"):
if "=" in arg:
mx.abort("Mixing + and = in -G: option specification: " + arg)
arg = "-Dgraal." + arg[len("-G:+") :] + "=true"
elif arg.startswith("-G:-"):
if "=" in arg:
mx.abort("Mixing - and = in -G: option specification: " + arg)
arg = "-Dgraal." + arg[len("-G:+") :] + "=false"
elif arg.startswith("-G:"):
if "=" not in arg:
mx.abort('Missing "=" in non-boolean -G: option specification: ' + arg)
arg = "-Dgraal." + arg[len("-G:") :]
return arg
args = map(translateGOption, args)
if "-G:+PrintFlags" in args and "-Xcomp" not in args:
mx.warn("Using -G:+PrintFlags may have no effect without -Xcomp as Graal initialization is lazy")
bcp = [mx.distribution("truffle:TRUFFLE_API").classpath_repr()]
if _jvmciModes[_vm.jvmciMode]:
bcp.extend([d.get_classpath_repr() for d in _bootClasspathDists])
args = ["-Xbootclasspath/p:" + os.pathsep.join(bcp)] + args
# Remove JVMCI from class path. It's only there to support compilation.
cpIndex, cp = mx.find_classpath_arg(args)
if cp:
jvmciLib = mx.library("JVMCI").path
cp = os.pathsep.join([e for e in cp.split(os.pathsep) if e != jvmciLib])
args[cpIndex] = cp
# Set the default JVMCI compiler
jvmciCompiler = _compilers[-1]
args = ["-Djvmci.compiler=" + jvmciCompiler] + args
if "-version" in args:
ignoredArgs = args[args.index("-version") + 1 :]
if len(ignoredArgs) > 0:
mx.log(
"Warning: The following options will be ignored by the vm because they come after the '-version' argument: "
+ " ".join(ignoredArgs)
)
return jdk.processArgs(args, addDefaultArgs=addDefaultArgs)
def deploy_binary_if_master(args):
"""If the active branch is 'master', deploy binaries for the primary suite to remote maven repository."""
primary_branch = 'master'
_, vc_root = mx.VC.get_vc_root(_suite.dir)
active_branch = mx.VC.get_vc(vc_root).active_branch(_suite.dir)
deploy_binary = mx.command_function('deploy-binary')
if active_branch == primary_branch:
return deploy_binary(args)
else:
mx.warn(
'The active branch is "%s". Binaries are deployed only if the active branch is "%s".'
% (active_branch, primary_branch))
return 0
def _check_bootstrap_config(args):
"""
Issues a warning if `args` denote -XX:+BootstrapJVMCI but -XX:-UseJVMCICompiler.
"""
bootstrap = False
useJVMCICompiler = False
for arg in args:
if arg == '-XX:+BootstrapJVMCI':
bootstrap = True
elif arg == '-XX:+UseJVMCICompiler':
useJVMCICompiler = True
if bootstrap and not useJVMCICompiler:
mx.warn('-XX:+BootstrapJVMCI is ignored since -XX:+UseJVMCICompiler is not enabled')
def _check_bootstrap_config(args):
"""
Issues a warning if `args` denote -XX:+BootstrapJVMCI but -XX:-UseJVMCICompiler.
"""
bootstrap = False
useJVMCICompiler = False
for arg in args:
if arg == '-XX:+BootstrapJVMCI':
bootstrap = True
elif arg == '-XX:+UseJVMCICompiler':
useJVMCICompiler = True
if bootstrap and not useJVMCICompiler:
mx.warn('-XX:+BootstrapJVMCI is ignored since -XX:+UseJVMCICompiler is not enabled')
def native_image_on_jvm(args, **kwargs):
save_args = []
for arg in args:
if arg == '--no-server' or arg.startswith('--server'):
mx.warn('Ignoring server-mode native-image argument ' + arg)
else:
save_args.append(arg)
driver_cp = [join(suite_native_image_root(), 'lib', subdir, '*.jar') for subdir in ['boot', 'jvmci', 'graalvm']]
driver_cp += [join(suite_native_image_root(), 'lib', 'svm', tail) for tail in ['*.jar', join('builder', '*.jar')]]
driver_cp = list(itertools.chain.from_iterable(glob.glob(cp) for cp in driver_cp))
run_java(['-Dnative-image.root=' + suite_native_image_root(), '-cp', ":".join(driver_cp),
mx.dependency('substratevm:SVM_DRIVER').mainClass] + save_args, **kwargs)
def setupNodeEnvironment(args, add_graal_vm_args=True):
args = args if args else []
mode, vmArgs, progArgs = _parseArgs(args)
setLibraryPath()
if mx.suite('vm', fatalIfMissing=False) is not None and mx.suite(
'substratevm', fatalIfMissing=False) is not None:
_prepare_svm_env()
return mode, vmArgs, progArgs
if mx.suite('vm', fatalIfMissing=False) is not None or mx.suite(
'substratevm', fatalIfMissing=False) is not None:
mx.warn(
"Running on the JVM.\nIf you want to run on SubstrateVM, you need to dynamically import both '/substratevm' and '/vm'.\nExample: 'mx --env svm node'"
)
_setEnvVar('JAVA_HOME', _java_home())
if mx.suite('compiler', fatalIfMissing=False) is None:
_setEnvVar('GRAAL_SDK_JAR_PATH', mx.distribution('sdk:GRAAL_SDK').path)
_setEnvVar('LAUNCHER_COMMON_JAR_PATH',
mx.distribution('sdk:LAUNCHER_COMMON').path)
_setEnvVar('TRUFFLENODE_JAR_PATH', mx.distribution('TRUFFLENODE').path)
node_jvm_cp = (os.environ['NODE_JVM_CLASSPATH'] +
os.pathsep) if 'NODE_JVM_CLASSPATH' in os.environ else ''
node_cp = node_jvm_cp + mx.classpath(['TRUFFLENODE'] + ([
'tools:CHROMEINSPECTOR', 'tools:TRUFFLE_PROFILER', 'tools:AGENTSCRIPT'
] if mx.suite('tools', fatalIfMissing=False) is not None else []))
_setEnvVar('NODE_JVM_CLASSPATH', node_cp)
prevPATH = os.environ['PATH']
_setEnvVar('PATH',
"%s:%s" % (join(_suite.mxDir, 'fake_launchers'), prevPATH))
if _has_jvmci() and add_graal_vm_args:
if mx.suite('graal-enterprise', fatalIfMissing=False):
# explicitly require the enterprise compiler configuration
vmArgs += ['-Dgraal.CompilerConfiguration=enterprise']
if mx.suite('compiler', fatalIfMissing=False):
vmArgs += [
'-Djvmci.Compiler=graal', '-XX:+UnlockExperimentalVMOptions',
'-XX:+EnableJVMCI'
]
if isinstance(_suite, BinarySuite):
mx.logv('%s is a binary suite' % _suite.name)
tarfilepath = mx.distribution('TRUFFLENODE_GRAALVM_SUPPORT').path
with tarfile.open(tarfilepath, 'r:') as tar:
mx.logv('Extracting {} to {}'.format(tarfilepath, _suite.dir))
tar.extractall(_suite.dir)
return mode, vmArgs, progArgs
def checkLinks(javadocDir):
href = re.compile('(?<=href=").*?(?=")')
filesToCheck = {}
for root, _, files in os.walk(javadocDir):
for f in files:
if f.endswith('.html'):
html = os.path.join(root, f)
content = open(html, 'r').read()
for url in href.findall(content):
full = urljoin(html, url)
sectionIndex = full.find('#')
questionIndex = full.find('?')
minIndex = sectionIndex
if minIndex < 0:
minIndex = len(full)
if questionIndex >= 0 and questionIndex < minIndex:
minIndex = questionIndex
path = full[0:minIndex]
sectionNames = filesToCheck.get(path, [])
if sectionIndex >= 0:
s = full[sectionIndex + 1:]
sectionNames = sectionNames + [(html, s)]
else:
sectionNames = sectionNames + [(html, None)]
filesToCheck[path] = sectionNames
err = False
for referencedfile, sections in filesToCheck.items():
if referencedfile.startswith(
'javascript:') or referencedfile.startswith(
'http:') or referencedfile.startswith(
'https:') or referencedfile.startswith('mailto:'):
continue
if not exists(referencedfile):
mx.warn('Referenced file ' + referencedfile +
' does not exist. Referenced from ' + sections[0][0])
err = True
else:
content = open(referencedfile, 'r').read()
for path, s in sections:
if not s == None:
where = content.find('name="' + s + '"')
if where == -1:
mx.warn('There should be section ' + s + ' in ' +
referencedfile + ". Referenced from " + path)
err = True
if err:
mx.abort('There are wrong references in Javadoc')
def _parseVmArgs(jdk, args, addDefaultArgs=True):
args = mx.expand_project_in_args(args, insitu=False)
jacocoArgs = mx_gate.get_jacoco_agent_args()
if jacocoArgs:
args = jacocoArgs + args
# Support for -G: options
def translateGOption(arg):
if arg.startswith('-G:+'):
if '=' in arg:
mx.abort('Mixing + and = in -G: option specification: ' + arg)
arg = '-Dgraal.' + arg[len('-G:+'):] + '=true'
elif arg.startswith('-G:-'):
if '=' in arg:
mx.abort('Mixing - and = in -G: option specification: ' + arg)
arg = '-Dgraal.' + arg[len('-G:+'):] + '=false'
elif arg.startswith('-G:'):
if '=' not in arg:
mx.abort('Missing "=" in non-boolean -G: option specification: ' + arg)
arg = '-Dgraal.' + arg[len('-G:'):]
return arg
# add default graal.options.file and translate -G: options
options_file = join(mx.primary_suite().dir, 'graal.options')
options_file_arg = ['-Dgraal.options.file=' + options_file] if exists(options_file) else []
args = options_file_arg + map(translateGOption, args)
if '-G:+PrintFlags' in args and '-Xcomp' not in args:
mx.warn('Using -G:+PrintFlags may have no effect without -Xcomp as Graal initialization is lazy')
bcp = [mx.distribution('truffle:TRUFFLE_API').classpath_repr()]
if _jvmciModes[_vm.jvmciMode]:
bcp.extend([d.get_classpath_repr() for d in _bootClasspathDists])
args = ['-Xbootclasspath/p:' + os.pathsep.join(bcp)] + args
# Remove JVMCI from class path. It's only there to support compilation.
cpIndex, cp = mx.find_classpath_arg(args)
if cp:
jvmciLib = mx.library('JVMCI').path
cp = os.pathsep.join([e for e in cp.split(os.pathsep) if e != jvmciLib])
args[cpIndex] = cp
# Set the default JVMCI compiler
jvmciCompiler = _compilers[-1]
args = ['-Djvmci.Compiler=' + jvmciCompiler] + args
if '-version' in args:
ignoredArgs = args[args.index('-version') + 1:]
if len(ignoredArgs) > 0:
mx.log("Warning: The following options will be ignored by the vm because they come after the '-version' argument: " + ' '.join(ignoredArgs))
return jdk.processArgs(args, addDefaultArgs=addDefaultArgs)
def _write_cached_testclasses(cachesDir, jar, testclasses):
"""
Writes `testclasses` to a cache file specific to `jar`.
:param str cachesDir: directory containing files with cached test lists
:param list testclasses: a list of test class names
"""
cache = join(cachesDir, basename(jar) + '.testclasses')
try:
with open(cache, 'w') as fp:
for classname in testclasses:
print >> fp, classname
except IOError as e:
mx.warn('Error writing to ' + cache + ': ' + str(e))
def _write_cached_testclasses(cachesDir, jar, testclasses):
"""
Writes `testclasses` to a cache file specific to `jar`.
:param str cachesDir: directory containing files with cached test lists
:param list testclasses: a list of test class names
"""
cache = join(cachesDir, basename(jar) + '.testclasses')
try:
with open(cache, 'w') as fp:
for classname in testclasses:
print >> fp, classname
except IOError as e:
mx.warn('Error writing to ' + cache + ': ' + str(e))
def _get_bear_version():
global _bear_version
if _bear_version == '<uninitialized>':
try:
output = mx._check_output_str(['bear', '--version'], stderr=subprocess.STDOUT)
except OSError:
output = ''
m = _bear_version_regex.search(output)
if m:
_bear_version = int(m.group(1))
else:
mx.warn("Could not find bear, will not produce compilation database for make projects.")
_bear_version = None
return _bear_version
def cran_pkg_tests(list_file, graalvm_home=None):
cache_args = []
cache = os.environ.get('FASTR_PKGS_CACHE_OPT')
if cache is None:
mx.warn(
"If you want to use R packages cache, export environment variable FASTR_PKGS_CACHE_OPT. See option '--cache-pkgs' of 'mx pkgtest' for the syntax."
)
else:
cache_args += ['--cache-pkgs', cache]
result = pkgtest(["--verbose"] + cache_args +
common_pkg_tests_args(graalvm_home) +
["--pkg-filelist", list_file])
if result != 0:
mx.abort("package test failed")
def _choose_jdk_binary(jdk_binaries, quiet=False):
if quiet:
return _get_jdk_binary_or_abort(jdk_binaries, _DEFAULT_JDK_ID)
index = 1
default_choice = 1
choices = sorted(jdk_binaries.items())
for jdk_id, jdk_binary in choices:
if jdk_id == _DEFAULT_JDK_ID:
default_choice = index
default = "*"
else:
default = " "
prefix = '[{index}]{default}'.format(index=index, default=default)
print("{prefix:5} {jdk_id} | {version}".format(
prefix=prefix,
jdk_id=jdk_id.ljust(25),
version=jdk_binary._version))
index += 1
print('{:5} Other version'.format('[{}]'.format(index)))
while True:
try:
try:
choice = input("Select JDK> ")
except SyntaxError: # Empty line
choice = ""
if choice == "":
index = default_choice - 1
else:
index = int(choice) - 1
if index < 0:
raise IndexError(choice)
if index == len(choices):
choice = input("Select base JDK (1 .. {})> ".format(index))
base_index = int(choice) - 1
if base_index < 0 or base_index >= index:
raise IndexError(choice)
base_jdk = choices[base_index][1]
version = input("Enter version [{}]> ".format(
base_jdk._version))
if version == "":
version = base_jdk._version
return base_jdk.with_version(version)
return choices[index][1]
except (NameError, IndexError) as e:
mx.warn("Invalid selection: {}".format(e))
def native_image_on_jvm(args, **kwargs):
save_args = []
for arg in args:
if arg == '--no-server' or arg.startswith('--server'):
mx.warn('Ignoring server-mode native-image argument ' + arg)
else:
save_args.append(arg)
driver_cp = [join(suite_native_image_root(), 'lib', subdir, '*.jar') for subdir in ['boot', 'jvmci', 'graalvm']]
driver_cp += [join(suite_native_image_root(), 'lib', 'svm', tail) for tail in ['*.jar', join('builder', '*.jar')]]
driver_cp = list(itertools.chain.from_iterable(glob.glob(cp) for cp in driver_cp))
svmVersion = suite.release_version(snapshotSuffix='SNAPSHOT')
run_java(['-Dorg.graalvm.version=' + svmVersion, '-Dnative-image.root=' + suite_native_image_root(), '-cp', os.pathsep.join(driver_cp),
mx.dependency('substratevm:SVM_DRIVER').mainClass] + save_args, **kwargs)
def _is_batik_supported(jdk):
"""
Determines if Batik runs on the given jdk. Batik's JPEGRegistryEntry contains a reference
to TruncatedFileException, which is specific to the Sun/Oracle JDK. On a different JDK,
this results in a NoClassDefFoundError: com/sun/image/codec/jpeg/TruncatedFileException
"""
try:
subprocess.check_output(
[jdk.javap, 'com.sun.image.codec.jpeg.TruncatedFileException'])
return True
except subprocess.CalledProcessError:
mx.warn(
'Batik uses Sun internal class com.sun.image.codec.jpeg.TruncatedFileException which is not present in '
+ jdk.home)
return False
def extra_agent_run_arg(self, benchmark, args, image_run_args):
jmhOptions = self._extractJMHOptions(args)
for index, option in enumerate(jmhOptions):
argument = jmhOptions[index +
1] if index + 1 < len(jmhOptions) else None
if option == '-f' and argument != '0':
mx.warn(
f"JMH native images don't support -f with non-zero argument {argument}, ignoring it"
)
elif option.startswith('-jvmArgs'):
mx.warn(
f"JMH native images don't support option {option}, ignoring it"
)
return super(JMHJarBasedNativeImageBenchmarkMixin,
self).extra_agent_run_arg(benchmark, args, image_run_args)
def native_image_context(common_args=None,
hosted_assertions=True,
debug_gr_8964=False):
common_args = [] if common_args is None else common_args
base_args = ['-H:Path=' + svmbuild_dir()]
if debug_gr_8964:
base_args += ['-Ddebug_gr_8964=true']
if mx.get_opts().verbose:
base_args += ['--verbose']
if mx.get_opts().very_verbose:
base_args += ['--verbose-server']
if hosted_assertions:
base_args += native_image_context.hosted_assertions
native_image_cmd = native_image_path(suite_native_image_root())
if exists(native_image_cmd):
def _native_image(args, **kwargs):
mx.run([native_image_cmd] + args, **kwargs)
else:
mx.warn(
'native-image executable not found. Fallback to `mx native-image`')
_native_image = native_image_on_jvm
def query_native_image(all_args, option):
out = mx.LinesOutputCapture()
_native_image(['--dry-run'] + all_args, out=out)
for line in out.lines:
_, sep, after = line.partition(option)
if sep:
return after.split(' ')[0].rstrip()
return None
def native_image_func(args, debug_gr_8964=False):
all_args = base_args + common_args + args
path = query_native_image(all_args, '-H:Path=')
name = query_native_image(all_args, '-H:Name=')
image = join(path, name)
_native_image(all_args)
return image
try:
if exists(native_image_cmd):
_native_image(['--server-wipe'])
yield native_image_func
finally:
if exists(native_image_cmd):
_native_image(['--server-shutdown'])
def checkLinks(javadocDir):
href = re.compile('(?<=href=").*?(?=")')
filesToCheck = {}
for root, _, files in os.walk(javadocDir):
for f in files:
if f.endswith('.html'):
html = os.path.join(root, f)
content = open(html, 'r').read()
for url in href.findall(content):
full = urljoin(html, url)
sectionIndex = full.find('#')
questionIndex = full.find('?')
minIndex = sectionIndex
if minIndex < 0:
minIndex = len(full)
if questionIndex >= 0 and questionIndex < minIndex:
minIndex = questionIndex
path = full[0:minIndex]
sectionNames = filesToCheck.get(path, [])
if sectionIndex >= 0:
s = full[sectionIndex + 1:]
sectionNames = sectionNames + [(html, s)]
else:
sectionNames = sectionNames + [(html, None)]
filesToCheck[path] = sectionNames
err = False
for referencedfile, sections in filesToCheck.items():
if referencedfile.startswith('javascript:') or referencedfile.startswith('http:') or referencedfile.startswith('https:') or referencedfile.startswith('mailto:'):
continue
if not exists(referencedfile):
mx.warn('Referenced file ' + referencedfile + ' does not exist. Referenced from ' + sections[0][0])
err = True
else:
content = open(referencedfile, 'r').read()
for path, s in sections:
if not s == None:
whereName = content.find('name="' + s + '"')
whereId = content.find('id="' + s + '"')
if whereName == -1 and whereId == -1:
mx.warn('There should be section ' + s + ' in ' + referencedfile + ". Referenced from " + path)
err = True
if err:
mx.abort('There are wrong references in Javadoc')
def _read_cached_testclasses(cachesDir, jar):
"""
Reads the cached list of test classes in `jar`.
:param str cachesDir: directory containing files with cached test lists
:return: the cached list of test classes in `jar` or None if the cache doesn't
exist or is out of date
"""
cache = join(cachesDir, basename(jar) + '.testclasses')
if exists(cache) and mx.TimeStampFile(cache).isNewerThan(jar):
# Only use the cached result if the source jar is older than the cache file
try:
with open(cache) as fp:
return [line.strip() for line in fp.readlines()]
except IOError as e:
mx.warn('Error reading from ' + cache + ': ' + str(e))
return None
def checkGOption(arg):
if arg.startswith('-G:+'):
if '=' in arg:
mx.abort('Mixing + and = in -G: option specification: ' + arg)
translation = '-Dgraal.' + arg[len('-G:+'):] + '=true'
elif arg.startswith('-G:-'):
if '=' in arg:
mx.abort('Mixing - and = in -G: option specification: ' + arg)
translation = '-Dgraal.' + arg[len('-G:+'):] + '=false'
elif arg.startswith('-G:'):
if '=' not in arg:
mx.abort('Missing "=" in non-boolean -G: option specification: ' + arg)
translation = '-Dgraal.' + arg[len('-G:'):]
else:
return arg
mx.warn('Support for -G options is deprecated and will soon be removed. Replace "' + arg + '" with "' + translation + '"')
return translation
def _read_cached_testclasses(cachesDir, jar):
"""
Reads the cached list of test classes in `jar`.
:param str cachesDir: directory containing files with cached test lists
:return: the cached list of test classes in `jar` or None if the cache doesn't
exist or is out of date
"""
cache = join(cachesDir, basename(jar) + '.testclasses')
if exists(cache) and mx.TimeStampFile(cache).isNewerThan(jar):
# Only use the cached result if the source jar is older than the cache file
try:
with open(cache) as fp:
return [line.strip() for line in fp.readlines()]
except IOError as e:
mx.warn('Error reading from ' + cache + ': ' + str(e))
return None
def setupNodeEnvironment(args, add_graal_vm_args=True):
args = args if args else []
mode, vmArgs, progArgs = _parseArgs(args)
setLibraryPath()
if _is_windows:
processDevkitRoot()
if mx.suite('vm', fatalIfMissing=False) is not None and mx.suite(
'substratevm', fatalIfMissing=False) is not None:
_prepare_svm_env()
return mode, vmArgs, progArgs
if mx.suite('vm', fatalIfMissing=False) is not None or mx.suite(
'substratevm', fatalIfMissing=False) is not None:
mx.warn(
"Running on the JVM.\nIf you want to run on SubstrateVM, you need to dynamically import both '/substratevm' and '/vm'.\nExample: 'mx --env svm node'"
)
_setEnvVar('JAVA_HOME', _java_home())
# if mx.suite('compiler', fatalIfMissing=False) is None:
# _setEnvVar('GRAAL_SDK_JAR_PATH', mx.distribution('sdk:GRAAL_SDK').path)
_setEnvVar('LAUNCHER_COMMON_JAR_PATH',
mx.distribution('sdk:LAUNCHER_COMMON').path)
_setEnvVar('TRUFFLENODE_JAR_PATH', mx.distribution('TRUFFLENODE').path)
node_jvm_cp = (os.environ['NODE_JVM_CLASSPATH'] +
pathsep) if 'NODE_JVM_CLASSPATH' in os.environ else ''
node_cp = node_jvm_cp + mx.classpath(['TRUFFLENODE'] + (
['tools:CHROMEINSPECTOR', 'tools:TRUFFLE_PROFILER', 'tools:INSIGHT']
if mx.suite('tools', fatalIfMissing=False) is not None else []))
_setEnvVar('NODE_JVM_CLASSPATH', node_cp)
prevPATH = os.environ['PATH']
_setEnvVar(
'PATH',
"%s%s%s" % (join(_suite.mxDir, 'fake_launchers'), pathsep, prevPATH))
if isinstance(_suite, BinarySuite):
mx.logv('%s is a binary suite' % _suite.name)
tarfilepath = mx.distribution('TRUFFLENODE_GRAALVM_SUPPORT').path
with tarfile.open(tarfilepath, 'r:') as tar:
mx.logv('Extracting {} to {}'.format(tarfilepath, _suite.dir))
tar.extractall(_suite.dir)
return mode, vmArgs, progArgs
def _replace(self, m):
var = m.group(1)
if var in self._subst:
fn = self._subst[var]
if self._hasArg[var]:
arg = m.group(3)
return fn(arg)
else:
if m.group(3) is not None:
mx.warn('Ignoring argument in substitution ' + m.group(0))
if callable(fn):
return fn()
else:
return fn
elif self._chain is not None:
return self._chain._replace(m)
else:
mx.abort('Unknown substitution: ' + m.group(0))
def native_image_on_jvm(args, **kwargs):
save_args = []
for arg in args:
if arg == '--no-server' or arg.startswith('--server'):
mx.warn('Ignoring server-mode native-image argument ' + arg)
else:
save_args.append(arg)
_ensure_vm_built()
if mx.is_windows():
config = graalvm_jvm_configs[-1]
executable = vm_native_image_path(config)
else:
vm_link = join(svmbuild_dir(), 'vm')
executable = join(vm_link, 'bin', 'native-image')
if not exists(executable):
mx.abort("Can not find " + executable + "\nDid you forget to build? Try `mx build`")
mx.run([executable, '-H:CLibraryPath=' + clibrary_libpath()] + save_args, **kwargs)
def hsdis(args, copyToDir=None):
"""download the hsdis library
This is needed to support HotSpot's assembly dumping features.
By default it downloads the Intel syntax version, use the 'att' argument to install AT&T syntax."""
flavor = None
if mx.get_arch() == "amd64":
flavor = mx.get_env('HSDIS_SYNTAX')
if flavor is None:
flavor = 'intel'
if 'att' in args:
flavor = 'att'
libpattern = mx.add_lib_suffix('hsdis-' + mx.get_arch() + '-' + mx.get_os() + '-%s')
sha1s = {
'att/hsdis-amd64-windows-%s.dll' : 'bcbd535a9568b5075ab41e96205e26a2bac64f72',
'att/hsdis-amd64-linux-%s.so' : '36a0b8e30fc370727920cc089f104bfb9cd508a0',
'att/hsdis-amd64-darwin-%s.dylib' : 'c1865e9a58ca773fdc1c5eea0a4dfda213420ffb',
'intel/hsdis-amd64-windows-%s.dll' : '6a388372cdd5fe905c1a26ced614334e405d1f30',
'intel/hsdis-amd64-linux-%s.so' : '0d031013db9a80d6c88330c42c983fbfa7053193',
'intel/hsdis-amd64-darwin-%s.dylib' : '67f6d23cbebd8998450a88b5bef362171f66f11a',
'hsdis-sparcv9-solaris-%s.so': '970640a9af0bd63641f9063c11275b371a59ee60',
'hsdis-sparcv9-linux-%s.so': '0c375986d727651dee1819308fbbc0de4927d5d9',
}
if flavor:
flavoredLib = flavor + "/" + libpattern
else:
flavoredLib = libpattern
if flavoredLib not in sha1s:
mx.warn("hsdis with flavor '{}' not supported on this plattform or architecture".format(flavor))
return
sha1 = sha1s[flavoredLib]
lib = flavoredLib % (sha1)
path = join(_suite.get_output_root(), lib)
if not exists(path):
sha1path = path + '.sha1'
mx.download_file_with_sha1('hsdis', path, ['https://lafo.ssw.uni-linz.ac.at/pub/hsdis/' + lib], sha1, sha1path, True, True, sources=False)
if copyToDir is not None and exists(copyToDir):
destFileName = mx.add_lib_suffix('hsdis-' + mx.get_arch())
mx.logv('Copying {} to {}'.format(path, copyToDir + os.sep + destFileName))
shutil.copy(path, copyToDir + os.sep + destFileName)
def attribute_events(self, perf_data):
assert self.low_address is not None and self.high_address is not None
attributed = 0
unknown = 0
missing = 0
for event in perf_data.events:
if self.low_address <= event.pc < self.high_address:
if self.add_event(event):
attributed += 1
else:
missing += 1
elif event.symbol == '[Unknown]':
unknown += 1
if missing > 50:
# some versions of JVMTI leave out the stubs section of nmethod which occassionally gets ticks
# so a small number of missing ticks should be ignored.
mx.warn(
'{} events of {} could not be mapped to generated code'.format(
missing, attributed + missing))
def _read_cached_testclasses(cachesDir, jar, jdk):
"""
Reads the cached list of test classes in `jar`.
:param str cachesDir: directory containing files with cached test lists
:param JDKConfig jdk: the JDK for which the cached list of classes must be found
:return: the cached list of test classes in `jar` or None if the cache doesn't
exist or is out of date
"""
jdkVersion = '.jdk' + str(jdk.javaCompliance)
cache = join(cachesDir, basename(jar) + jdkVersion + '.testclasses')
if exists(cache) and mx.TimeStampFile(cache).isNewerThan(jar):
# Only use the cached result if the source jar is older than the cache file
try:
with open(cache) as fp:
return [line.strip() for line in fp.readlines()]
except IOError as e:
mx.warn('Error reading from ' + cache + ': ' + str(e))
return None
def gate_body(args, tasks):
with Task('Vm: GraalVM dist names', tasks, tags=['names']) as t:
if t:
mx_sdk_vm.verify_graalvm_configs(suites=['vm', 'vm-enterprise'])
with Task('Vm: Basic GraalVM Tests', tasks, tags=[VmGateTasks.compiler]) as t:
if t and mx_sdk_vm_impl.has_component('GraalVM compiler'):
# 1. the build must be a GraalVM
# 2. the build must be JVMCI-enabled since the 'GraalVM compiler' component is registered
mx_sdk_vm_impl.check_versions(mx_sdk_vm_impl.graalvm_output(), graalvm_version_regex=mx_sdk_vm_impl.graalvm_version_regex, expect_graalvm=True, check_jvmci=True)
if mx_sdk_vm_impl.has_component('LibGraal'):
libgraal_location = mx_sdk_vm_impl.get_native_image_locations('LibGraal', 'jvmcicompiler')
if libgraal_location is None:
mx.warn("Skipping libgraal tests: no library enabled in the LibGraal component")
else:
extra_vm_arguments = ['-XX:+UseJVMCICompiler', '-XX:+UseJVMCINativeLibrary', '-XX:JVMCILibPath=' + dirname(libgraal_location)]
if args.extra_vm_argument:
extra_vm_arguments += args.extra_vm_argument
# run avrora on the GraalVM binary itself
with Task('LibGraal Compiler:Basic', tasks, tags=[VmGateTasks.libgraal]) as t:
if t: _test_libgraal_basic(extra_vm_arguments)
with Task('LibGraal Compiler:FatalErrorHandling', tasks, tags=[VmGateTasks.libgraal]) as t:
if t: _test_libgraal_fatal_error_handling()
with Task('LibGraal Compiler:CTW', tasks, tags=[VmGateTasks.libgraal]) as t:
if t: _test_libgraal_ctw(extra_vm_arguments)
import mx_compiler
mx_compiler.compiler_gate_benchmark_runner(tasks, extra_vm_arguments, prefix='LibGraal Compiler:')
with Task('LibGraal Truffle:unittest', tasks, tags=[VmGateTasks.libgraal]) as t:
if t: _test_libgraal_truffle(extra_vm_arguments)
else:
mx.warn("Skipping libgraal tests: component not enabled")
gate_substratevm(tasks)
gate_sulong(tasks)
gate_python(tasks)
gate_svm_sl_tck(tasks)
gate_svm_truffle_tck_js(tasks)
def mx_post_parse_cmd_line(opts):
mx.addJDKFactory(_JVMCI_JDK_TAG, mx.JavaCompliance('9'), JVMCIJDKFactory())
mx.set_java_command_default_jdk_tag(_JVMCI_JDK_TAG)
jdkTag = mx.get_jdk_option().tag
jvmVariant = None
debugLevel = None
jvmciMode = None
if opts.jdk_jvm_variant is not None:
jvmVariant = opts.jdk_jvm_variant
if jdkTag and jdkTag != _JVMCI_JDK_TAG:
mx.warn('Ignoring "--jdk-jvm-variant" option as "--jdk" tag is not "' + _JVMCI_JDK_TAG + '"')
if opts.jdk_debug_level is not None:
debugLevel = _translateLegacyDebugLevel(opts.jdk_debug_level)
if jdkTag and jdkTag != _JVMCI_JDK_TAG:
mx.warn('Ignoring "--jdk-debug-level" option as "--jdk" tag is not "' + _JVMCI_JDK_TAG + '"')
if opts.jvmci_mode is not None:
jvmciMode = opts.jvmci_mode
if jdkTag and jdkTag != _JVMCI_JDK_TAG:
mx.warn('Ignoring "--jvmci-mode" option as "--jdk" tag is not "' + _JVMCI_JDK_TAG + '"')
_vm.update(jvmVariant, debugLevel, jvmciMode)
for jdkDist in jdkDeployedDists:
dist = jdkDist.dist()
if isinstance(jdkDist, JvmciJDKDeployedDist):
dist.set_archiveparticipant(JVMCIArchiveParticipant(dist))
def mx_post_parse_cmd_line(opts):
mx.set_java_command_default_jdk_tag(_JVMCI_JDK_TAG)
jdkTag = mx.get_jdk_option().tag
jvmVariant = None
debugLevel = None
jvmciMode = None
if opts.jdk_jvm_variant is not None:
jvmVariant = opts.jdk_jvm_variant
if jdkTag and jdkTag != _JVMCI_JDK_TAG:
mx.warn(
'Ignoring "--jdk-jvm-variant" option as "--jdk" tag is not "' +
_JVMCI_JDK_TAG + '"')
if opts.jdk_debug_level is not None:
debugLevel = _translateLegacyDebugLevel(opts.jdk_debug_level)
if jdkTag and jdkTag != _JVMCI_JDK_TAG:
mx.warn(
'Ignoring "--jdk-debug-level" option as "--jdk" tag is not "' +
_JVMCI_JDK_TAG + '"')
if opts.jvmci_mode is not None:
jvmciMode = opts.jvmci_mode
if jdkTag and jdkTag != _JVMCI_JDK_TAG:
mx.warn('Ignoring "--jvmci-mode" option as "--jdk" tag is not "' +
_JVMCI_JDK_TAG + '"')
_vm.update(jvmVariant, debugLevel, jvmciMode)
def mx_post_parse_cmd_line(opts):
mx.set_java_command_default_jdk_tag(_JVMCI_JDK_TAG)
jdkTag = mx.get_jdk_option().tag
jvmVariant = None
debugLevel = None
jvmciMode = None
if opts.jdk_jvm_variant is not None:
jvmVariant = opts.jdk_jvm_variant
if jdkTag and jdkTag != _JVMCI_JDK_TAG:
mx.warn('Ignoring "--jdk-jvm-variant" option as "--jdk" tag is not "' + _JVMCI_JDK_TAG + '"')
if opts.jdk_debug_level is not None:
debugLevel = _translateLegacyDebugLevel(opts.jdk_debug_level)
if jdkTag and jdkTag != _JVMCI_JDK_TAG:
mx.warn('Ignoring "--jdk-debug-level" option as "--jdk" tag is not "' + _JVMCI_JDK_TAG + '"')
if opts.jvmci_mode is not None:
jvmciMode = opts.jvmci_mode
if jdkTag and jdkTag != _JVMCI_JDK_TAG:
mx.warn('Ignoring "--jvmci-mode" option as "--jdk" tag is not "' + _JVMCI_JDK_TAG + '"')
_vm.update(jvmVariant, debugLevel, jvmciMode)
def jackpot(args, suite=None, nonZeroIsFatal=False):
"""run Jackpot 11.1 against non-test Java projects"""
jackpotHome = mx.get_env('JACKPOT_HOME', None)
if jackpotHome:
jackpotJar = join(jackpotHome, 'jackpot.jar')
else:
jackpotJar = mx.library('JACKPOT').get_path(resolve=True)
assert exists(jackpotJar)
if suite is None:
suite = mx.primary_suite()
nonTestProjects = [p for p in mx.projects() if _should_test_project(p)]
if not nonTestProjects:
return 0
groups = []
for p in nonTestProjects:
javacClasspath = []
deps = []
p.walk_deps(visit=lambda dep, edge: deps.append(dep) if dep.isLibrary() or dep.isJavaProject() else None)
annotationProcessorOnlyDeps = []
if len(p.annotation_processors()) > 0:
for apDep in p.annotation_processors():
if not apDep in deps:
deps.append(apDep)
annotationProcessorOnlyDeps.append(apDep)
for dep in deps:
if dep == p:
continue
if dep in annotationProcessorOnlyDeps:
continue
javacClasspath.append(dep.classpath_repr(resolve=True))
sourceLevel = min(p.javaCompliance.value, 9)
groups = groups + ['--group', "--classpath " + mx._separatedCygpathU2W(_escape_string(os.pathsep.join(javacClasspath))) + " --source " + str(sourceLevel) + " " + " ".join([_escape_string(d) for d in p.source_dirs()])]
cmd = ['--add-exports=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED', '--add-opens=java.base/java.net=ALL-UNNAMED', '--add-opens=java.desktop/sun.awt=ALL-UNNAMED']
cmd = cmd + ['-classpath', mx._cygpathU2W(jackpotJar), 'org.netbeans.modules.jackpot30.cmdline.Main']
jackCmd = ['--fail-on-warnings', '--progress'] + args + groups
jdk = mx.get_jdk(mx.JavaCompliance("11+"), cancel='cannot run Jackpot', purpose="run Jackpot")
if jdk is None:
mx.warn('Skipping Jackpot since JDK 11+ is not available')
return 0
else:
with tempfile.NamedTemporaryFile(mode='w', suffix='.jackpot') as f:
for c in jackCmd:
print(c, file=f)
f.flush()
ret = mx.run_java(cmd + ['@' + f.name], nonZeroIsFatal=nonZeroIsFatal, jdk=jdk)
if ret != 0:
mx.warn('To simulate the failure execute `mx -p {0} jackpot`.'.format(suite.dir))
mx.warn('To fix the error automatically try `mx -p {0} jackpot --apply`'.format(suite.dir))
return ret
def _write_cached_testclasses(cachesDir, jar, jdk, testclasses, excludedclasses):
"""
Writes `testclasses` to a cache file specific to `jar`.
:param str cachesDir: directory containing files with cached test lists
:param JDKConfig jdk: the JDK for which the cached list of classes must be written
:param list testclasses: a list of test class names
:param list excludedclasses: a list of excluded class names
"""
jdkVersion = '.jdk' + str(jdk.javaCompliance)
cache = join(cachesDir, basename(jar) + jdkVersion + '.testclasses')
exclusions = join(cachesDir, basename(jar) + jdkVersion + '.excludedclasses')
try:
with open(cache, 'w') as fp:
for classname in testclasses:
print(classname, file=fp)
with open(exclusions, 'w') as fp:
if excludedclasses:
mx.warn('Unsupported class files listed in ' + exclusions)
for classname in excludedclasses:
print(classname[1:], file=fp)
except IOError as e:
mx.warn('Error writing to ' + cache + ': ' + str(e))
def __add__(self, arcname, contents):
if arcname.startswith('META-INF/providers/'):
if self.isTest:
# The test distributions must not have their @ServiceProvider
# generated providers converted to real services otherwise
# bad things can happen such as InvocationPlugins being registered twice.
pass
else:
provider = arcname[len('META-INF/providers/'):]
for service in contents.strip().split(os.linesep):
assert service
self.services.setdefault(service, []).append(provider)
return True
elif arcname.endswith('_OptionDescriptors.class'):
if self.isTest:
mx.warn('@Option defined in test code will be ignored: ' + arcname)
else:
# Need to create service files for the providers of the
# jdk.vm.ci.options.Options service created by
# jdk.vm.ci.options.processor.OptionProcessor.
provider = arcname[:-len('.class'):].replace('/', '.')
self.services.setdefault('com.oracle.graal.options.OptionDescriptors', []).append(provider)
return False
def __add__(self, arcname, contents):
if arcname.startswith('META-INF/providers/'):
if self.isTest:
# The test distributions must not have their @ServiceProvider
# generated providers converted to real services otherwise
# bad things can happen such as InvocationPlugins being registered twice.
pass
else:
provider = arcname[len('META-INF/providers/'):]
for service in contents.strip().split(os.linesep):
assert service
self.services.setdefault(service, []).append(provider)
return True
elif arcname.endswith('_OptionDescriptors.class'):
if self.isTest:
mx.warn('@Option defined in test code will be ignored: ' + arcname)
else:
# Need to create service files for the providers of the
# jdk.internal.vm.ci.options.Options service created by
# jdk.internal.vm.ci.options.processor.OptionProcessor.
provider = arcname[:-len('.class'):].replace('/', '.')
self.services.setdefault('org.graalvm.compiler.options.OptionDescriptors', []).append(provider)
return False
def intellij_read_sdks():
sdks = dict()
if mx.is_linux() or mx.is_openbsd() or mx.is_sunos() or mx.is_windows():
xmlSdks = glob.glob(os.path.expanduser("~/.IdeaIC*/config/options/jdk.table.xml")) + \
glob.glob(os.path.expanduser("~/.IntelliJIdea*/config/options/jdk.table.xml")) + \
glob.glob(os.path.expanduser("~/.config/JetBrains/IdeaIC*/options/jdk.table.xml")) + \
glob.glob(os.path.expanduser("~/.config/JetBrains/IntelliJIdea*/options/jdk.table.xml"))
elif mx.is_darwin():
xmlSdks = \
glob.glob(os.path.expanduser("~/Library/Application Support/JetBrains/IdeaIC*/options/jdk.table.xml")) + \
glob.glob(os.path.expanduser("~/Library/Application Support/JetBrains/IntelliJIdea*/options/jdk.table.xml")) + \
glob.glob(os.path.expanduser("~/Library/Preferences/IdeaIC*/options/jdk.table.xml")) + \
glob.glob(os.path.expanduser("~/Library/Preferences/IntelliJIdea*/options/jdk.table.xml"))
else:
mx.warn("Location of IntelliJ SDK definitions on {} is unknown".format(mx.get_os()))
return sdks
if len(xmlSdks) == 0:
mx.warn("IntelliJ SDK definitions not found")
return sdks
verRE = re.compile(r'^.*[/\\]\.?(IntelliJIdea|IdeaIC)([^/\\]+)[/\\].*$')
def verSort(path):
match = verRE.match(path)
return match.group(2) + (".a" if match.group(1) == "IntellijIC" else ".b")
xmlSdks.sort(key=verSort)
xmlSdk = xmlSdks[-1] # Pick the most recent IntelliJ version, preferring Ultimate over Community edition.
mx.log("Using SDK definitions from {}".format(xmlSdk))
versionRegexes = {}
versionRegexes[intellij_java_sdk_type] = re.compile(r'^java\s+version\s+"([^"]+)"$|^version\s+(.+)$|^([\d._]+)$')
versionRegexes[intellij_python_sdk_type] = re.compile(r'^Python\s+(.+)$')
# Examples:
# truffleruby 19.2.0-dev-2b2a7f81, like ruby 2.6.2, Interpreted JVM [x86_64-linux]
# ver.2.2.4p230 ( revision 53155) p230
versionRegexes[intellij_ruby_sdk_type] = re.compile(r'^\D*(\d[^ ,]+)')
for sdk in etreeParse(xmlSdk).getroot().findall("component[@name='ProjectJdkTable']/jdk[@version='2']"):
name = sdk.find("name").get("value")
kind = sdk.find("type").get("value")
home = realpath(os.path.expanduser(sdk.find("homePath").get("value").replace('$USER_HOME$', '~')))
if home.find('$APPLICATION_HOME_DIR$') != -1:
# Don't know how to convert this into a real path so ignore it
continue
versionRE = versionRegexes.get(kind)
if not versionRE or sdk.find("version") is None:
# ignore unknown kinds
continue
match = versionRE.match(sdk.find("version").get("value"))
if match:
version = match.group(1)
sdks[home] = {'name': name, 'type': kind, 'version': version}
mx.logv("Found SDK {} with values {}".format(home, sdks[home]))
else:
mx.warn(u"Couldn't understand Java version specification \"{}\" for {} in {}".format(sdk.find("version").get("value"), home, xmlSdk))
return sdks
def choose_dist(quiet=False):
if quiet:
return JdkDistribution.by_name(JdkDistribution._DEFAULT_JDK)
index = 1
default_choice = 1
for dist in JdkDistribution._jdk_distributions:
if dist.get_name() == JdkDistribution._DEFAULT_JDK:
default_choice = index
default = "*"
else:
default = " "
print("[{index}]{default} {name} | {version}".format(
index=index,
name=dist.get_name().ljust(25),
version=dist.get_version(),
default=default))
index += 1
while True:
try:
try:
choice = input("Select JDK>")
except SyntaxError: # Empty line
choice = ""
if choice == "":
index = default_choice - 1
else:
index = int(choice) - 1
if index < 0:
raise IndexError
return JdkDistribution._jdk_distributions[index]
except (SyntaxError, NameError, IndexError):
mx.warn("Invalid selection!")
def _ninja_deps(cls): # pylint: disable=no-self-argument
deps = []
try:
subprocess.check_output(['ninja', '--version'], stderr=subprocess.STDOUT)
except OSError:
dep = mx.library('NINJA', False)
if dep:
deps.append(dep.qualifiedName())
Ninja.binary = mx.join(dep.get_path(False), 'ninja')
else:
# necessary until GR-13214 is resolved
mx.warn('Make `ninja` binary available via PATH to build native projects.')
try:
import ninja_syntax # pylint: disable=unused-variable, unused-import
except ImportError:
dep = mx.library('NINJA_SYNTAX')
deps.append(dep.qualifiedName())
module_path = mx.join(dep.get_path(False), 'ninja_syntax-{}'.format(dep.version))
mx.ensure_dir_exists(module_path) # otherwise, import machinery will ignore it
sys.path.append(module_path)
return deps
def extractArguments(cli_args):
vmArgs = []
rubyArgs = []
classpath = []
print_command = False
classic = False
main_class = "org.jruby.Main"
jruby_opts = os.environ.get('JRUBY_OPTS')
if jruby_opts:
jruby_opts = jruby_opts.split(' ')
for args in [jruby_opts, cli_args]:
while args:
arg = args.pop(0)
if arg == '-X+T':
pass # Just drop it
elif arg == '-X+TM':
main_class = "org.jruby.truffle.Main"
elif arg == '-Xclassic':
classic = True
elif arg == '-J-cmd':
print_command = True
elif arg.startswith('-J-G:+'):
rewritten = '-Dgraal.' + arg[6:] + '=true'
mx.warn(arg + ' is deprecated - use -J' + rewritten +
' instead')
vmArgs.append(rewritten)
elif arg.startswith('-J-G:-'):
rewritten = '-Dgraal.' + arg[6:] + '=false'
mx.warn(arg + ' is deprecated - use -J' + rewritten +
' instead')
vmArgs.append(rewritten)
elif arg.startswith('-J-G:'):
rewritten = '-Dgraal.' + arg[5:]
mx.warn(arg + ' is deprecated - use -J' + rewritten +
' instead')
vmArgs.append(rewritten)
elif arg == '-J-cp' or arg == '-J-classpath':
cp = args.pop(0)
if cp[:2] == '-J':
cp = cp[2:]
classpath.append(cp)
elif arg.startswith('-J-'):
vmArgs.append(arg[2:])
elif arg.startswith('-X+') or arg.startswith('-X-'):
rubyArgs.append(arg)
elif arg.startswith('-X'):
vmArgs.append('-Djruby.' + arg[2:])
else:
rubyArgs.append(arg)
rubyArgs.extend(args)
break
return vmArgs, rubyArgs, classpath, print_command, classic, main_class
def gate_body(args, tasks):
with Task('Vm: GraalVM dist names', tasks, tags=['names']) as t:
if t:
mx_sdk_vm.verify_graalvm_configs(suites=['vm', 'vm-enterprise'])
with Task('Vm: Basic GraalVM Tests', tasks, tags=[VmGateTasks.compiler]) as t:
if t and mx_sdk_vm_impl.has_component('GraalVM compiler'):
# 1. the build must be a GraalVM
# 2. the build must be JVMCI-enabled since the 'GraalVM compiler' component is registered
mx_sdk_vm_impl.check_versions(mx_sdk_vm_impl.graalvm_output(), graalvm_version_regex=mx_sdk_vm_impl.graalvm_version_regex, expect_graalvm=True, check_jvmci=True)
libgraal_suite_name = 'substratevm'
if mx.suite(libgraal_suite_name, fatalIfMissing=False) is not None:
import mx_substratevm
# Use `short_name` rather than `name` since the code that follows
# should be executed also when "LibGraal Enterprise" is registered
if mx_sdk_vm_impl.has_component(mx_substratevm.libgraal.short_name):
libgraal_location = mx_sdk_vm_impl.get_native_image_locations(mx_substratevm.libgraal.short_name, 'jvmcicompiler')
if libgraal_location is None:
mx.warn("Skipping libgraal tests: no library enabled in the LibGraal component")
else:
extra_vm_arguments = ['-XX:+UseJVMCICompiler', '-XX:+UseJVMCINativeLibrary', '-XX:JVMCILibPath=' + dirname(libgraal_location)]
if args.extra_vm_argument:
extra_vm_arguments += args.extra_vm_argument
# run avrora on the GraalVM binary itself
with Task('LibGraal Compiler:Basic', tasks, tags=[VmGateTasks.libgraal]) as t:
if t: _test_libgraal_basic(extra_vm_arguments)
with Task('LibGraal Compiler:FatalErrorHandling', tasks, tags=[VmGateTasks.libgraal]) as t:
if t: _test_libgraal_fatal_error_handling()
with Task('LibGraal Compiler:CTW', tasks, tags=[VmGateTasks.libgraal]) as t:
if t: _test_libgraal_ctw(extra_vm_arguments)
import mx_compiler
mx_compiler.compiler_gate_benchmark_runner(tasks, extra_vm_arguments, prefix='LibGraal Compiler:')
with Task('LibGraal Truffle:unittest', tasks, tags=[VmGateTasks.libgraal]) as t:
if t: _test_libgraal_truffle(extra_vm_arguments)
else:
mx.warn("Skipping libgraal tests: component not enabled")
else:
mx.warn("Skipping libgraal tests: suite '{suite}' not found. Did you forget to dynamically import it? (--dynamicimports {suite})".format(suite=libgraal_suite_name))
gate_substratevm(tasks)
gate_substratevm(tasks, quickbuild=True)
gate_sulong(tasks)
gate_python(tasks)
gate_svm_sl_tck(tasks)
gate_svm_truffle_tck_js(tasks)
def extractArguments(cli_args):
vmArgs = []
rubyArgs = []
classpath = []
print_command = False
jruby_opts = os.environ.get('JRUBY_OPTS')
if jruby_opts:
jruby_opts = jruby_opts.split(' ')
for args in [jruby_opts, cli_args]:
while args:
arg = args.pop(0)
if arg == '-X+T':
# ignore - default
pass
elif arg == '-Xclassic':
mx.error('-Xclassic no longer supported')
elif arg == '-J-cmd':
print_command = True
elif arg.startswith('-J-G:+'):
rewritten = '-Dgraal.'+arg[6:]+'=true'
mx.warn(arg + ' is deprecated - use -J' + rewritten + ' instead')
vmArgs.append(rewritten)
elif arg.startswith('-J-G:-'):
rewritten = '-Dgraal.'+arg[6:]+'=false'
mx.warn(arg + ' is deprecated - use -J' + rewritten + ' instead')
vmArgs.append(rewritten)
elif arg.startswith('-J-G:'):
rewritten = '-Dgraal.'+arg[5:]
mx.warn(arg + ' is deprecated - use -J' + rewritten + ' instead')
vmArgs.append(rewritten)
elif arg == '-J-cp' or arg == '-J-classpath':
cp = args.pop(0)
if cp[:2] == '-J':
cp = cp[2:]
classpath.append(cp)
elif arg.startswith('-J-'):
vmArgs.append(arg[2:])
elif arg.startswith('-X+') or arg.startswith('-X-') or arg.startswith('-Xlog='):
rubyArgs.append(arg)
elif arg.startswith('-X'):
vmArgs.append('-Djruby.'+arg[2:])
else:
rubyArgs.append(arg)
rubyArgs.extend(args)
break
return vmArgs, rubyArgs, classpath, print_command
def truffle_language_ensure(language_flag, version=None, native_image_root=None, early_exit=False, extract=True, debug_gr_8964=False):
"""
Ensures that we have a valid suite for the given language_flag, by downloading a binary if necessary
and providing the suite distribution artifacts in the native-image directory hierachy (via symlinks).
:param language_flag: native-image language_flag whose truffle-language we want to use
:param version: if not specified and no TRUFFLE_<LANG>_VERSION set latest binary deployed master revision gets downloaded
:param native_image_root: the native_image_root directory where the the artifacts get installed to
:return: language suite for the given language_flag
"""
if not native_image_root:
native_image_root = suite_native_image_root()
version_env_var = 'TRUFFLE_' + language_flag.upper() + '_VERSION'
if not version and os.environ.has_key(version_env_var):
version = os.environ[version_env_var]
if language_flag not in flag_suitename_map:
mx.abort('No truffle-language uses language_flag \'' + language_flag + '\'')
language_dir = join('languages', language_flag)
if early_exit and exists(join(native_image_root, language_dir)):
mx.logv('Early exit mode: Language subdir \'' + language_flag + '\' exists. Skip suite.import_suite.')
return None
language_entry = flag_suitename_map[language_flag]
language_suite_name = language_entry[0]
language_repo_name = language_entry[3] if len(language_entry) > 3 else None
urlinfos = [
mx.SuiteImportURLInfo(mx_urlrewrites.rewriteurl('https://curio.ssw.jku.at/nexus/content/repositories/snapshots'),
'binary',
mx.vc_system('binary'))
]
failure_warning = None
if not version and not mx.suite(language_suite_name, fatalIfMissing=False):
# If no specific version requested use binary import of last recently deployed master version
repo_suite_name = language_repo_name if language_repo_name else language_suite_name
repo_url = mx_urlrewrites.rewriteurl('https://github.com/graalvm/{0}.git'.format(repo_suite_name))
version = mx.SuiteImport.resolve_git_branchref(repo_url, 'binary', abortOnError=False)
if not version:
failure_warning = 'Resolving \'binary\' against ' + repo_url + ' failed'
language_suite = suite.import_suite(
language_suite_name,
version=version,
urlinfos=urlinfos,
kind=None,
in_subdir=bool(language_repo_name)
)
if not language_suite:
if failure_warning:
mx.warn(failure_warning)
mx.abort('Binary suite not found and no local copy of ' + language_suite_name + ' available.')
if not extract:
if not exists(join(native_image_root, language_dir)):
mx.abort('Language subdir \'' + language_flag + '\' should already exist with extract=False')
return language_suite
language_suite_depnames = language_entry[1]
language_deps = language_suite.dists + language_suite.libs
language_deps = [dep for dep in language_deps if dep.name in language_suite_depnames]
native_image_layout(language_deps, language_dir, native_image_root, debug_gr_8964=debug_gr_8964)
language_suite_nativedistnames = language_entry[2]
language_nativedists = [dist for dist in language_suite.dists if dist.name in language_suite_nativedistnames]
native_image_extract(language_nativedists, language_dir, native_image_root)
option_properties = join(language_suite.mxDir, 'native-image.properties')
target_path = remove_existing_symlink(join(native_image_root, language_dir, 'native-image.properties'))
if exists(option_properties):
if not exists(target_path):
mx.logv('Add symlink to ' + str(option_properties))
symlink_or_copy(option_properties, target_path, debug_gr_8964=debug_gr_8964)
else:
native_image_option_properties('languages', language_flag, native_image_root)
return language_suite
def junit(args, harness, parser=None, jdk_default=None):
"""run Junit tests"""
suppliedParser = parser is not None
parser = parser if suppliedParser else ArgumentParser(prog='mx junit')
parser.add_argument('--tests', action='store', help='pattern to match test classes')
parser.add_argument('--J', dest='vm_args', action='append', help='target VM arguments (e.g. --J @-dsa)', metavar='@<args>')
parser.add_argument('--jdk', action='store', help='jdk to use')
if suppliedParser:
parser.add_argument('remainder', nargs=REMAINDER, metavar='...')
args = parser.parse_args(args)
vmArgs = ['-ea', '-esa']
if args.vm_args:
vmArgs = vmArgs + mx_fastr.split_j_args(args.vm_args)
testfile = os.environ.get('MX_TESTFILE', None)
if testfile is None:
(_, testfile) = tempfile.mkstemp(".testclasses", "mx")
os.close(_)
candidates = []
if args.jdk:
jdk = mx.get_jdk(tag=args.jdk)
if not jdk:
mx.abort("jdk '" + args.jdk + "' not found")
else:
if not jdk_default:
jdk = mx.get_jdk()
else:
jdk = jdk_default
for p in mx.projects(opt_limit_to_suite=True):
if not p.isJavaProject() or jdk.javaCompliance < p.javaCompliance:
continue
candidates += _find_classes_with_annotations(p, None, ['@Test']).keys()
tests = [] if args.tests is None else [name for name in args.tests.split(',')]
classes = []
if len(tests) == 0:
classes = candidates
else:
for t in tests:
found = False
for c in candidates:
if t in c:
found = True
classes.append(c)
if not found:
mx.warn('no tests matched by substring "' + t + '"')
vmArgs += mx.get_runtime_jvm_args([pcp.name for pcp in mx.projects(opt_limit_to_suite=True) if pcp.isJavaProject() and pcp.javaCompliance <= jdk.javaCompliance], jdk=jdk)
if len(classes) != 0:
if len(classes) == 1:
testClassArgs = ['--testclass', classes[0]]
else:
with open(testfile, 'w') as f:
for c in classes:
f.write(c + '\n')
testClassArgs = ['--testsfile', testfile]
junitArgs = ['com.oracle.truffle.r.test.FastRJUnitWrapper'] + testClassArgs
rc = harness(args, vmArgs, jdk, junitArgs)
return rc
else:
return 0
def after(self, bmSuiteArgs):
if hasattr(self, "keepScratchDir") and self.keepScratchDir:
mx.warn("Scratch directory NOT deleted (--keep-scratch): {0}".format(self.workdir))
elif self.workdir:
rmtree(self.workdir)
def _jdk():
jdk = mx.get_jdk()
mx.warn('Assuming ' + str(jdk) + ' contains JVM executed by ' + name)
return _jdk
def stderr_collector(x):
stderrdata.append(x)
mx.warn(x.rstrip())
def _parseVmArgs(args, addDefaultArgs=True):
args = mx.expand_project_in_args(args, insitu=False)
argsPrefix = []
jacocoArgs = mx_gate.get_jacoco_agent_args()
if jacocoArgs:
argsPrefix.extend(jacocoArgs)
# Check for -G: options
def checkGOption(arg):
if arg.startswith('-G:+'):
if '=' in arg:
mx.abort('Mixing + and = in -G: option specification: ' + arg)
translation = '-Dgraal.' + arg[len('-G:+'):] + '=true'
elif arg.startswith('-G:-'):
if '=' in arg:
mx.abort('Mixing - and = in -G: option specification: ' + arg)
translation = '-Dgraal.' + arg[len('-G:+'):] + '=false'
elif arg.startswith('-G:'):
if '=' not in arg:
mx.abort('Missing "=" in non-boolean -G: option specification: ' + arg)
translation = '-Dgraal.' + arg[len('-G:'):]
else:
return arg
mx.warn('Support for -G options is deprecated and will soon be removed. Replace "' + arg + '" with "' + translation + '"')
return translation
# add default graal.options.file
options_file = join(mx.primary_suite().dir, 'graal.options')
if exists(options_file):
argsPrefix.append('-Dgraal.options.file=' + options_file)
args = [checkGOption(a) for a in args]
if '-Dgraal.PrintFlags=true' in args and '-Xcomp' not in args:
mx.warn('Using -Dgraal.PrintFlags=true may have no effect without -Xcomp as Graal initialization is lazy')
if isJDK8:
argsPrefix.append('-Djvmci.class.path.append=' + os.pathsep.join((e.get_path() for e in _jvmci_classpath)))
argsPrefix.append('-Xbootclasspath/a:' + os.pathsep.join([dep.classpath_repr() for dep in _bootclasspath_appends]))
else:
deployedDists = [entry.dist() for entry in _jvmci_classpath] + \
[e for e in _bootclasspath_appends if e.isJARDistribution()]
deployedModules = [as_java_module(dist, jdk) for dist in deployedDists]
# Set or update module path to include Graal and its dependencies as modules
graalModulepath = []
for deployedModule in deployedModules:
graalModulepath.extend([jmd.jarpath for jmd in deployedModule.modulepath if jmd.jarpath])
graalModulepath.append(deployedModule.jarpath)
graalModulepath = _uniqify(graalModulepath)
# Update added exports to include concealed JDK packages required by Graal
addedExports = {}
args = _extract_added_exports(args, addedExports)
for deployedModule in deployedModules:
for concealingModule, packages in deployedModule.concealedRequires.iteritems():
# No need to explicitly export JVMCI - it's exported via reflection
if concealingModule != 'jdk.vm.ci':
for package in packages:
addedExports.setdefault(concealingModule + '/' + package, set()).add(deployedModule.name)
for export, targets in addedExports.iteritems():
argsPrefix.append('-XaddExports:' + export + '=' + ','.join(sorted(targets)))
# Extend or set -modulepath argument
mpUpdated = False
for mpIndex in range(len(args)):
if args[mpIndex] in ['-modulepath', '-mp']:
assert mpIndex + 1 < len(args), 'VM option ' + args[mpIndex] + ' requires an argument'
args[mpIndex + 1] = os.pathsep.join(_uniqify(args[mpIndex + 1].split(os.pathsep) + graalModulepath))
mpUpdated = True
break
if not mpUpdated:
argsPrefix.append('-modulepath')
argsPrefix.append(os.pathsep.join(graalModulepath))
# Set the JVMCI compiler to Graal
argsPrefix.append('-Djvmci.Compiler=graal')
if '-version' in args:
ignoredArgs = args[args.index('-version') + 1:]
if len(ignoredArgs) > 0:
mx.log("Warning: The following options will be ignored by the VM because they come after the '-version' argument: " + ' '.join(ignoredArgs))
return jdk.processArgs(argsPrefix + args, addDefaultArgs=addDefaultArgs)