def clean(self, forBuild=False):
if exists(self.icuDir):
mx.rmtree(self.icuDir)
def gate_body(args, tasks):
with Task('Vm: GraalVM dist names', tasks, tags=['names']) as t:
if t:
mx_sdk_vm.verify_graalvm_configs(suites=['vm', 'vm-enterprise'])
with Task('Vm: Basic GraalVM Tests', tasks, tags=[VmGateTasks.compiler]) as t:
if t and mx_sdk_vm_impl.has_component('GraalVM compiler'):
# 1. the build must be a GraalVM
# 2. the build must be JVMCI-enabled since the 'GraalVM compiler' component is registered
mx_sdk_vm_impl.check_versions(mx_sdk_vm_impl.graalvm_output(), graalvm_version_regex=mx_sdk_vm_impl.graalvm_version_regex, expect_graalvm=True, check_jvmci=True)
if mx_sdk_vm_impl.has_component('LibGraal'):
libgraal_location = mx_sdk_vm_impl.get_native_image_locations('LibGraal', 'jvmcicompiler')
if libgraal_location is None:
mx.warn("Skipping libgraal tests: no library enabled in the LibGraal component")
else:
extra_vm_arguments = ['-XX:+UseJVMCICompiler', '-XX:+UseJVMCINativeLibrary', '-XX:JVMCILibPath=' + dirname(libgraal_location)]
if args.extra_vm_argument:
extra_vm_arguments += args.extra_vm_argument
import mx_compiler
# run avrora on the GraalVM binary itself
with Task('LibGraal Compiler:GraalVM DaCapo-avrora', tasks, tags=[VmGateTasks.libgraal]) as t:
if t:
java_exe = join(mx_sdk_vm_impl.graalvm_home(), 'bin', 'java')
mx.run([java_exe,
'-XX:+UseJVMCICompiler',
'-XX:+UseJVMCINativeLibrary',
'-jar', mx.library('DACAPO').get_path(True), 'avrora'])
# Ensure that fatal errors in libgraal route back to HotSpot
testdir = mkdtemp()
try:
cmd = [java_exe,
'-XX:+UseJVMCICompiler',
'-XX:+UseJVMCINativeLibrary',
'-Dlibgraal.CrashAt=length,hashCode',
'-Dlibgraal.CrashAtIsFatal=true',
'-jar', mx.library('DACAPO').get_path(True), 'avrora']
out = mx.OutputCapture()
exitcode = mx.run(cmd, cwd=testdir, nonZeroIsFatal=False, out=out)
if exitcode == 0:
if 'CrashAtIsFatal: no fatalError function pointer installed' in out.data:
# Executing a VM that does not configure fatal errors handling
# in libgraal to route back through the VM.
pass
else:
mx.abort('Expected following command to result in non-zero exit code: ' + ' '.join(cmd))
else:
hs_err = None
testdir_entries = listdir(testdir)
for name in testdir_entries:
if name.startswith('hs_err_pid') and name.endswith('.log'):
hs_err = join(testdir, name)
if hs_err is None:
mx.abort('Expected a file starting with "hs_err_pid" in test directory. Entries found=' + str(testdir_entries))
with open(join(testdir, hs_err)) as fp:
contents = fp.read()
if 'Fatal error in JVMCI' not in contents:
mx.abort('Expected "Fatal error in JVMCI" to be in contents of ' + hs_err + ':' + linesep + contents)
finally:
mx.rmtree(testdir)
with Task('LibGraal Compiler:CTW', tasks, tags=[VmGateTasks.libgraal]) as t:
if t:
mx_compiler.ctw([
'-DCompileTheWorld.Config=Inline=false CompilationFailureAction=ExitVM',
'-esa',
'-XX:+EnableJVMCI',
'-DCompileTheWorld.MultiThreaded=true',
'-Dgraal.InlineDuringParsing=false',
'-Dgraal.TrackNodeSourcePosition=true',
'-DCompileTheWorld.Verbose=false',
'-DCompileTheWorld.HugeMethodLimit=4000',
'-DCompileTheWorld.MaxCompiles=150000',
'-XX:ReservedCodeCacheSize=300m',
], extra_vm_arguments)
mx_compiler.compiler_gate_benchmark_runner(tasks, extra_vm_arguments, prefix='LibGraal Compiler:')
with Task('LibGraal Truffle:unittest', tasks, tags=[VmGateTasks.libgraal]) as t:
if t:
def _unittest_config_participant(config):
vmArgs, mainClass, mainClassArgs = config
def is_truffle_fallback(arg):
fallback_args = [
"-Dtruffle.TruffleRuntime=com.oracle.truffle.api.impl.DefaultTruffleRuntime",
"-Dgraalvm.ForcePolyglotInvalid=true"
]
return arg in fallback_args
newVmArgs = [arg for arg in vmArgs if not is_truffle_fallback(arg)]
return (newVmArgs, mainClass, mainClassArgs)
mx_unittest.add_config_participant(_unittest_config_participant)
excluded_tests = environ.get("TEST_LIBGRAAL_EXCLUDE")
if excluded_tests:
with NamedTemporaryFile(prefix='blacklist.', mode='w', delete=False) as fp:
fp.file.writelines([l + '\n' for l in excluded_tests.split()])
unittest_args = ["--blacklist", fp.name]
else:
unittest_args = []
unittest_args = unittest_args + ["--enable-timing", "--verbose"]
compiler_log_file = "graal-compiler.log"
mx_unittest.unittest(unittest_args + extra_vm_arguments + [
"-Dpolyglot.engine.AllowExperimentalOptions=true",
"-Dpolyglot.engine.CompileImmediately=true",
"-Dpolyglot.engine.BackgroundCompilation=false",
"-Dpolyglot.engine.TraceCompilation=true",
"-Dpolyglot.log.file={0}".format(compiler_log_file),
"-Dgraalvm.locatorDisabled=true",
"truffle"])
if exists(compiler_log_file):
remove(compiler_log_file)
else:
mx.warn("Skipping libgraal tests: component not enabled")
gate_substratevm(tasks)
gate_sulong(tasks)
gate_python(tasks)
gate_svm_sl_tck(tasks)
gate_svm_truffle_tck_js(tasks)
def clean(self, forBuild=False):
outputDir = self.subject.output_dir()
if not forBuild and os.path.exists(outputDir):
mx.rmtree(outputDir)
def clean(self, forBuild=False):
mx.rmtree(self.subject.out_dir, ignore_errors=True)
def clean(self, forBuild=False):
if exists(self.subject.get_output_root()):
mx.rmtree(self.subject.get_output_root())
def testdownstream(suite,
repoUrls,
relTargetSuiteDir,
mxCommands,
branch=None):
"""
Tests a downstream repo against the current working directory state of `suite`.
:param mx.Suite suite: the suite to test against the downstream repo
:param list repoUrls: URLs of downstream repos to clone, the first of which is the repo being tested
:param str relTargetSuiteDir: directory of the downstream suite to test relative to the top level
directory of the downstream repo being tested
:param list mxCommands: argument lists for the mx commands run in downstream suite being tested
:param str branch: name of branch to look for in downstream repo(s)
"""
assert len(repoUrls) > 0
repoUrls = [mx_urlrewrites.rewriteurl(url) for url in repoUrls]
workDir = join(suite.get_output_root(), 'testdownstream')
# A mirror of each suites in the same repo as `suite` is created via copying
rel_mirror = os.path.relpath(suite.dir,
mx.SuiteModel.siblings_dir(suite.dir))
in_subdir = os.sep in rel_mirror
suites_in_repo = [suite]
if in_subdir:
base = os.path.dirname(suite.dir)
for e in os.listdir(base):
candidate = join(base, e)
if candidate != suite.dir:
mxDir = mx._is_suite_dir(candidate)
if mxDir:
matches = [s for s in mx.suites() if s.dir == candidate]
if len(matches) == 0:
suites_in_repo.append(
mx.SourceSuite(mxDir, primary=False, load=False))
else:
suites_in_repo.append(matches[0])
for suite_in_repo in suites_in_repo:
if suite_in_repo.vc_dir and suite_in_repo.dir != suite_in_repo.vc_dir:
mirror = join(workDir, basename(suite_in_repo.vc_dir),
suite_in_repo.name)
else:
mirror = join(workDir, suite_in_repo.name)
if exists(mirror):
mx.rmtree(mirror)
output_root = mx._safe_path(suite_in_repo.get_output_root())
def ignore_output_root(d, names):
mx.log('Copying ' + d)
if d == os.path.dirname(output_root):
mx.log('Omitting ' + output_root)
return [os.path.basename(output_root)]
return []
mx.copytree(suite_in_repo.dir,
mirror,
ignore=ignore_output_root,
symlinks=True)
targetDir = None
for repoUrl in repoUrls:
# Deduce a target name from the target URL
url = _urllib_parse.urlparse(repoUrl)
targetName = url.path
if targetName.rfind('/') != -1:
targetName = targetName[targetName.rfind('/') + 1:]
if targetName.endswith('.git'):
targetName = targetName[0:-len('.git')]
repoWorkDir = join(workDir, targetName)
git = mx.GitConfig()
if exists(repoWorkDir):
git.pull(repoWorkDir)
else:
git.clone(repoUrl, repoWorkDir)
# See if there's a matching (non-master) branch and use it if there is
if not branch:
branch = git.git_command(
suite.dir, ['rev-parse', '--abbrev-ref', 'HEAD']).strip()
if branch != 'master':
git.git_command(repoWorkDir, ['checkout', branch],
abortOnError=False)
if not targetDir:
targetDir = repoWorkDir
assert not isabs(relTargetSuiteDir)
targetSuiteDir = join(targetDir, relTargetSuiteDir)
assert targetSuiteDir.startswith(targetDir)
mxpy = None if suite != mx._mx_suite else join(mirror, 'mx.py')
for command in mxCommands:
mx.logv('[running "mx ' + ' '.join(command) + '" in ' +
targetSuiteDir + ']')
mx.run_mx(command, targetSuiteDir, mxpy=mxpy)
def jlink_new_jdk(jdk, dst_jdk_dir, module_dists, ignore_dists,
root_module_names=None,
missing_export_target_action='create',
with_source=lambda x: True,
vendor_info=None,
dedup_legal_notices=True,
use_upgrade_module_path=False):
"""
Uses jlink from `jdk` to create a new JDK image in `dst_jdk_dir` with `module_dists` and
their dependencies added to the JDK image, replacing any existing modules of the same name.
:param JDKConfig jdk: source JDK
:param str dst_jdk_dir: path to use for the jlink --output option
:param list module_dists: list of distributions defining modules
:param list ignore_dists: list of distributions that should be ignored for missing_export_target_action
:param list root_module_names: list of strings naming the module root set for the new JDK image.
The named modules must either be in `module_dists` or in `jdk`. If None, then
the root set will be all the modules in ``module_dists` and `jdk`.
:param str missing_export_target_action: the action to perform for a qualified export target that
is not present in `module_dists` and does not have a hash stored in java.base.
The choices are:
"create" - an empty module is created
"error" - raise an error
None - do nothing
:param lambda with_source: returns True if the sources of a module distribution must be included in the new JDK
:param dict vendor_info: values for the jlink vendor options added by JDK-8232080
:param bool use_upgrade_module_path: if True, then instead of linking `module_dists` into the image, resolve
them via --upgrade-module-path at image runtime
:return bool: False if use_upgrade_module_path == True and the existing image is up to date otherwise True
"""
assert callable(with_source)
if jdk.javaCompliance < '9':
mx.abort('Cannot derive a new JDK from ' + jdk.home + ' with jlink since it is not JDK 9 or later')
exploded_java_base_module = join(jdk.home, 'modules', 'java.base')
if exists(exploded_java_base_module):
mx.abort('Cannot derive a new JDK from ' + jdk.home + ' since it appears to be a developer build with exploded modules')
jimage = join(jdk.home, 'lib', 'modules')
jmods_dir = join(jdk.home, 'jmods')
if not isfile(jimage):
mx.abort('Cannot derive a new JDK from ' + jdk.home + ' since ' + jimage + ' is missing or is not an ordinary file')
if not isdir(jmods_dir):
mx.abort('Cannot derive a new JDK from ' + jdk.home + ' since ' + jmods_dir + ' is missing or is not a directory')
# Exclude jdk.aot due to GR-10545 and JDK-8255616
jdk_modules = {jmd.name: jmd for jmd in jdk.get_modules() if jmd.name != 'jdk.aot'}
modules = [as_java_module(dist, jdk) for dist in module_dists]
module_names = frozenset((m.name for m in modules))
all_module_names = frozenset(list(jdk_modules.keys())) | module_names
# Read hashes stored in java.base (the only module in the JDK where hashes are stored)
hashes = _read_java_base_hashes(jdk)
build_dir = mx.ensure_dir_exists(join(dst_jdk_dir + ".build"))
# Directory under dst_jdk_dir for artifacts related to use_upgrade_module_path
upgrade_dir = join(dst_jdk_dir, 'upgrade_modules_support')
# Map from JavaModuleDescriptors to post-jlink jar location.
synthetic_modules = OrderedDict()
try:
ignore_module_names = set(mx_javamodules.get_module_name(mx.dependency(ignore_dist)) for ignore_dist in ignore_dists)
# Synthesize modules for targets of qualified exports that are not present in `modules`.
# Without this, runtime module resolution will fail due to missing modules.
target_requires = {}
for jmd in modules:
for targets in jmd.exports.values():
for target in targets:
if target not in all_module_names and target not in ignore_module_names and target not in hashes:
target_requires.setdefault(target, set()).add(jmd.name)
if target_requires and missing_export_target_action is not None:
if missing_export_target_action == 'error':
mx.abort('Target(s) of qualified exports cannot be resolved: ' + '.'.join(target_requires.keys()))
assert missing_export_target_action == 'create', 'invalid value for missing_export_target_action: ' + str(missing_export_target_action)
for name, requires in sorted(target_requires.items()):
module_jar = join(build_dir, name + '.jar')
jmd = JavaModuleDescriptor(name, {}, requires={module: [] for module in requires}, uses=set(), provides={}, jarpath=module_jar)
module_build_dir = mx.ensure_dir_exists(join(build_dir, name))
module_info = jmd.as_module_info()
module_info_java = join(module_build_dir, 'module-info.java')
module_info_class = join(module_build_dir, 'module-info.class')
dst_module_jar = join(upgrade_dir, name + '.jar')
synthetic_modules[jmd] = dst_module_jar
if use_upgrade_module_path and exists(dst_module_jar):
with ZipFile(dst_module_jar, 'r') as zf:
previous_module_info = zf.read('module-info.java').decode()
if previous_module_info == module_info:
mx.logv('[Reusing synthetic module {}]'.format(name))
os.rename(dst_module_jar, module_jar)
continue
mx.logv('[Rebuilding synthetic module {} as module descriptor changed]'.format(name))
with open(module_info_java, 'w') as fp:
fp.write(module_info)
mx.run([jdk.javac, '-d', module_build_dir,
'--limit-modules=java.base,' + ','.join(jmd.requires.keys()),
'--module-path=' + os.pathsep.join((m.jarpath for m in modules)),
module_info_java])
with ZipFile(module_jar, 'w') as zf:
zf.write(module_info_java, 'module-info.java')
zf.write(module_info_class, 'module-info.class')
if exists(jmd.get_jmod_path()):
os.remove(jmd.get_jmod_path())
if not use_upgrade_module_path:
mx.run([jdk.javac.replace('javac', 'jmod'), 'create', '--class-path=' + module_build_dir, jmd.get_jmod_path()])
modules.extend(synthetic_modules.keys())
module_names = frozenset((m.name for m in modules))
all_module_names = frozenset(list(jdk_modules.keys())) | module_names
# Edit lib/security/default.policy in java.base
patched_java_base = _patch_default_security_policy(build_dir, jmods_dir, dst_jdk_dir)
# Now build the new JDK image with jlink
jlink = [jdk.javac.replace('javac', 'jlink')]
if jdk_enables_jvmci_by_default(jdk):
# On JDK 9+, +EnableJVMCI forces jdk.internal.vm.ci to be in the root set
jlink += ['-J-XX:-EnableJVMCI', '-J-XX:-UseJVMCICompiler']
jlink.append('--add-modules=' + ','.join(_get_image_root_modules(root_module_names, module_names, jdk_modules.keys(), use_upgrade_module_path)))
module_path = patched_java_base + os.pathsep + jmods_dir
if modules and not use_upgrade_module_path:
module_path = os.pathsep.join((m.get_jmod_path(respect_stripping=True) for m in modules)) + os.pathsep + module_path
jlink.append('--module-path=' + module_path)
jlink.append('--output=' + dst_jdk_dir)
# These options are derived from how OpenJDK runs jlink to produce the final runtime image.
jlink.extend(['-J-XX:+UseSerialGC', '-J-Xms32M', '-J-Xmx512M', '-J-XX:TieredStopAtLevel=1'])
jlink.append('-J-Dlink.debug=true')
if dedup_legal_notices:
jlink.append('--dedup-legal-notices=error-if-not-same-content')
jlink.append('--keep-packaged-modules=' + join(dst_jdk_dir, 'jmods'))
vm_options_path = join(upgrade_dir, 'vm_options')
vm_options = _get_image_vm_options(jdk, use_upgrade_module_path, modules, synthetic_modules)
if vm_options:
jlink.append('--add-options=' + ' '.join(vm_options))
if jdk_has_new_jlink_options(jdk) and vendor_info is not None:
for name, value in vendor_info.items():
jlink.append('--' + name + '=' + value)
release_file = join(jdk.home, 'release')
if isfile(release_file):
jlink.append('--release-info=' + release_file)
if exists(dst_jdk_dir):
if use_upgrade_module_path and _vm_options_match(vm_options, vm_options_path):
mx.logv('[Existing JDK image {} is up to date]'.format(dst_jdk_dir))
return False
mx.rmtree(dst_jdk_dir)
# TODO: investigate the options below used by OpenJDK to see if they should be used:
# --order-resources: specifies order of resources in generated lib/modules file.
# This is apparently not so important if a CDS archive is available.
# --generate-jli-classes: pre-generates a set of java.lang.invoke classes.
# See https://github.com/openjdk/jdk/blob/master/make/GenerateLinkOptData.gmk
mx.logv('[Creating JDK image in {}]'.format(dst_jdk_dir))
mx.run(jlink)
if use_upgrade_module_path:
# Move synthetic upgrade modules into final location
for jmd, jarpath in synthetic_modules.items():
mx.ensure_dir_exists(dirname(jarpath))
os.rename(jmd.jarpath, jarpath)
# Persist VM options cooked into image to be able to skip a subsequent
# jlink execution if the options do not change.
with open(vm_options_path, 'w') as fp:
fp.write(os.linesep.join(vm_options))
# Create src.zip in new JDK image
_copy_src_zip(jdk.home, dst_jdk_dir, modules, lambda jmd: not use_upgrade_module_path and with_source(jmd.dist))
mx.logv('[Copying static libraries]')
lib_directory = join(jdk.home, 'lib', 'static')
if exists(lib_directory):
dst_lib_directory = join(dst_jdk_dir, 'lib', 'static')
try:
mx.copytree(lib_directory, dst_lib_directory)
except shutil.Error as e:
# On AArch64, there can be a problem in the copystat part
# of copytree which occurs after file and directory copying
# has successfully completed. Since the metadata doesn't
# matter in this case, just ensure that the content was copied.
for root, _, lib_files in os.walk(lib_directory):
relative_root = os.path.relpath(root, dst_lib_directory)
for lib in lib_files:
src_lib_path = join(root, lib)
dst_lib_path = join(dst_lib_directory, relative_root, lib)
if not exists(dst_lib_path):
mx.abort('Error copying static libraries: {} missing in {}{}Original copytree error: {}'.format(
join(relative_root, lib), dst_lib_directory, os.linesep, e))
src_lib_hash = mx.sha1OfFile(src_lib_path)
dst_lib_hash = mx.sha1OfFile(dst_lib_path)
if src_lib_hash != dst_lib_hash:
mx.abort('Error copying static libraries: {} (hash={}) and {} (hash={}) differ{}Original copytree error: {}'.format(
src_lib_path, src_lib_hash,
dst_lib_path, dst_lib_hash,
os.linesep, e))
# Allow older JDK versions to work
else:
lib_prefix = mx.add_lib_prefix('')
lib_suffix = mx.add_static_lib_suffix('')
lib_directory = join(jdk.home, 'lib')
dst_lib_directory = join(dst_jdk_dir, 'lib')
for f in os.listdir(lib_directory):
if f.startswith(lib_prefix) and f.endswith(lib_suffix):
lib_path = join(lib_directory, f)
if isfile(lib_path):
shutil.copy2(lib_path, dst_lib_directory)
finally:
if not mx.get_opts().verbose:
# Preserve build directory so that javac command can be re-executed
# by cutting and pasting verbose output.
shutil.rmtree(build_dir)
if not use_upgrade_module_path:
# Create CDS archive (https://openjdk.java.net/jeps/341).
out = mx.OutputCapture()
mx.logv('[Creating CDS shared archive]')
if mx.run([mx.exe_suffix(join(dst_jdk_dir, 'bin', 'java')), '-Xshare:dump', '-Xmx128M', '-Xms128M'], out=out, err=out, nonZeroIsFatal=False) != 0:
mx.log(out.data)
mx.abort('Error generating CDS shared archive')
else:
# -Xshare is incompatible with --upgrade-module-path
pass
return True
def clean(self, forBuild=False):
for file in self.subject.getResults():
os.remove(file)
for path in [join(self.subject.dir, m) for m in ['dist', 'node_modules']]:
if exists(path):
mx.rmtree(path)
def testdownstream(suite,
repoUrls,
relTargetSuiteDir,
mxCommands,
branch=None):
"""
Tests a downstream repo against the current working directory state of `suite`.
:param mx.Suite suite: the suite to test against the downstream repo
:param list repoUrls: URLs of downstream repos to clone, the first of which is the repo being tested
:param str relTargetSuiteDir: directory of the downstream suite to test relative to the top level
directory of the downstream repo being tested
:param list mxCommands: argument lists for the mx commands run in downstream suite being tested
:param str branch: name(s) of branch to look for in downstream repo(s)
"""
assert len(repoUrls) > 0
repoUrls = [mx_urlrewrites.rewriteurl(url) for url in repoUrls]
workDir = join(suite.get_output_root(), 'testdownstream')
# A mirror of each suites in the same repo as `suite` is created via copying
rel_mirror = os.path.relpath(suite.dir,
mx.SuiteModel.siblings_dir(suite.dir))
in_subdir = os.sep in rel_mirror
suites_in_repo = [suite]
if in_subdir:
base = os.path.dirname(suite.dir)
for e in os.listdir(base):
candidate = join(base, e)
if candidate != suite.dir:
mxDir = mx._is_suite_dir(candidate)
if mxDir:
matches = [s for s in mx.suites() if s.dir == candidate]
if len(matches) == 0:
suites_in_repo.append(
mx.SourceSuite(mxDir, primary=False, load=False))
else:
suites_in_repo.append(matches[0])
if suite.vc:
vc_metadir = mx._safe_path(mx.VC.get_vc(suite.vc_dir).metadir())
blacklist = {suite.vc_dir: [join(suite.vc_dir, vc_metadir)]}
else:
blacklist = {}
for suite_in_repo in suites_in_repo:
output_root = mx._safe_path(suite_in_repo.get_output_root())
blacklist.setdefault(dirname(output_root), []).append(output_root)
def omitted_dirs(d, names):
mx.log('Copying ' + d)
to_omit = []
for blacklisted_dir in blacklist.get(d, []):
mx.log('Omitting ' + blacklisted_dir)
to_omit.append(basename(blacklisted_dir))
return to_omit
if suite.vc_dir and suite.dir != suite.vc_dir:
mirror = join(workDir, basename(suite.vc_dir))
else:
mirror = join(workDir, suite.name)
if exists(mirror):
mx.rmtree(mirror)
mx.copytree(suite.vc_dir, mirror, ignore=omitted_dirs, symlinks=True)
targetDir = None
for repoUrl in repoUrls:
# Deduce a target name from the target URL
url = _urllib_parse.urlparse(repoUrl)
targetName = url.path
if targetName.rfind('/') != -1:
targetName = targetName[targetName.rfind('/') + 1:]
if targetName.endswith('.git'):
targetName = targetName[0:-len('.git')]
repoWorkDir = join(workDir, targetName)
git = mx.GitConfig()
if exists(repoWorkDir):
git.pull(repoWorkDir)
else:
git.clone(repoUrl, repoWorkDir)
if branch is None:
branch = []
elif isinstance(branch, str):
branch = [branch]
else:
assert isinstance(branch, list)
# fall back to the branch of the main repo
active_branch = git.active_branch(suite.dir, abortOnError=False)
if active_branch:
branch.append(active_branch)
updated = False
for branch_name in branch:
if git.update_to_branch(repoWorkDir,
branch_name,
abortOnError=False):
updated = True
break
if not updated:
mx.warn('Could not update {} to any of the following branches: {}'.
format(repoWorkDir, ', '.join(branch)))
if not targetDir:
targetDir = repoWorkDir
assert not isabs(relTargetSuiteDir)
targetSuiteDir = join(targetDir, relTargetSuiteDir)
assert targetSuiteDir.startswith(targetDir)
mxpy = None if suite != mx._mx_suite else join(mirror, 'mx.py')
for command in mxCommands:
mx.logv('[running "mx ' + ' '.join(command) + '" in ' +
targetSuiteDir + ']')
mx.run_mx(command, targetSuiteDir, mxpy=mxpy)
def clean(self, forBuild=False):
_output_dir = join(_suite.dir, self.subject.outputDir)
if exists(_output_dir):
mx.rmtree(_output_dir)
def gc_dists(args):
""" Garbage collect mx distributions."""
parser = argparse.ArgumentParser(prog='mx gc-dists', description='''Garbage collect layout distributions.
By default, it collects all found layout distributions that are *not* part of the current configuration (see `--keep-current`).
This command respects mx level suite filtering (e.g., `mx --suite my-suite gc-dists`).
''', epilog='''If the environment variable `MX_GC_AFTER_BUILD` is set, %(prog)s will be executed after `mx build`
using the content of the environment variable as parameters.''')
# mutually exclusive groups do not support title and description - wrapping in another group as a workaround
action_group_desc = parser.add_argument_group('actions', 'What to do with the result. One of the following arguments is required.')
action_group = action_group_desc.add_mutually_exclusive_group(required=True)
action_group.add_argument('-f', '--force', action='store_true', help='remove layout distributions without further questions')
action_group.add_argument('-n', '--dry-run', action='store_true', help='show what would be removed without actually doing anything')
action_group.add_argument('-i', '--interactive', action='store_true', help='ask for every layout distributions whether it should be removed')
keep_current_group_desc = parser.add_argument_group('current configuration handling', description='How to deal with the current configuration, i.e., what `mx build` would rebuild.')
keep_current_group = keep_current_group_desc.add_mutually_exclusive_group()
keep_current_group.add_argument('--keep-current', action='store_true', default=True, help='keep layout distributions of the current configuration (default)')
keep_current_group.add_argument('--no-keep-current', action='store_false', dest='keep_current', help='remove layout distributions of the current configuration')
filter_group = parser.add_argument_group('result filters', description='Filter can be combined.')
filter_group.add_argument('--reverse', action='store_true', help='reverse the result')
filter_group.add_argument('--older-than', action=TimeAction, help='only show results older than the specified point in time (format: {})'.format(TimeAction.fmt.replace('%', '%%')))
try:
parsed_args = parser.parse_args(args)
except ValueError as ve:
parser.error(str(ve))
suites = mx.suites(opt_limit_to_suite=True, includeBinary=False, include_mx=False)
c = []
for s in suites:
c += _gc_layout_dists(s, parsed_args)
if not c:
mx.log("Nothing to do!")
return
if parsed_args.older_than:
c = [x for x in c if x[1] < parsed_args.older_than]
# sort by mod date
c = sorted(c, key=lambda x: x[1], reverse=parsed_args.reverse)
# calculate max sizes
max_path = 0
max_mod_time = 0
max_size = 0
for path, mod_time, size in c:
max_path = max(len(path), max_path)
max_mod_time = max(len(_format_datetime(mod_time)), max_mod_time)
max_size = max(len(_format_bytes(size)), max_size)
msg_fmt = '{0:<' + str(max_path) + '} modified {1:<' + str(max_mod_time + len(' ago')) +'} {2:<' + str(max_size) + '}'
size_sum = 0
for path, mod_time, size in c:
if parsed_args.dry_run:
mx.log(msg_fmt.format(path, _format_datetime(mod_time) + ' ago', _format_bytes(size)))
size_sum += size
else:
msg = '{0} (modified {1} ago, size {2})'.format(path, _format_datetime(mod_time), _format_bytes(size))
if parsed_args.force or parsed_args.interactive and mx.ask_yes_no('Delete ' + msg):
mx.log('rm ' + path)
mx.rmtree(path)
size_sum += size
if parsed_args.dry_run:
mx.log('Would free ' + _format_bytes(size_sum))
else:
mx.log('Freed ' + _format_bytes(size_sum))
def _graal_nodejs_post_gate_runner(args, tasks):
_setEnvVar('NODE_INTERNAL_ERROR_CHECK', 'true')
with Task('UnitTests',
tasks,
tags=[GraalNodeJsTags.allTests, GraalNodeJsTags.unitTests]) as t:
if t:
_setEnvVar('NODE_JVM_CLASSPATH',
mx.distribution('graal-js:TRUFFLE_JS_TESTS').path)
commonArgs = ['-ea', '-esa']
unitTestDir = join('test', 'graal')
for dir_name in 'node_modules', 'build':
p = join(unitTestDir, dir_name)
if exists(p):
mx.rmtree(p)
npm([
'--scripts-prepend-node-path=auto', 'install',
'--nodedir=' + _suite.dir
] + commonArgs,
cwd=unitTestDir)
npm(['--scripts-prepend-node-path=auto', 'test'] + commonArgs,
cwd=unitTestDir)
with Task('TestNpm', tasks, tags=[GraalNodeJsTags.allTests]) as t:
if t:
tmpdir = tempfile.mkdtemp()
try:
npm(['init', '-y'], cwd=tmpdir)
npm([
'--scripts-prepend-node-path=auto', 'install',
'--nodedir=' + _suite.dir, '--build-from-source',
'microtime'
],
cwd=tmpdir)
node(['-e', 'console.log(require("microtime").now());'],
cwd=tmpdir)
finally:
mx.rmtree(tmpdir, ignore_errors=True)
with Task('TestNpx', tasks, tags=[GraalNodeJsTags.allTests]) as t:
if t:
npx(['cowsay', 'GraalVM rules!'])
with Task(
'JniProfilerTests',
tasks,
tags=[GraalNodeJsTags.allTests,
GraalNodeJsTags.jniProfilerTests]) as t:
if t:
commonArgs = ['-ea', '-esa']
unitTestDir = join(
mx.project(
'com.oracle.truffle.trufflenode.jniboundaryprofiler').dir,
'tests')
for dir_name in 'node_modules', 'build':
p = join(unitTestDir, dir_name)
if exists(p):
mx.rmtree(p)
npm([
'--scripts-prepend-node-path=auto', 'install',
'--nodedir=' + _suite.dir
] + commonArgs,
cwd=unitTestDir)
node(['-profile-native-boundary', 'test.js'] + commonArgs,
cwd=unitTestDir)
with Task('TestNodeInstrument', tasks,
tags=[GraalNodeJsTags.allTests]) as t:
if t:
testnodeInstrument([])
def fetch_jdk(args):
"""fetches required JDK version
If mx is not passed the --quiet flag, menu will be printed for available JDK selection.
"""
args = _parse_fetchsettings(args)
distribution = args["java-distribution"]
base_path = args["base-path"]
artifact = distribution.get_folder_name()
final_path = distribution.get_final_path(base_path)
url = mx_urlrewrites.rewriteurl(distribution.get_url())
sha_url = url + ".sha1"
archive_name = distribution.get_archive_name()
archive_target_location = join(base_path, archive_name)
if not is_quiet():
if not mx.ask_yes_no("Install {} to {}".format(artifact, final_path),
default='y'):
mx.abort("JDK installation canceled")
if exists(final_path):
if args["keep-archive"]:
mx.warn(
"The --keep-archive option is ignored when the JDK is already installed."
)
mx.log("Requested JDK is already installed at {}".format(final_path))
else:
# Try to extract on the same file system as the target to be able to atomically move the result.
with mx.TempDir(parent_dir=base_path) as temp_dir:
mx.log("Fetching {} archive from {}...".format(artifact, url))
archive_location = join(temp_dir, archive_name)
mx._opts.no_download_progress = is_quiet()
sha1_hash = mx._hashFromUrl(sha_url).decode('utf-8')
mx.download_file_with_sha1(artifact,
archive_location, [url],
sha1_hash,
archive_location + '.sha1',
resolve=True,
mustExist=True,
sources=False)
untar = mx.TarExtractor(archive_location)
mx.log("Installing {} to {}...".format(artifact, final_path))
extracted_path = join(temp_dir, 'extracted')
try:
untar.extract(extracted_path)
except:
mx.rmtree(temp_dir, ignore_errors=True)
mx.abort("Error parsing archive. Please try again")
jdk_root_folder = get_extracted_jdk_archive_root_folder(
extracted_path)
if args["keep-archive"]:
atomic_file_move_with_fallback(archive_location,
archive_target_location)
atomic_file_move_with_fallback(
archive_location + '.sha1',
archive_target_location + ".sha1")
mx.log(
"Archive is located at {}".format(archive_target_location))
atomic_file_move_with_fallback(
join(extracted_path, jdk_root_folder), final_path)
curr_path = final_path
if mx.is_darwin() and exists(join(final_path, 'Contents', 'Home')):
if args["strip-contents-home"]:
with mx.TempDir() as tmp_path:
shutil.move(final_path, join(tmp_path, 'jdk'))
shutil.move(join(tmp_path, 'jdk', 'Contents', 'Home'),
final_path)
else:
final_path = join(final_path, 'Contents', 'Home')
if "alias" in args:
alias_full_path = join(base_path, args["alias"])
if os.path.islink(alias_full_path):
os.unlink(alias_full_path)
elif exists(alias_full_path):
mx.abort(
alias_full_path +
' exists and it is not an existing symlink so it can not be used for a new symlink. Please remove it manually.'
)
if not (mx.is_windows() or mx.is_cygwin()):
os.symlink(abspath(curr_path), alias_full_path)
else:
mx.copytree(curr_path, alias_full_path,
symlinks=True) # fallback for windows
final_path = alias_full_path
mx.log("Run the following to set JAVA_HOME in your shell:")
shell = os.environ.get("SHELL")
if shell is None:
shell = ''
mx.log(get_setvar_format(shell) % ("JAVA_HOME", abspath(final_path)))