def igv(args):
"""run the Ideal Graph Visualizer"""
logFile = '.ideal_graph_visualizer.log'
with open(join(_suite.dir, logFile), 'w') as fp:
mx.logv('[Ideal Graph Visualizer log is in ' + fp.name + ']')
nbplatform = join(_suite.dir, 'src', 'share', 'tools', 'IdealGraphVisualizer', 'nbplatform')
# Remove NetBeans platform if it is earlier than the current supported version
if exists(nbplatform):
updateTrackingFile = join(nbplatform, 'platform', 'update_tracking', 'org-netbeans-core.xml')
if not exists(updateTrackingFile):
mx.log('Could not find \'' + updateTrackingFile + '\', removing NetBeans platform')
shutil.rmtree(nbplatform)
else:
dom = xml.dom.minidom.parse(updateTrackingFile)
currentVersion = mx.VersionSpec(dom.getElementsByTagName('module_version')[0].getAttribute('specification_version'))
supportedVersion = mx.VersionSpec('3.43.1')
if currentVersion < supportedVersion:
mx.log('Replacing NetBeans platform version ' + str(currentVersion) + ' with version ' + str(supportedVersion))
shutil.rmtree(nbplatform)
elif supportedVersion < currentVersion:
mx.log('Supported NetBeans version in igv command should be updated to ' + str(currentVersion))
if not exists(nbplatform):
mx.logv('[This execution may take a while as the NetBeans platform needs to be downloaded]')
env = _igvBuildEnv()
# make the jar for Batik 1.7 available.
env['IGV_BATIK_JAR'] = mx.library('BATIK').get_path(True)
if mx.run(['ant', '-f', mx._cygpathU2W(join(_suite.dir, 'src', 'share', 'tools', 'IdealGraphVisualizer', 'build.xml')), '-l', mx._cygpathU2W(fp.name), 'run'], env=env, nonZeroIsFatal=False):
mx.abort("IGV ant build & launch failed. Check '" + logFile + "'. You can also try to delete 'src/share/tools/IdealGraphVisualizer/nbplatform'.")
def native_image_layout(dists, subdir, native_image_root, debug_gr_8964=False):
if not dists:
return
dest_path = join(native_image_root, subdir)
# Cleanup leftovers from previous call
if exists(dest_path):
if debug_gr_8964:
mx.log('[mx_substratevm.native_image_layout: remove_tree: ' + dest_path + ']')
remove_tree(dest_path)
mkpath(dest_path)
# Create symlinks to conform with native-image directory layout scheme
def symlink_jar(jar_path):
if debug_gr_8964:
def log_stat(prefix, file_name):
file_stat = os.stat(file_name)
mx.log(' ' + prefix + '.st_mode: ' + oct(file_stat.st_mode))
mx.log(' ' + prefix + '.st_mtime: ' + time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime(file_stat.st_mtime)))
dest_jar = join(dest_path, basename(jar_path))
mx.log('[mx_substratevm.native_image_layout.symlink_jar: symlink_or_copy')
mx.log(' src: ' + jar_path)
log_stat('src', jar_path)
mx.log(' dst: ' + dest_jar)
symlink_or_copy(jar_path, dest_jar, debug_gr_8964)
log_stat('dst', dest_jar)
mx.log(']')
else:
symlink_or_copy(jar_path, join(dest_path, basename(jar_path)), debug_gr_8964)
for dist in dists:
mx.logv('Add ' + type(dist).__name__ + ' ' + str(dist) + ' to ' + dest_path)
symlink_jar(dist.path)
if not dist.isBaseLibrary() and dist.sourcesPath:
symlink_jar(dist.sourcesPath)
def hsdis(args, copyToDir=None):
"""download the hsdis library
This is needed to support HotSpot's assembly dumping features.
By default it downloads the Intel syntax version, use the 'att' argument to install AT&T syntax."""
flavor = 'intel'
if 'att' in args:
flavor = 'att'
if mx.get_arch() == "sparcv9":
flavor = "sparcv9"
lib = mx.add_lib_suffix('hsdis-' + mx.get_arch())
path = join(_suite.dir, 'lib', lib)
sha1s = {
'att/hsdis-amd64.dll' : 'bcbd535a9568b5075ab41e96205e26a2bac64f72',
'att/hsdis-amd64.so' : '58919ba085d4ef7a513f25bae75e7e54ee73c049',
'intel/hsdis-amd64.dll' : '6a388372cdd5fe905c1a26ced614334e405d1f30',
'intel/hsdis-amd64.so' : '844ed9ffed64fe9599638f29a8450c50140e3192',
'intel/hsdis-amd64.dylib' : 'fdb13ef0d7d23d93dacaae9c98837bea0d4fc5a2',
'sparcv9/hsdis-sparcv9.so': '970640a9af0bd63641f9063c11275b371a59ee60',
}
flavoredLib = flavor + "/" + lib
if flavoredLib not in sha1s:
mx.logv("hsdis not supported on this plattform or architecture")
return
if not exists(path):
sha1 = sha1s[flavoredLib]
sha1path = path + '.sha1'
mx.download_file_with_sha1('hsdis', path, ['https://lafo.ssw.uni-linz.ac.at/pub/hsdis/' + flavoredLib], sha1, sha1path, True, True, sources=False)
if copyToDir is not None and exists(copyToDir):
shutil.copy(path, copyToDir)
def _log_ignored_component(kept, ignored):
"""
:type kept: GraalVmComponent
:type ignored: GraalVmComponent
"""
mx.logv('Suites \'{}\' and \'{}\' are registering a component with the same short name (\'{}\'), with priority \'{}\' and \'{}\' respectively.'.format(kept.suite.name, ignored.suite.name, kept.short_name, kept.priority, ignored.priority))
mx.logv('Ignoring the one from suite \'{}\'.'.format(ignored.suite.name))
def jdkartifactstats(args):
"""show stats about JDK deployed Graal artifacts"""
artifacts = {}
jdkDir = get_jvmci_jdk().home
def _getDeployedJars():
if JVMCI_VERSION < 9:
for root, _, filenames in os.walk(join(jdkDir, 'jre', 'lib')):
for f in filenames:
if f.endswith('.jar') and not f.endswith('.stripped.jar'):
yield join(root, f)
else:
for jdkDist in jdkDeployedDists:
dist = jdkDist.dist()
if isinstance(jdkDist, JvmciJDKDeployedDist):
yield dist.path
for jar in _getDeployedJars():
f = basename(jar)
if 'truffle' in f:
if 'enterprise' in f:
artifacts.setdefault('GraalEnterpriseTruffle', []).append(jar)
else:
artifacts.setdefault('GraalTruffle', []).append(jar)
elif 'enterprise' in f:
artifacts.setdefault('GraalEnterprise', []).append(jar)
elif 'jvmci' in f:
artifacts.setdefault('JVMCI', []).append(jar)
elif 'graal' in f:
artifacts.setdefault('Graal', []).append(jar)
else:
mx.logv('ignored: ' + jar)
print '{:>10} {:>10} {:>10} {}'.format('All', 'NoVars', 'None', 'Jar')
for category in sorted(artifacts.viewkeys()):
jars = artifacts[category]
if jars:
totals = (0, 0, 0)
print
for j in jars:
gSize = os.path.getsize(j)
stripped = j[:-len('.jar')] + '.stripped.jar'
mx.run([mx.get_jdk().pack200, '--repack', '--quiet', '-J-Djava.util.logging.config.file=', '-DLocalVariableTypeTable=strip', '-DLocalVariableTable=strip', stripped, j])
gLinesSourceSize = os.path.getsize(stripped)
mx.run([mx.get_jdk().pack200, '--repack', '--quiet', '-J-Djava.util.logging.config.file=', '-G', stripped, j])
gNoneSize = os.path.getsize(stripped)
os.remove(stripped)
print '{:10,} {:10,} {:10,} {}:{}'.format(gSize, gLinesSourceSize, gNoneSize, category, basename(j))
t1, t2, t3 = totals
totals = (t1 + gSize, t2 + gLinesSourceSize, t3 + gNoneSize)
t1, t2, t3 = totals
print '{:10,} {:10,} {:10,} {}'.format(t1, t2, t3, category)
jvmLib = join(jdkDir, relativeVmLibDirInJdk(), get_vm(), mx.add_lib_suffix(mx.add_lib_prefix('jvm')))
print
if exists(jvmLib):
print '{:10,} {}'.format(os.path.getsize(jvmLib), jvmLib)
else:
print '{:>10} {}'.format('<missing>', jvmLib)
def native_image_extract(dists, subdir, native_image_root):
target_dir = join(native_image_root, subdir)
for distribution in dists:
mx.logv('Add dist {} to {}'.format(distribution, target_dir))
if distribution.path.endswith('tar'):
compressedFile = tarfile.open(distribution.path, 'r:')
elif distribution.path.endswith('jar') or distribution.path.endswith('zip'):
compressedFile = zipfile.ZipFile(distribution.path)
else:
raise mx.abort('Unsupported compressed file ' + distribution.path)
compressedFile.extractall(target_dir)
def mavenSetup():
mavenDir = join(_suite.dir, 'mxbuild', 'mvn')
maven_repo_arg = '-Dmaven.repo.local=' + mavenDir
env = os.environ.copy()
env['JRUBY_BUILD_MORE_QUIET'] = 'true'
# HACK: since the maven executable plugin does not configure the
# java executable that is used we unfortunately need to append it to the PATH
javaHome = os.getenv('JAVA_HOME')
if javaHome:
env["PATH"] = javaHome + '/bin' + os.pathsep + env["PATH"]
mx.logv('Setting PATH to {}'.format(os.environ["PATH"]))
mx.run(['java', '-version'])
return maven_repo_arg, env
def native_image_option_properties(option_kind, option_flag, native_image_root):
target_dir = join(native_image_root, option_kind, option_flag)
target_path = remove_existing_symlink(join(target_dir, 'native-image.properties'))
option_properties = None
for svm_suite in svmSuites:
candidate = join(svm_suite.mxDir, option_kind + '-' + option_flag + '.properties')
if exists(candidate):
option_properties = candidate
if option_properties:
mx.logv('Add symlink to ' + str(option_properties))
mkpath(target_dir)
symlink_or_copy(option_properties, target_path)
def build_number():
"""
Get the current build number from the BUILD_NUMBER environment variable. If BUILD_NUMBER is not set or not a number,
a default of -1 is returned.
:return: the build number
:rtype: int
"""
build_num = mx.get_env("BUILD_NUMBER", default="-1")
try:
return int(build_num)
except ValueError:
mx.logv("Could not parse the build number from BUILD_NUMBER. Expected int, instead got: {0}".format(build_num))
return -1
def mavenSetup():
buildPack = join(_suite.dir, 'jruby-build-pack/maven')
mavenDir = buildPack if isdir(buildPack) else join(_suite.dir, 'mxbuild/mvn')
maven_repo_arg = '-Dmaven.repo.local=' + mavenDir
env = os.environ.copy()
if not mx.get_opts().verbose:
env['JRUBY_BUILD_MORE_QUIET'] = 'true'
# HACK: since the maven executable plugin does not configure the
# java executable that is used we unfortunately need to prepend it to the PATH
javaHome = os.getenv('JAVA_HOME')
if javaHome:
env["PATH"] = javaHome + '/bin' + os.pathsep + env["PATH"]
mx.logv('Setting PATH to {}'.format(os.environ["PATH"]))
mx.run(['java', '-version'], env=env)
return maven_repo_arg, env
def build(self):
if not self.subject.build:
mx.log("...skip build of {}".format(self.subject))
return
mx.log('...perform build of {}'.format(self.subject))
rubyDir = _suite.dir
mavenDir = os.path.join(rubyDir, 'mxbuild', 'mvn')
# HACK: since the maven executable plugin does not configure the
# java executable that is used we unfortunately need to append it to the PATH
javaHome = os.getenv('JAVA_HOME')
if javaHome:
os.environ["PATH"] = os.environ["JAVA_HOME"] + '/bin' + os.pathsep + os.environ["PATH"]
mx.logv('Setting PATH to {}'.format(os.environ["PATH"]))
mx.logv('Calling java -version')
mx.run(['java', '-version'])
# Truffle version
truffle = mx.suite('truffle')
truffle_commit = truffle.vc.parent(truffle.dir)
maven_version_arg = '-Dtruffle.version=' + truffle_commit
maven_repo_arg = '-Dmaven.repo.local=' + mavenDir
mx.run_mx(['maven-install', '--repo', mavenDir, '--only', 'TRUFFLE_API,TRUFFLE_DEBUG,TRUFFLE_DSL_PROCESSOR,TRUFFLE_TCK'], suite=truffle)
open(os.path.join(rubyDir, 'VERSION'), 'w').write('graal-vm\n')
# Build jruby-truffle
env = os.environ.copy()
env['JRUBY_BUILD_MORE_QUIET'] = 'true'
mx.run_maven(['-q', '--version', maven_repo_arg], nonZeroIsFatal=False, cwd=rubyDir, env=env)
mx.log('Building without tests')
mx.run_maven(['-q', '-DskipTests', maven_version_arg, maven_repo_arg], cwd=rubyDir, env=env)
mx.log('Building complete version')
mx.run_maven(['-q', '-Pcomplete', '-DskipTests', maven_version_arg, maven_repo_arg], cwd=rubyDir, env=env)
mx.run(['zip', '-d', 'maven/jruby-complete/target/jruby-complete-graal-vm.jar', 'META-INF/jruby.home/lib/*'], cwd=rubyDir)
mx.run(['bin/jruby', 'bin/gem', 'install', 'bundler', '-v', '1.10.6'], cwd=rubyDir)
mx.log('...finished build of {}'.format(self.subject))
def testdownstream(args):
"""test downstream users of GraalCore"""
parser = ArgumentParser(prog='mx testdownstream')
parser.add_argument('--target', action='store', help='URL of client repo to clone', required=True, metavar='<url>')
parser.add_argument('--suitedir', action='store', help='directory of target suite in client repo', default='.', metavar='<path>')
parser.add_argument('-C', dest='clientMxCmd', action='append', help='arg to mx command run on client (e.g., -C-v -C--strict-compliance -Cgate)', default=[], metavar='<arg>')
args = parser.parse_args(args)
workDir = join(_suite.get_output_root(), 'testdownstream')
mirror = join(workDir, _suite.name)
if exists(mirror):
shutil.rmtree(mirror)
mx.ensure_dir_exists(mirror)
for f in os.listdir(_suite.dir):
subDir = join(_suite.dir, f)
if subDir == _suite.get_output_root():
continue
src = join(_suite.dir, f)
dst = join(mirror, f)
mx.logv('[Creating symlink from {} to {}]'.format(dst, src))
os.symlink(src, dst)
# Deduce a target name from the target URL
url = urlparse(args.target)
targetName = url.path
if targetName.rfind('/') != -1:
targetName = targetName[targetName.rfind('/') + 1:]
if targetName.endswith('.git'):
targetName = targetName[0:-len('.git')]
targetDir = join(workDir, targetName)
git = mx.GitConfig()
if exists(targetDir):
git.pull(targetDir)
else:
git.clone(args.target, targetDir)
# See if there's a matching (non-master) branch downstream and use it if there is
branch = git.git_command(_suite.dir, ['rev-parse', '--abbrev-ref', 'HEAD']).strip()
if branch != 'master':
git.git_command(targetDir, ['checkout', branch], abortOnError=False)
targetSuiteDir = join(targetDir, args.suitedir)
cmd = ['--java-home=' + mx.get_jdk().home] + args.clientMxCmd
mx.logv('[running "mx ' + ' '.join(cmd) + '" in ' + targetSuiteDir + ']')
return mx.run_mx(cmd, targetSuiteDir)
def hsdis(args, copyToDir=None):
"""download the hsdis library
This is needed to support HotSpot's assembly dumping features.
By default it downloads the Intel syntax version, use the 'att' argument to install AT&T syntax."""
flavor = None
if mx.get_arch() == "amd64":
flavor = mx.get_env('HSDIS_SYNTAX')
if flavor is None:
flavor = 'intel'
if 'att' in args:
flavor = 'att'
libpattern = mx.add_lib_suffix('hsdis-' + mx.get_arch() + '-' + mx.get_os() + '-%s')
sha1s = {
'att/hsdis-amd64-windows-%s.dll' : 'bcbd535a9568b5075ab41e96205e26a2bac64f72',
'att/hsdis-amd64-linux-%s.so' : '36a0b8e30fc370727920cc089f104bfb9cd508a0',
'att/hsdis-amd64-darwin-%s.dylib' : 'c1865e9a58ca773fdc1c5eea0a4dfda213420ffb',
'intel/hsdis-amd64-windows-%s.dll' : '6a388372cdd5fe905c1a26ced614334e405d1f30',
'intel/hsdis-amd64-linux-%s.so' : '0d031013db9a80d6c88330c42c983fbfa7053193',
'intel/hsdis-amd64-darwin-%s.dylib' : '67f6d23cbebd8998450a88b5bef362171f66f11a',
'hsdis-sparcv9-solaris-%s.so': '970640a9af0bd63641f9063c11275b371a59ee60',
'hsdis-sparcv9-linux-%s.so': '0c375986d727651dee1819308fbbc0de4927d5d9',
}
if flavor:
flavoredLib = flavor + "/" + libpattern
else:
flavoredLib = libpattern
if flavoredLib not in sha1s:
mx.warn("hsdis with flavor '{}' not supported on this plattform or architecture".format(flavor))
return
sha1 = sha1s[flavoredLib]
lib = flavoredLib % (sha1)
path = join(_suite.get_output_root(), lib)
if not exists(path):
sha1path = path + '.sha1'
mx.download_file_with_sha1('hsdis', path, ['https://lafo.ssw.uni-linz.ac.at/pub/hsdis/' + lib], sha1, sha1path, True, True, sources=False)
if copyToDir is not None and exists(copyToDir):
destFileName = mx.add_lib_suffix('hsdis-' + mx.get_arch())
mx.logv('Copying {} to {}'.format(path, copyToDir + os.sep + destFileName))
shutil.copy(path, copyToDir + os.sep + destFileName)
def _netbeansinit_suite(args,
suite,
refreshOnly=False,
buildProcessorJars=True):
mxOutputDir = mx.ensure_dir_exists(suite.get_mx_output_dir())
configZip = mx.TimeStampFile(join(mxOutputDir, 'netbeans-config.zip'))
configLibsZip = join(mxOutputDir, 'eclipse-config-libs.zip')
if refreshOnly and not configZip.exists():
return
if mx_ideconfig._check_ide_timestamp(suite, configZip, 'netbeans'):
mx.logv('[NetBeans configurations are up to date - skipping]')
return
files = []
libFiles = []
jdks = set()
for p in suite.projects:
if not p.isJavaProject():
continue
if exists(join(p.dir, 'plugin.xml')): # eclipse plugin project
continue
includedInDists = [d for d in suite.dists if p in d.archived_deps()]
_netbeansinit_project(p, jdks, files, libFiles, includedInDists)
mx.log('If using NetBeans:')
# http://stackoverflow.com/questions/24720665/cant-resolve-jdk-internal-package
mx.log(
' 1. Edit etc/netbeans.conf in your NetBeans installation and modify netbeans_default_options variable to include "-J-DCachingArchiveProvider.disableCtSym=true"'
)
mx.log(
' 2. Ensure that the following platform(s) are defined (Tools -> Java Platforms):'
)
for jdk in jdks:
mx.log(' JDK_' + str(jdk.version))
mx.log(
' 3. Open/create a Project Group for the directory containing the projects (File -> Project Group -> New Group... -> Folder of Projects)'
)
mx_ideconfig._zip_files(files, suite.dir, configZip.path)
mx_ideconfig._zip_files(libFiles, suite.dir, configLibsZip)
def setupNodeEnvironment(args, add_graal_vm_args=True):
args = args if args else []
mode, vmArgs, progArgs = _parseArgs(args)
setLibraryPath()
if _is_windows:
processDevkitRoot()
if mx.suite('vm', fatalIfMissing=False) is not None and mx.suite('substratevm', fatalIfMissing=False) is not None:
_prepare_svm_env()
return mode, vmArgs, progArgs
if mx.suite('vm', fatalIfMissing=False) is not None or mx.suite('substratevm', fatalIfMissing=False) is not None:
mx.warn("Running on the JVM.\nIf you want to run on SubstrateVM, you need to dynamically import both '/substratevm' and '/vm'.\nExample: 'mx --env svm node'")
_setEnvVar('JAVA_HOME', _java_home())
if mx.suite('compiler', fatalIfMissing=False) is None:
_setEnvVar('GRAAL_SDK_JAR_PATH', mx.distribution('sdk:GRAAL_SDK').path)
_setEnvVar('LAUNCHER_COMMON_JAR_PATH', mx.distribution('sdk:LAUNCHER_COMMON').path)
_setEnvVar('TRUFFLENODE_JAR_PATH', mx.distribution('TRUFFLENODE').path)
node_jvm_cp = (os.environ['NODE_JVM_CLASSPATH'] + pathsep) if 'NODE_JVM_CLASSPATH' in os.environ else ''
node_cp = node_jvm_cp + mx.classpath(['TRUFFLENODE'] + (['tools:CHROMEINSPECTOR', 'tools:TRUFFLE_PROFILER', 'tools:AGENTSCRIPT'] if mx.suite('tools', fatalIfMissing=False) is not None else []))
_setEnvVar('NODE_JVM_CLASSPATH', node_cp)
prevPATH = os.environ['PATH']
_setEnvVar('PATH', "%s%s%s" % (join(_suite.mxDir, 'fake_launchers'), pathsep, prevPATH))
if _has_jvmci() and add_graal_vm_args:
if mx.suite('graal-enterprise', fatalIfMissing=False):
# explicitly require the enterprise compiler configuration
vmArgs += ['-Dgraal.CompilerConfiguration=enterprise']
if mx.suite('compiler', fatalIfMissing=False):
vmArgs += ['-Djvmci.Compiler=graal', '-XX:+UnlockExperimentalVMOptions', '-XX:+EnableJVMCI']
if isinstance(_suite, BinarySuite):
mx.logv('%s is a binary suite' % _suite.name)
tarfilepath = mx.distribution('TRUFFLENODE_GRAALVM_SUPPORT').path
with tarfile.open(tarfilepath, 'r:') as tar:
mx.logv('Extracting {} to {}'.format(tarfilepath, _suite.dir))
tar.extractall(_suite.dir)
return mode, vmArgs, progArgs
def commits_in_range(self):
grep_filter = str(self.config.commits_filter).replace('[',
r'\[').replace(
']', r'\]')
commits_filter = self.build_steps.git_log_filtering_strategy()
git_log_command_base = commits_filter + [
'--grep=' + grep_filter, '--pretty=format:%h|%ct|%s'
]
if self.config.start_commit is not None:
commits_range = [
self.config.start_commit + '..' + self.config.end_commit
]
elif self.config.after is not None:
commits_range = [
'--after="' + self.config.after + '"',
'--before="' + self.config.before + '"'
]
else:
commits_range = [
'--after="' + self.build_steps.default_start_date() + '"',
'--before="' + self.config.before + '"'
]
mx.log('Commits Filtering Strategy: ' +
' '.join(commits_filter + commits_range))
unparsed_commits = subprocess.check_output(
git_log_command_base + commits_range,
universal_newlines=True).splitlines()
def commit_parser(c):
commit_hash, date, msg = c.split('|', 2)
commit = Commit(commit_hash, msg,
datetime.fromtimestamp(int(date)))
mx.logv(commit)
return commit
mx.logv('---------- Commits in range to analyze')
commits = list(map(commit_parser, unparsed_commits))
mx.log('Commits in range to analyze: {}, hashes [{} - {}]'.format(
len(commits), commits[len(commits) - 1].hash, commits[0].hash))
return commits
def setupNodeEnvironment(args, add_graal_vm_args=True):
javaHome = _getJdkHome()
_setEnvVar('JAVA_HOME', javaHome)
if mx.suite('compiler', fatalIfMissing=False) is None:
_setEnvVar('GRAAL_SDK_JAR_PATH', mx.distribution('sdk:GRAAL_SDK').path)
_setEnvVar('TRUFFLE_JAR_PATH',
mx.distribution('truffle:TRUFFLE_API').path)
_setEnvVar('LAUNCHER_COMMON_JAR_PATH',
mx.distribution('sdk:LAUNCHER_COMMON').path)
_setEnvVar('TRUFFLENODE_JAR_PATH', mx.distribution('TRUFFLENODE').path)
_setEnvVar(
'NODE_JVM_CLASSPATH',
mx.classpath(['TRUFFLENODE'] + (
['tools:CHROMEINSPECTOR', 'tools:TRUFFLE_PROFILER'] if mx.
suite('tools', fatalIfMissing=False) is not None else [])))
setLibraryPath()
prevPATH = os.environ['PATH']
_setEnvVar('PATH',
"%s:%s" % (join(_suite.mxDir, 'fake_launchers'), prevPATH))
args = args if args else []
mode, vmArgs, progArgs = _parseArgs(args)
if add_graal_vm_args:
if mx.suite('graal-enterprise', fatalIfMissing=False):
# explicitly require the enterprise compiler configuration
vmArgs += ['-Dgraal.CompilerConfiguration=enterprise']
if mx.suite('compiler', fatalIfMissing=False):
vmArgs += [
'-Djvmci.Compiler=graal', '-XX:+UnlockExperimentalVMOptions',
'-XX:+EnableJVMCI'
]
if isinstance(_suite, BinarySuite):
mx.logv('%s is a binary suite' % _suite.name)
tarfilepath = mx.distribution('TRUFFLENODE_GRAALVM_SUPPORT').path
with tarfile.open(tarfilepath, 'r:') as tar:
mx.logv('Extracting {} to {}'.format(tarfilepath, _suite.dir))
tar.extractall(_suite.dir)
return mode, vmArgs, progArgs
def python_build_watch(args):
"""
Watch the suite and on any changes to .class, .jar, .h, or .c files rebuild.
By default, rebuilds only the archives and non-Java projects.
"""
parser = ArgumentParser(prog='mx python-build-watch')
parser.add_argument('--full', action='store_true', help='Run a full mx build', required=False)
parser.add_argument('--graalvm', action='store_true', help='Build a graalvm', required=False)
parser.add_argument('--no-java', action='store_true', help='Build only archives and native projects [default]', required=False)
args = parser.parse_args(args)
if sum([args.full, args.graalvm, args.no_java]) > 1:
mx.abort("Only one of --full, --graalvm, --no-java can be specified")
if args.full:
suffixes = [".c", ".h", ".class", ".jar", ".java"]
excludes = [".*\\.py$"]
elif args.graalvm:
suffixes = [".c", ".h", ".class", ".jar", ".java", ".py"]
excludes = ["mx_.*\\.py$"]
else:
suffixes = [".c", ".h", ".class", ".jar"]
excludes = [".*\\.py$", ".*\\.java$"]
cmd = ["inotifywait", "-q", "-e", "close_write,moved_to", "-r", "--format=%f"]
for e in excludes:
cmd += ["--exclude", e]
cmd += ["@%s" % os.path.join(SUITE.dir, ".git"), SUITE.dir]
while True:
out = mx.OutputCapture()
mx.run(cmd, out=out)
changed_file = out.data.strip()
mx.logv(changed_file)
if any(changed_file.endswith(ext) for ext in [".c", ".h", ".class", ".jar"]):
mx.log("Build needed ...")
time.sleep(2)
if args.full:
mx.command_function("build")()
elif args.graalvm:
mx.log(python_gvm())
else:
nativebuild([])
mx.log("Build done.")
def native_image_layout(dists, subdir, native_image_root, debug_gr_8964=False):
if not dists:
return
dest_path = join(native_image_root, subdir)
# Cleanup leftovers from previous call
if exists(dest_path):
if debug_gr_8964:
mx.log('[mx_substratevm.native_image_layout: remove_tree: ' +
dest_path + ']')
remove_tree(dest_path)
mkpath(dest_path)
# Create symlinks to conform with native-image directory layout scheme
def symlink_jar(jar_path):
if debug_gr_8964:
def log_stat(prefix, file_name):
file_stat = os.stat(file_name)
mx.log(' ' + prefix + '.st_mode: ' + oct(file_stat.st_mode))
mx.log(' ' + prefix + '.st_mtime: ' + time.strftime(
'%Y-%m-%dT%H:%M:%SZ', time.gmtime(file_stat.st_mtime)))
dest_jar = join(dest_path, basename(jar_path))
mx.log(
'[mx_substratevm.native_image_layout.symlink_jar: symlink_or_copy'
)
mx.log(' src: ' + jar_path)
log_stat('src', jar_path)
mx.log(' dst: ' + dest_jar)
symlink_or_copy(jar_path, dest_jar, debug_gr_8964)
log_stat('dst', dest_jar)
mx.log(']')
else:
symlink_or_copy(jar_path, join(dest_path, basename(jar_path)),
debug_gr_8964)
for dist in dists:
mx.logv('Add ' + type(dist).__name__ + ' ' + str(dist) + ' to ' +
dest_path)
symlink_jar(dist.path)
if not dist.isBaseLibrary() and dist.sourcesPath:
symlink_jar(dist.sourcesPath)
def native_image_layout(dists, subdir, native_image_root):
dest_path = join(native_image_root, subdir)
# Cleanup leftovers from previous call
if exists(dest_path):
remove_tree(dest_path)
mkpath(dest_path)
# Create symlinks to conform with native-image directory layout scheme
def symlink_jar(jar_path):
relsymlink(jar_path, join(dest_path, basename(jar_path)))
for dist in dists:
mx.logv('Add ' + type(dist).__name__ + ' ' + str(dist) + ' to ' +
dest_path)
if dist.isLibrary():
dist.get_path(resolve=True)
dist.get_source_path(resolve=True)
symlink_jar(dist.path)
if not dist.isLibrary():
symlink_jar(dist.sourcesPath)
def checkCFiles(target, reason):
error = False
files_to_check = []
if os.path.isfile(target):
files_to_check.append(target)
else:
for path, _, files in os.walk(target):
for f in files:
if f.endswith('.c') or f.endswith('.cpp') or f.endswith(
'.h') or f.endswith('.hpp'):
files_to_check.append(os.path.join(path, f))
if not files_to_check:
mx.logv("clang-format: no files found {} ({})".format(target, reason))
return True
mx.logv("clang-format: checking {} ({}, {} files)".format(
target, reason, len(files_to_check)))
for f in files_to_check:
if not checkCFile(f):
error = True
return not error
def _fetch_test_suite(dest, library_names):
def _get_lib_path(_lib_name):
return mx.library(_lib_name).get_path(resolve=True)
_extract = False
for _lib_name in library_names:
if not exists(dest) or getmtime(
_get_lib_path(_lib_name)) > getmtime(dest):
mx.logv('{} needs to be extracted'.format(_lib_name))
_extract = True
break
if _extract:
if exists(dest):
mx.logv('Deleting the old test directory {}'.format(dest))
shutil.rmtree(dest)
mx.ensure_dir_exists(dest)
for _lib_name in library_names:
with tarfile.open(_get_lib_path(_lib_name), 'r') as _tar:
_tar.extractall(dest)
def _runmultimake(args):
"""run the JDK make process for one or more configurations"""
jvmVariantsDefault = ','.join(_jdkJvmVariants)
debugLevelsDefault = ','.join(_jdkDebugLevels)
parser = ArgumentParser(prog='mx multimake')
parser.add_argument('--jdk-jvm-variants', '--vms', help='a comma separated list of VMs to build (default: ' + jvmVariantsDefault + ')', metavar='<args>', default=jvmVariantsDefault)
parser.add_argument('--jdk-debug-levels', '--builds', help='a comma separated list of JDK debug levels (default: ' + debugLevelsDefault + ')', metavar='<args>', default=debugLevelsDefault)
parser.add_argument('-n', '--no-check', action='store_true', help='omit running "java -version" after each build')
select = parser.add_mutually_exclusive_group()
select.add_argument('-c', '--console', action='store_true', help='send build output to console instead of log files')
select.add_argument('-d', '--output-dir', help='directory for log files instead of current working directory', default=os.getcwd(), metavar='<dir>')
args = parser.parse_args(args)
jvmVariants = args.jdk_jvm_variants.split(',')
debugLevels = [_translateLegacyDebugLevel(dl) for dl in args.jdk_debug_levels.split(',')]
allStart = time.time()
for jvmVariant in jvmVariants:
for debugLevel in debugLevels:
if not args.console:
logFile = join(mx.ensure_dir_exists(args.output_dir), jvmVariant + '-' + debugLevel + '.log')
log = open(logFile, 'wb')
start = time.time()
mx.log('BEGIN: ' + jvmVariant + '-' + debugLevel + '\t(see: ' + logFile + ')')
verbose = ['-v'] if mx.get_opts().verbose else []
# Run as subprocess so that output can be directed to a file
cmd = [sys.executable, '-u', mx.__file__] + verbose + ['--jdk-jvm-variant=' + jvmVariant, '--jdk-debug-level=' + debugLevel, 'make']
mx.logv("executing command: " + str(cmd))
subprocess.check_call(cmd, cwd=_suite.dir, stdout=log, stderr=subprocess.STDOUT)
duration = datetime.timedelta(seconds=time.time() - start)
mx.log('END: ' + jvmVariant + '-' + debugLevel + '\t[' + str(duration) + ']')
else:
with VM(jvmVariant=jvmVariant, debugLevel=debugLevel):
_runmake([])
if not args.no_check:
with VM(jvmciMode='jit'):
run_vm(['-XX:-BootstrapJVMCI', '-version'])
allDuration = datetime.timedelta(seconds=time.time() - allStart)
mx.log('TOTAL TIME: ' + '[' + str(allDuration) + ']')
def _runmultimake(args):
"""run the JDK make process for one or more configurations"""
jvmVariantsDefault = ','.join(_jdkJvmVariants)
debugLevelsDefault = ','.join(_jdkDebugLevels)
parser = ArgumentParser(prog='mx multimake')
parser.add_argument('--jdk-jvm-variants', '--vms', help='a comma separated list of VMs to build (default: ' + jvmVariantsDefault + ')', metavar='<args>', default=jvmVariantsDefault)
parser.add_argument('--jdk-debug-levels', '--builds', help='a comma separated list of JDK debug levels (default: ' + debugLevelsDefault + ')', metavar='<args>', default=debugLevelsDefault)
parser.add_argument('-n', '--no-check', action='store_true', help='omit running "java -version" after each build')
select = parser.add_mutually_exclusive_group()
select.add_argument('-c', '--console', action='store_true', help='send build output to console instead of log files')
select.add_argument('-d', '--output-dir', help='directory for log files instead of current working directory', default=os.getcwd(), metavar='<dir>')
args = parser.parse_args(args)
jvmVariants = args.jdk_jvm_variants.split(',')
debugLevels = [_translateLegacyDebugLevel(dl) for dl in args.jdk_debug_levels.split(',')]
allStart = time.time()
for jvmVariant in jvmVariants:
for debugLevel in debugLevels:
if not args.console:
logFile = join(mx.ensure_dir_exists(args.output_dir), jvmVariant + '-' + debugLevel + '.log')
log = open(logFile, 'wb')
start = time.time()
mx.log('BEGIN: ' + jvmVariant + '-' + debugLevel + '\t(see: ' + logFile + ')')
verbose = ['-v'] if mx.get_opts().verbose else []
# Run as subprocess so that output can be directed to a file
cmd = [sys.executable, '-u', mx.__file__] + verbose + ['--jdk-jvm-variant=' + jvmVariant, '--jdk-debug-level=' + debugLevel, 'make']
mx.logv("executing command: " + str(cmd))
subprocess.check_call(cmd, cwd=_suite.dir, stdout=log, stderr=subprocess.STDOUT)
duration = datetime.timedelta(seconds=time.time() - start)
mx.log('END: ' + jvmVariant + '-' + debugLevel + '\t[' + str(duration) + ']')
else:
with VM(jvmVariant=jvmVariant, debugLevel=debugLevel):
_runmake([])
if not args.no_check:
with VM(jvmciMode='jit'):
run_vm(['-XX:-BootstrapJVMCI', '-version'])
allDuration = datetime.timedelta(seconds=time.time() - allStart)
mx.log('TOTAL TIME: ' + '[' + str(allDuration) + ']')
def build(self):
if not self.subject.build:
mx.log("...skip build of {}".format(self.subject))
return
mx.log('...perform build of {}'.format(self.subject))
rubyDir = _suite.dir
# HACK: since the maven executable plugin does not configure the
# java executable that is used we unfortunately need to append it to the PATH
javaHome = os.getenv('JAVA_HOME')
if javaHome:
os.environ["PATH"] = os.environ["JAVA_HOME"] + '/bin' + os.pathsep + os.environ["PATH"]
mx.logv('Setting PATH to {}'.format(os.environ["PATH"]))
mx.logv('Calling java -version')
mx.run(['java', '-version'])
# Truffle version
truffle = mx.suite('truffle')
truffle_commit = truffle.vc.parent(truffle.dir)
mx.run_mx(['maven-install'], suite=truffle)
open(join(rubyDir, 'VERSION'), 'w').write('graal-vm\n')
# Build jruby-truffle and
mx.run(['find', '.'], nonZeroIsFatal=False, cwd=rubyDir)
mx.run_maven(['--version'], nonZeroIsFatal=False, cwd=rubyDir)
mx.log('Building without tests')
mx.run_maven(['-DskipTests', '-Dtruffle.version=' + truffle_commit], cwd=rubyDir)
mx.log('Building complete version')
mx.run_maven(['-Pcomplete', '-DskipTests', '-Dtruffle.version=' + truffle_commit], cwd=rubyDir)
mx.run(['zip', '-d', 'maven/jruby-complete/target/jruby-complete-graal-vm.jar', 'META-INF/jruby.home/lib/*'], cwd=rubyDir)
mx.run(['bin/jruby', 'bin/gem', 'install', 'bundler', '-v', '1.10.6'], cwd=rubyDir)
def spotbugs(args,
fbArgs=None,
suite=None,
projects=None,
jarFileName='spotbugs.jar'):
projectsToTest = [p for p in mx.projects() if _should_test_project(p)]
projectsByVersion = {}
for p in projectsToTest:
compat = p.suite.getMxCompatibility()
spotbugsVersion = compat.spotbugs_version()
if spotbugsVersion not in projectsByVersion:
projectsByVersion[spotbugsVersion] = []
projectsByVersion[spotbugsVersion].append(p)
resultcode = 0
for spotbugsVersion, versionProjects in projectsByVersion.items():
mx.logv('Running spotbugs version {} on projects {}'.format(
spotbugsVersion, versionProjects))
resultcode = max(
resultcode,
_spotbugs(args, fbArgs, suite, versionProjects, spotbugsVersion))
return resultcode
def hsdis(args, copyToDir=None):
"""download the hsdis library
This is needed to support HotSpot's assembly dumping features.
By default it downloads the Intel syntax version, use the 'att' argument to install AT&T syntax."""
flavor = 'intel'
if 'att' in args:
flavor = 'att'
if mx.get_arch() == "sparcv9":
flavor = "sparcv9"
lib = mx.add_lib_suffix('hsdis-' + mx.get_arch())
path = join(_suite.dir, 'lib', lib)
sha1s = {
'att/hsdis-amd64.dll': 'bcbd535a9568b5075ab41e96205e26a2bac64f72',
'att/hsdis-amd64.so': '58919ba085d4ef7a513f25bae75e7e54ee73c049',
'intel/hsdis-amd64.dll': '6a388372cdd5fe905c1a26ced614334e405d1f30',
'intel/hsdis-amd64.so': '844ed9ffed64fe9599638f29a8450c50140e3192',
'intel/hsdis-amd64.dylib': 'fdb13ef0d7d23d93dacaae9c98837bea0d4fc5a2',
'sparcv9/hsdis-sparcv9.so': '970640a9af0bd63641f9063c11275b371a59ee60',
}
flavoredLib = flavor + "/" + lib
if flavoredLib not in sha1s:
mx.logv("hsdis not supported on this plattform or architecture")
return
if not exists(path):
sha1 = sha1s[flavoredLib]
sha1path = path + '.sha1'
mx.download_file_with_sha1(
'hsdis',
path, ['https://lafo.ssw.uni-linz.ac.at/pub/hsdis/' + flavoredLib],
sha1,
sha1path,
True,
True,
sources=False)
if copyToDir is not None and exists(copyToDir):
shutil.copy(path, copyToDir)
def process_dep(dep, edge):
if dep is proj:
return
if dep.isLibrary() or dep.isJARDistribution() or dep.isMavenProject():
libraries.add(dep)
moduleXml.element('orderEntry', attributes={'type': 'library', 'name': dep.name, 'level': 'project'})
elif dep.isJavaProject():
dependencies_project_packages.update(dep.defined_java_packages())
referenced_modules.add(dep.name)
moduleXml.element('orderEntry', attributes={'type': 'module', 'module-name': dep.name})
elif dep.isJdkLibrary():
jdk_libraries.add(dep)
if jdk.javaCompliance < dep.jdkStandardizedSince:
moduleXml.element('orderEntry', attributes={'type': 'library', 'name': dep.name, 'level': 'project'})
else:
mx.logv("{} skipping {} for {}".format(p, dep, jdk)) #pylint: disable=undefined-loop-variable
elif dep.isJreLibrary():
pass
elif dep.isClasspathDependency():
moduleXml.element('orderEntry', attributes={'type': 'library', 'name': dep.name, 'level': 'project'})
else:
mx.abort("Dependency not supported: {0} ({1})".format(dep, dep.__class__.__name__))
def mavenSetup():
maven_args = []
env = os.environ.copy()
if not mx.get_opts().verbose:
maven_args.append('-q')
buildPack = join(_suite.dir, 'jruby-build-pack/maven')
if isdir(buildPack):
maven_args.append('-Dmaven.repo.local=' + buildPack)
elif 'CI' in env and 'TRAVIS' not in env:
maven_args.append('-Dmaven.repo.local=' +
join(_suite.dir, 'mxbuild/mvn'))
if not mx.get_opts().verbose:
env['JRUBY_BUILD_MORE_QUIET'] = 'true'
# HACK: since the maven executable plugin does not configure the
# java executable that is used we unfortunately need to prepend it to the PATH
javaHome = os.getenv('JAVA_HOME')
if javaHome:
env["PATH"] = javaHome + '/bin' + os.pathsep + env["PATH"]
mx.logv('Setting PATH to {}'.format(env["PATH"]))
mx.run(['java', '-version'], env=env)
return maven_args, env
def run_js(vmArgs, jsArgs, nonZeroIsFatal, out, err, cwd):
bench_conf, should_bench_compile_server = _get_bench_conf(vmArgs)
_, _, image_path = _bench_image_params(bench_conf)
if should_bench_compile_server and not exists(image_path):
for _ in range(_IMAGE_BENCH_REPETITIONS):
with mx_substratevm.native_image_context(
config=mx_substratevm._graalvm_js_config,
build_if_missing=True):
_bench_compile_server(bench_conf, out)
image_path = bench_jsimage(bench_conf, out=out, err=err)
if image_path:
vmArgs = [
vmArg for vmArg in vmArgs
if not (vmArg.startswith(_conf_arg_prefix)
or vmArg == '--bench-compilation-server')
]
all_args = vmArgs + jsArgs
mx.logv("Running substratevm image '%s' with '%s' i.e.:" %
(image_path, all_args))
mx.logv(image_path + " " + " ".join(all_args))
runner = mx_substratevm.ProcessRunner([image_path] + all_args,
stderr=subprocess.STDOUT)
timeout_factor, _ = _bench_configs[bench_conf]
with _timedelta('IMAGERUN: ', out=out):
returncode, stdoutdata, _ = runner.run(8 * 60 * timeout_factor)
if runner.timedout:
if nonZeroIsFatal:
mx.abort('Javascript benchmark timeout')
return -1
out(stdoutdata)
return returncode
if nonZeroIsFatal:
mx.abort('Javascript image building for js-benchmarks failed')
return -1
def native_image_layout(dists, subdir, native_image_root, debug_gr_8964=False):
if not dists:
return
dest_path = join(native_image_root, subdir)
# Cleanup leftovers from previous call
if exists(dest_path):
if debug_gr_8964:
mx.log('[mx_substratevm.native_image_layout: remove_tree: ' +
dest_path + ']')
remove_tree(dest_path)
mkpath(dest_path)
# Create symlinks to conform with native-image directory layout scheme
# GR-8964: Copy the jar instead of symlinking to it.
def symlink_jar(jar_path):
if debug_gr_8964:
dest_jar = join(dest_path, basename(jar_path))
if debug_gr_8964:
mx.log('[mx_substratevm.native_image_layout.symlink_jar: copy2' + \
'\n src: ' + jar_path + \
'\n dst: ' + dest_jar)
copy2(jar_path, dest_jar)
dest_stat = os.stat(dest_jar)
if debug_gr_8964:
mx.log(' ' + \
' .st_mode: ' + oct(dest_stat.st_mode) + \
' .st_mtime: ' + time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime(dest_stat.st_mtime)) + ']')
else:
relsymlink(jar_path, join(dest_path, basename(jar_path)))
for dist in dists:
mx.logv('Add ' + type(dist).__name__ + ' ' + str(dist) + ' to ' +
dest_path)
symlink_jar(dist.path)
if not dist.isLibrary():
symlink_jar(dist.sourcesPath)
def _pkgtest_args(args):
graalvm_home = None
if 'FASTR_GRAALVM' in os.environ:
graalvm_home = os.environ['FASTR_GRAALVM']
elif 'GRAALVM_FASTR' in os.environ:
graalvm_home = os.environ['GRAALVM_FASTR']
pkgtest_args = []
pkgtest_args += ["--fastr-home"]
pkgtest_args += [_fastr_suite.dir]
if graalvm_home:
# In GRAALVM mode, we assume FastR is not built so we need to
_gnur_suite = mx.suite('gnur')
pkgtest_args += ["--gnur-home"]
pkgtest_args += [join(_gnur_suite.dir, 'gnur', _gnur_suite.extensions.r_version())]
pkgtest_args += ["--graalvm-home"]
pkgtest_args += [graalvm_home]
else:
pkgtest_args += ["--gnur-home"]
pkgtest_args += [_gnur_path()]
mx.log(args)
full_args = pkgtest_args + list(args)
mx.logv(full_args)
return full_args
def _run(self, *args, **kwargs):
cmd = [self.binary, '-j', self.parallelism]
if mx.get_opts().very_verbose:
cmd += ['-v']
cmd += args
out = kwargs.get('out', mx.OutputCapture())
err = kwargs.get('err', subprocess.STDOUT)
if mx.get_opts().verbose:
if callable(out) and '-n' not in args:
out = mx.TeeOutputCapture(out)
if callable(err):
err = mx.TeeOutputCapture(err)
try:
rc = mx.run(cmd, nonZeroIsFatal=False, out=out, err=err, cwd=self.build_dir)
except OSError as e:
if e.errno != errno.EACCES:
mx.abort('Error executing \'{}\': {}'.format(' '.join(cmd), str(e)))
mx.logv('{} is not executable. Trying to change permissions...'.format(self.binary))
os.chmod(self.binary, 0o755)
self._run(*args, **kwargs) # retry
else:
not rc or mx.abort(rc if mx.get_opts().verbose else out.data) # pylint: disable=expression-not-assigned
def _parseArgs(args):
arguments = list(args)
debugArg = '--debug'
if debugArg in arguments:
mx.log('Running in debug mode. The --debug argument is handled by mx and not passed as program argument')
arguments.remove(debugArg)
mode = 'Debug'
else:
mode = 'Release'
vmArgs, progArgs = parse_js_args(arguments)
if mx.suite('compiler', fatalIfMissing=False):
import mx_compiler
vmArgs = mx_compiler._parseVmArgs(vmArgs)
if '-d64' in vmArgs:
mx.logv('[_parseArgs] removing -d64 from vmArgs')
vmArgs.remove('-d64')
mx.logv('[_parseArgs] mode: %s' % mode)
mx.logv('[_parseArgs] vmArgs: %s' % vmArgs)
mx.logv('[_parseArgs] progArgs: %s' % progArgs)
return mode, vmArgs, progArgs
def _setEnvVar(name, val):
if val:
mx.logv('Setting environment variable %s=%s' % (name, val))
os.environ[name] = val
def jlink_new_jdk(jdk, dst_jdk_dir, module_dists, root_module_names=None, missing_export_target_action='create', with_source=lambda x: True):
"""
Uses jlink from `jdk` to create a new JDK image in `dst_jdk_dir` with `module_dists` and
their dependencies added to the JDK image, replacing any existing modules of the same name.
:param JDKConfig jdk: source JDK
:param str dst_jdk_dir: path to use for the jlink --output option
:param list module_dists: list of distributions defining modules
:param list root_module_names: list of strings naming the module root set for the new JDK image.
The named modules must either be in `module_dists` or in `jdk`. If None, then
the root set will be all the modules in ``module_dists` and `jdk`.
:param str missing_export_target_action: the action to perform for a qualifed export target that
is not present in `module_dists` and does not have a hash stored in java.base.
The choices are:
"create" - an empty module is created
"error" - raise an error
None - do nothing
:param lambda with_source: returns True if the sources of a module distribution must be included in the new JDK
"""
assert callable(with_source)
if jdk.javaCompliance < '9':
mx.abort('Cannot derive a new JDK from ' + jdk.home + ' with jlink since it is not JDK 9 or later')
exploded_java_base_module = join(jdk.home, 'modules', 'java.base')
if exists(exploded_java_base_module):
mx.abort('Cannot derive a new JDK from ' + jdk.home + ' since it appears to be a developer build with exploded modules')
jimage = join(jdk.home, 'lib', 'modules')
jmods_dir = join(jdk.home, 'jmods')
if not isfile(jimage):
mx.abort('Cannot derive a new JDK from ' + jdk.home + ' since ' + jimage + ' is missing or is not an ordinary file')
if not isdir(jmods_dir):
mx.abort('Cannot derive a new JDK from ' + jdk.home + ' since ' + jmods_dir + ' is missing or is not a directory')
jdk_modules = {jmd.name : jmd for jmd in jdk.get_modules()}
modules = [as_java_module(dist, jdk) for dist in module_dists]
all_module_names = frozenset(list(jdk_modules.keys()) + [m.name for m in modules])
# Read hashes stored in java.base (the only module in the JDK where hashes are stored)
out = mx.LinesOutputCapture()
mx.run([jdk.exe_path('jmod'), 'describe', jdk_modules['java.base'].get_jmod_path()], out=out)
lines = out.lines
hashes = {}
for line in lines:
if line.startswith('hashes'):
parts = line.split()
assert len(parts) == 4, 'expected hashes line to have 4 fields, got {} fields: {}'.format(len(parts), line)
_, module_name, algorithm, hash_value = parts
hashes[module_name] = (algorithm, hash_value)
build_dir = mx.ensure_dir_exists(join(dst_jdk_dir + ".build"))
try:
# Handle targets of qualified exports that are not present in `modules`
target_requires = {}
for jmd in modules:
for targets in jmd.exports.values():
for target in targets:
if target not in all_module_names and target not in hashes:
target_requires.setdefault(target, set()).add(jmd.name)
if target_requires and missing_export_target_action is not None:
if missing_export_target_action == 'error':
mx.abort('Target(s) of qualified exports cannot be resolved: ' + '.'.join(target_requires.keys()))
assert missing_export_target_action == 'create', 'invalid value for missing_export_target_action: ' + str(missing_export_target_action)
extra_modules = []
for name, requires in target_requires.items():
module_jar = join(build_dir, name + '.jar')
jmd = JavaModuleDescriptor(name, {}, requires={module : [] for module in requires}, uses=set(), provides={}, jarpath=module_jar)
extra_modules.append(jmd)
module_build_dir = mx.ensure_dir_exists(join(build_dir, name))
module_info_java = join(module_build_dir, 'module-info.java')
module_info_class = join(module_build_dir, 'module-info.class')
with open(module_info_java, 'w') as fp:
print(jmd.as_module_info(), file=fp)
mx.run([jdk.javac, '-d', module_build_dir, \
'--limit-modules=java.base,' + ','.join(jmd.requires.keys()), \
'--module-path=' + os.pathsep.join((m.jarpath for m in modules)), \
module_info_java])
with ZipFile(module_jar, 'w') as zf:
zf.write(module_info_class, basename(module_info_class))
if exists(jmd.get_jmod_path()):
os.remove(jmd.get_jmod_path())
mx.run([jdk.javac.replace('javac', 'jmod'), 'create', '--class-path=' + module_build_dir, jmd.get_jmod_path()])
modules.extend(extra_modules)
all_module_names = frozenset(list(jdk_modules.keys()) + [m.name for m in modules])
# Extract src.zip from source JDK
jdk_src_zip = join(jdk.home, 'lib', 'src.zip')
dst_src_zip_contents = {}
if isfile(jdk_src_zip):
mx.logv('[Extracting ' + jdk_src_zip + ']')
with ZipFile(jdk_src_zip, 'r') as zf:
for name in zf.namelist():
if not name.endswith('/'):
dst_src_zip_contents[name] = zf.read(name)
else:
mx.warn("'{}' does not exist or is not a file".format(jdk_src_zip))
for jmd in modules:
# Remove existing sources for all the modules that we include
dst_src_zip_contents = {key : dst_src_zip_contents[key] for key in dst_src_zip_contents if not key.startswith(jmd.name)}
if with_source(jmd.dist):
# Add the sources that we can share.
# Extract module sources
jmd_src_zip = jmd.jarpath[0:-len('.jar')] + '.src.zip'
if isfile(jmd_src_zip):
mx.logv('[Extracting ' + jmd_src_zip + ']')
with ZipFile(jmd_src_zip, 'r') as zf:
for name in zf.namelist():
if not name.endswith('/'):
dst_src_zip_contents[jmd.name + '/' + name] = zf.read(name)
# Add module-info.java to sources
dst_src_zip_contents[jmd.name + '/module-info.java'] = jmd.as_module_info(extras_as_comments=False)
# Now build the new JDK image with jlink
jlink = [jdk.javac.replace('javac', 'jlink')]
if jdk_enables_jvmci_by_default(jdk):
# On JDK 9+, +EnableJVMCI forces jdk.internal.vm.ci to be in the root set
jlink.append('-J-XX:-EnableJVMCI')
if root_module_names is not None:
missing = frozenset(root_module_names) - all_module_names
if missing:
mx.abort('Invalid module(s): {}.\nAvailable modules: {}'.format(','.join(missing), ','.join(sorted(all_module_names))))
jlink.append('--add-modules=' + ','.join(root_module_names))
else:
jlink.append('--add-modules=' + ','.join(sorted(all_module_names)))
module_path = jmods_dir
if modules:
module_path = os.pathsep.join((m.get_jmod_path(respect_stripping=True) for m in modules)) + os.pathsep + module_path
jlink.append('--module-path=' + module_path)
jlink.append('--output=' + dst_jdk_dir)
# These options are inspired by how OpenJDK runs jlink to produce the final runtime image.
jlink.extend(['-J-XX:+UseSerialGC', '-J-Xms32M', '-J-Xmx512M', '-J-XX:TieredStopAtLevel=1'])
jlink.append('-J-Dlink.debug=true')
jlink.append('--dedup-legal-notices=error-if-not-same-content')
jlink.append('--keep-packaged-modules=' + join(dst_jdk_dir, 'jmods'))
# TODO: investigate the options below used by OpenJDK to see if they should be used:
# --release-info: this allow extra properties to be written to the <jdk>/release file
# --order-resources: specifies order of resources in generated lib/modules file.
# This is apparently not so important if a CDS archive is available.
# --generate-jli-classes: pre-generates a set of java.lang.invoke classes.
# See https://github.com/openjdk/jdk/blob/master/make/GenerateLinkOptData.gmk
mx.logv('[Creating JDK image]')
mx.run(jlink)
dst_src_zip = join(dst_jdk_dir, 'lib', 'src.zip')
mx.logv('[Creating ' + dst_src_zip + ']')
with ZipFile(dst_src_zip, 'w', compression=ZIP_DEFLATED, allowZip64=True) as zf:
for name, contents in sorted(dst_src_zip_contents.items()):
zf.writestr(name, contents)
mx.logv('[Copying static libraries]')
lib_prefix = mx.add_lib_prefix('')
lib_suffix = '.lib' if mx.is_windows() else '.a'
lib_directory = join(jdk.home, 'lib')
dst_lib_directory = join(dst_jdk_dir, 'lib')
for f in os.listdir(lib_directory):
if f.startswith(lib_prefix) and f.endswith(lib_suffix):
lib_path = join(lib_directory, f)
if isfile(lib_path):
shutil.copy2(lib_path, dst_lib_directory)
# Build the list of modules whose classes might have annotations
# to be processed by native-image (GR-15192).
with open(join(dst_jdk_dir, 'lib', 'native-image-modules.list'), 'w') as fp:
print('# Modules whose classes might have annotations processed by native-image', file=fp)
for m in modules:
print(m.name, file=fp)
finally:
if not mx.get_opts().verbose:
# Preserve build directory so that javac command can be re-executed
# by cutting and pasting verbose output.
shutil.rmtree(build_dir)
# Create CDS archive (https://openjdk.java.net/jeps/341).
out = mx.OutputCapture()
mx.logv('[Creating CDS shared archive]')
if mx.run([mx.exe_suffix(join(dst_jdk_dir, 'bin', 'java')), '-Xshare:dump', '-Xmx128M', '-Xms128M'], out=out, err=out, nonZeroIsFatal=False) != 0:
mx.log(out.data)
mx.abort('Error generating CDS shared archive')
def __init__(self,
suite,
name,
short_name,
license_files,
third_party_license_files,
jar_distributions=None,
builder_jar_distributions=None,
support_distributions=None,
support_headers_distributions=None,
support_libraries_distributions=None,
dir_name=None,
launcher_configs=None,
library_configs=None,
provided_executables=None,
polyglot_lib_build_args=None,
polyglot_lib_jar_dependencies=None,
polyglot_lib_build_dependencies=None,
has_polyglot_lib_entrypoints=False,
boot_jars=None,
jvmci_parent_jars=None,
priority=None,
installable=False,
post_install_msg=None,
installable_id=None,
dependencies=None):
"""
:param suite mx.Suite: the suite this component belongs to
:type name: str
:param str short_name: a short, unique name for this component
:param str | None | False dir_name: the directory name in which this component lives. If `None`, the `short_name` is used. If `False`, files are copied to the root-dir for the component type.
:param installable: Produce a distribution installable via `gu`
:param post_install_msg: Post-installation message to be printed
:param list[str] dependencies: a list of component names
:type license_files: list[str]
:type third_party_license_files: list[str]
:type provided_executables: list[str]
:type polyglot_lib_build_args: list[str]
:type polyglot_lib_jar_dependencies: list[str]
:type polyglot_lib_build_dependencies: list[str]
:type has_polyglot_lib_entrypoints: bool
:type boot_jars: list[str]
:type jvmci_parent_jars: list[str]
:type launcher_configs: list[LauncherConfig]
:type library_configs: list[LibraryConfig]
:type jar_distributions: list[str]
:type builder_jar_distributions: list[str]
:type support_distributions: list[str]
:type support_headers_distributions: list[str]
:type support_libraries_distributions: list[str]
:param int priority: priority with a higher value means higher priority
:type installable: bool
:type installable_id: str
:type post_install_msg: str
"""
if dependencies is None:
mx.logv('Component {} does not specify dependencies'.format(name))
self.suite = suite
self.name = name
self.short_name = short_name
self.dir_name = dir_name if dir_name is not None else short_name
self.license_files = license_files
self.third_party_license_files = third_party_license_files
self.dependency_names = dependencies or []
self.provided_executables = provided_executables or []
self.polyglot_lib_build_args = polyglot_lib_build_args or []
self.polyglot_lib_jar_dependencies = polyglot_lib_jar_dependencies or []
self.polyglot_lib_build_dependencies = polyglot_lib_build_dependencies or []
self.has_polyglot_lib_entrypoints = has_polyglot_lib_entrypoints
self.boot_jars = boot_jars or []
self.jvmci_parent_jars = jvmci_parent_jars or []
self.jar_distributions = jar_distributions or []
self.builder_jar_distributions = builder_jar_distributions or []
self.support_distributions = support_distributions or []
self.support_headers_distributions = support_headers_distributions or []
self.support_libraries_distributions = support_libraries_distributions or []
self.priority = priority or 0
self.launcher_configs = launcher_configs or []
self.library_configs = library_configs or []
self.installable = installable
self.post_install_msg = post_install_msg
self.installable_id = installable_id or self.dir_name
assert isinstance(self.jar_distributions, list)
assert isinstance(self.builder_jar_distributions, list)
assert isinstance(self.support_distributions, list)
assert isinstance(self.support_headers_distributions, list)
assert isinstance(self.support_libraries_distributions, list)
assert isinstance(self.license_files, list)
assert isinstance(self.third_party_license_files, list)
assert isinstance(self.provided_executables, list)
assert isinstance(self.polyglot_lib_build_args, list)
assert isinstance(self.polyglot_lib_jar_dependencies, list)
assert isinstance(self.polyglot_lib_build_dependencies, list)
assert isinstance(self.boot_jars, list)
assert isinstance(self.jvmci_parent_jars, list)
assert isinstance(self.launcher_configs, list)
assert isinstance(self.library_configs, list)
def truffle_language_ensure(language_flag,
version=None,
native_image_root=None):
'''
Ensures that we have a valid suite for the given language_flag, by downloading a binary if necessary
and providing the suite distribution artifacts in the native-image directory hierachy (via symlinks).
:param language_flag: native-image language_flag whose truffle-language we want to use
:param version: if not specified and no TRUFFLE_<LANG>_VERSION set latest binary deployed master revision gets downloaded
:param native_image_root: the native_image_root directory where the the artifacts get installed to
:return: language suite for the given language_flag
'''
if not native_image_root:
native_image_root = suite_native_image_root()
version_env_var = 'TRUFFLE_' + language_flag.upper() + '_VERSION'
if not version and os.environ.has_key(version_env_var):
version = os.environ[version_env_var]
if language_flag not in flag_suitename_map:
mx.abort('No truffle-language uses language_flag \'' + language_flag +
'\'')
if language_flag in session_language:
language_suite = session_language[language_flag]
mx.log('Reusing ' + language_flag + '.version=' +
str(language_suite.version()))
return language_suite
language_suite_name = flag_suitename_map[language_flag][0]
# Accessing truffle_language as source suite (--dynamicimports) has priority over binary suite import
language_suite = suite.import_suite(language_suite_name)
if not language_suite:
# Get the truffle_language suite via binary suite import
urlinfos = [
mx.SuiteImportURLInfo(
mx_urlrewrites.rewriteurl(
'https://curio.ssw.jku.at/nexus/content/repositories/snapshots'
), 'binary', mx.vc_system('binary'))
]
if not version:
# If no specific version requested use binary import of last recently deployed master version
version = 'git-bref:binary'
urlinfos.append(
mx.SuiteImportURLInfo(
mx_urlrewrites.rewriteurl(
'https://github.com/graalvm/{0}.git'.format(
language_suite_name)), 'source',
mx.vc_system('git')))
try:
language_suite = suite.import_suite(language_suite_name,
version=version,
urlinfos=urlinfos,
kind=None)
except (urllib2.URLError, SystemExit):
language_suite = suite.import_suite(language_suite_name)
if language_suite:
if version and session_language[
language_flag] and session_language[
language_flag].version() != version:
mx.abort('Cannot switch to ' + language_flag +
'.version=' + str(version) +
' without maven access.')
else:
mx.log('No maven access. Using already downloaded ' +
language_suite_name + ' binary suite.')
else:
mx.abort('No maven access and no local copy of ' +
language_suite_name + ' binary suite available.')
if not language_suite:
mx.abort('Binary suite not found and no local copy of ' +
language_suite_name + ' available.')
language_dir = join('languages', language_flag)
language_suite_depnames = flag_suitename_map[language_flag][1]
language_deps = [
dep for dep in language_suite.dists + language_suite.libs
if dep.name in language_suite_depnames
]
native_image_layout(language_deps, language_dir, native_image_root)
language_suite_nativedistnames = flag_suitename_map[language_flag][2]
language_nativedists = [
dist for dist in language_suite.dists
if dist.name in language_suite_nativedistnames
]
native_image_extract(language_nativedists, language_dir, native_image_root)
option_properties = join(language_suite.mxDir, 'native-image.properties')
target_path = join(native_image_root, language_dir,
'native-image.properties')
if islink(target_path):
os.remove(target_path)
if exists(option_properties):
mx.logv('Add symlink to ' + str(option_properties))
relsymlink(option_properties, target_path)
else:
native_image_option_properties('languages', language_flag,
native_image_root)
session_language[language_flag] = language_suite
return language_suite
def bootstrap_native_image(native_image_root, svmDistribution,
graalDistribution, librarySupportDistribution):
if not allow_native_image_build:
mx.logv(
'Detected building with ejc + --warning-as-error -> suppress bootstrap_native_image'
)
return
bootstrap_command = list(GRAAL_COMPILER_FLAGS)
bootstrap_command += locale_US_args()
bootstrap_command += substratevm_version_args()
bootstrap_command += ['-Dgraalvm.version=dev']
builder_classpath = classpath(svmDistribution)
imagecp_classpath = classpath(svmDistribution + ['substratevm:SVM_DRIVER'])
bootstrap_command += [
'-cp', builder_classpath,
'com.oracle.svm.hosted.NativeImageGeneratorRunner', '-imagecp',
imagecp_classpath, '-H:CLibraryPath=' + clibrary_libpath()
]
bootstrap_command += [
'-H:Path=' + dirname(native_image_path(native_image_root)),
'-H:Class=com.oracle.svm.driver.NativeImage',
'-H:Name=' + basename(native_image_path(native_image_root)),
'-H:-TruffleFeature', '-H:-ParseRuntimeOptions',
'-H:ReflectionConfigurationResources=com/oracle/svm/driver/ReflectionConfiguration.json'
]
if mx._opts.strip_jars:
bootstrap_command += ['-H:-VerifyNamingConventions']
run_java(bootstrap_command)
mx.logv('Built ' + native_image_path(native_image_root))
def native_image_layout_dists(subdir, dists):
native_image_dists = [mx.dependency(dist_name) for dist_name in dists]
native_image_layout(native_image_dists, subdir, native_image_root)
# Create native-image layout for sdk parts
native_image_layout_dists(join('lib', 'boot'), ['sdk:GRAAL_SDK'])
native_image_layout_dists(join('lib', 'graalvm'), ['sdk:LAUNCHER_COMMON'])
# Create native-image layout for compiler & jvmci parts
native_image_layout_dists(join('lib', 'jvmci'), graalDistribution)
jdk_config = mx.get_jdk()
jvmci_path = join(jdk_config.home, 'jre', 'lib', 'jvmci')
for symlink_name in os.listdir(jvmci_path):
relsymlink(join(jvmci_path, symlink_name),
join(native_image_root, 'lib', 'jvmci', symlink_name))
# Create native-image layout for truffle parts
native_image_layout_dists(join('lib', 'truffle'), ['truffle:TRUFFLE_API'])
# Create native-image layout for tools parts
native_image_layout_dists(join('tools', 'junit', 'builder'),
['mx:JUNIT_TOOL', 'JUNIT', 'HAMCREST'])
native_image_option_properties('tools', 'junit', native_image_root)
native_image_option_properties('tools', 'truffle', native_image_root)
native_image_layout_dists(join('tools', 'nfi', 'builder'),
['truffle:TRUFFLE_NFI'])
native_image_option_properties('tools', 'nfi', native_image_root)
# Create native-image layout for svm parts
svm_subdir = join('lib', 'svm')
native_image_layout_dists(svm_subdir, librarySupportDistribution)
native_image_layout_dists(
join(svm_subdir, 'builder'),
svmDistribution + ['substratevm:POINTSTO', 'substratevm:OBJECTFILE'])
for clibrary_path in clibrary_paths():
copy_tree(clibrary_path,
join(native_image_root, join(svm_subdir, 'clibraries')))
# Finally create symlink to native-image in svm_suite().dir
native_image_symlink_path = join(
svm_suite().dir, basename(native_image_path(native_image_root)))
if islink(native_image_symlink_path):
os.remove(native_image_symlink_path)
relsymlink(native_image_path(native_image_root), native_image_symlink_path)
def _setEnvVar(name, val, env=None):
_env = env or os.environ
if val:
mx.logv('Setting environment variable %s=%s' % (name, val))
_env[name] = val
def truffle_language_ensure(language_flag, version=None, native_image_root=None, early_exit=False, extract=True, debug_gr_8964=False):
"""
Ensures that we have a valid suite for the given language_flag, by downloading a binary if necessary
and providing the suite distribution artifacts in the native-image directory hierachy (via symlinks).
:param language_flag: native-image language_flag whose truffle-language we want to use
:param version: if not specified and no TRUFFLE_<LANG>_VERSION set latest binary deployed master revision gets downloaded
:param native_image_root: the native_image_root directory where the the artifacts get installed to
:return: language suite for the given language_flag
"""
if not native_image_root:
native_image_root = suite_native_image_root()
version_env_var = 'TRUFFLE_' + language_flag.upper() + '_VERSION'
if not version and os.environ.has_key(version_env_var):
version = os.environ[version_env_var]
if language_flag not in flag_suitename_map:
mx.abort('No truffle-language uses language_flag \'' + language_flag + '\'')
language_dir = join('languages', language_flag)
if early_exit and exists(join(native_image_root, language_dir)):
mx.logv('Early exit mode: Language subdir \'' + language_flag + '\' exists. Skip suite.import_suite.')
return None
language_entry = flag_suitename_map[language_flag]
language_suite_name = language_entry[0]
language_repo_name = language_entry[3] if len(language_entry) > 3 else None
urlinfos = [
mx.SuiteImportURLInfo(mx_urlrewrites.rewriteurl('https://curio.ssw.jku.at/nexus/content/repositories/snapshots'),
'binary',
mx.vc_system('binary'))
]
failure_warning = None
if not version and not mx.suite(language_suite_name, fatalIfMissing=False):
# If no specific version requested use binary import of last recently deployed master version
repo_suite_name = language_repo_name if language_repo_name else language_suite_name
repo_url = mx_urlrewrites.rewriteurl('https://github.com/graalvm/{0}.git'.format(repo_suite_name))
version = mx.SuiteImport.resolve_git_branchref(repo_url, 'binary', abortOnError=False)
if not version:
failure_warning = 'Resolving \'binary\' against ' + repo_url + ' failed'
language_suite = suite.import_suite(
language_suite_name,
version=version,
urlinfos=urlinfos,
kind=None,
in_subdir=bool(language_repo_name)
)
if not language_suite:
if failure_warning:
mx.warn(failure_warning)
mx.abort('Binary suite not found and no local copy of ' + language_suite_name + ' available.')
if not extract:
if not exists(join(native_image_root, language_dir)):
mx.abort('Language subdir \'' + language_flag + '\' should already exist with extract=False')
return language_suite
language_suite_depnames = language_entry[1]
language_deps = language_suite.dists + language_suite.libs
language_deps = [dep for dep in language_deps if dep.name in language_suite_depnames]
native_image_layout(language_deps, language_dir, native_image_root, debug_gr_8964=debug_gr_8964)
language_suite_nativedistnames = language_entry[2]
language_nativedists = [dist for dist in language_suite.dists if dist.name in language_suite_nativedistnames]
native_image_extract(language_nativedists, language_dir, native_image_root)
option_properties = join(language_suite.mxDir, 'native-image.properties')
target_path = remove_existing_symlink(join(native_image_root, language_dir, 'native-image.properties'))
if exists(option_properties):
if not exists(target_path):
mx.logv('Add symlink to ' + str(option_properties))
symlink_or_copy(option_properties, target_path, debug_gr_8964=debug_gr_8964)
else:
native_image_option_properties('languages', language_flag, native_image_root)
return language_suite
def _replace_host_vm(key):
host_vm = dims.get("host-vm")
if host_vm and host_vm.startswith(key):
dims['host-vm'] = key
mx.logv("[DEBUG] replace 'host-vm': '{key}-python' -> '{key}'".format(key=key))
def truffle_language_ensure(language_flag,
version=None,
native_image_root=None,
early_exit=False,
extract=True):
"""
Ensures that we have a valid suite for the given language_flag, by downloading a binary if necessary
and providing the suite distribution artifacts in the native-image directory hierachy (via symlinks).
:param language_flag: native-image language_flag whose truffle-language we want to use
:param version: if not specified and no TRUFFLE_<LANG>_VERSION set latest binary deployed master revision gets downloaded
:param native_image_root: the native_image_root directory where the the artifacts get installed to
:return: language suite for the given language_flag
"""
if not native_image_root:
native_image_root = suite_native_image_root()
version_env_var = 'TRUFFLE_' + language_flag.upper() + '_VERSION'
if not version and os.environ.has_key(version_env_var):
version = os.environ[version_env_var]
if language_flag not in flag_suitename_map:
mx.abort('No truffle-language uses language_flag \'' + language_flag +
'\'')
language_dir = join('languages', language_flag)
if early_exit and exists(join(native_image_root, language_dir)):
mx.logv('Early exit mode: Language subdir \'' + language_flag +
'\' exists. Skip suite.import_suite.')
return None
language_entry = flag_suitename_map[language_flag]
language_suite_name = language_entry[0]
language_repo_name = language_entry[3] if len(language_entry) > 3 else None
urlinfos = [
mx.SuiteImportURLInfo(
mx_urlrewrites.rewriteurl(
'https://curio.ssw.jku.at/nexus/content/repositories/snapshots'
), 'binary', mx.vc_system('binary'))
]
failure_warning = None
if not version and not mx.suite(language_suite_name, fatalIfMissing=False):
# If no specific version requested use binary import of last recently deployed master version
repo_suite_name = language_repo_name if language_repo_name else language_suite_name
repo_url = mx_urlrewrites.rewriteurl(
'https://github.com/graalvm/{0}.git'.format(repo_suite_name))
version = mx.SuiteImport.resolve_git_branchref(repo_url,
'binary',
abortOnError=False)
if not version:
failure_warning = 'Resolving \'binary\' against ' + repo_url + ' failed'
language_suite = suite.import_suite(language_suite_name,
version=version,
urlinfos=urlinfos,
kind=None,
in_subdir=bool(language_repo_name))
if not language_suite:
if failure_warning:
mx.warn(failure_warning)
mx.abort('Binary suite not found and no local copy of ' +
language_suite_name + ' available.')
if not extract:
if not exists(join(native_image_root, language_dir)):
mx.abort('Language subdir \'' + language_flag +
'\' should already exist with extract=False')
return language_suite
language_suite_depnames = language_entry[1]
language_deps = language_suite.dists + language_suite.libs
language_deps = [
dep for dep in language_deps if dep.name in language_suite_depnames
]
native_image_layout(language_deps, language_dir, native_image_root)
language_suite_nativedistnames = language_entry[2]
language_nativedists = [
dist for dist in language_suite.dists
if dist.name in language_suite_nativedistnames
]
native_image_extract(language_nativedists, language_dir, native_image_root)
option_properties = join(language_suite.mxDir, 'native-image.properties')
target_path = remove_existing_symlink(
join(native_image_root, language_dir, 'native-image.properties'))
if exists(option_properties):
if not exists(target_path):
mx.logv('Add symlink to ' + str(option_properties))
symlink_or_copy(option_properties, target_path)
else:
native_image_option_properties('languages', language_flag,
native_image_root)
return language_suite
def find_top(*args, **kwargs):
full_args = _pkgtest_args(args)
mx.logv(["find_top"] + full_args)
return pkgtest_load().find_top(args)
def build(self):
source_dir = self.subject.getSourceDir()
output_dir = self.subject.getOutputDir()
if not emcc_dir:
mx.abort("No EMCC_DIR specified - the source programs will not be compiled to .wasm.")
emcc_cmd = os.path.join(emcc_dir, "emcc")
gcc_cmd = os.path.join(gcc_dir, "gcc")
if mx.run([emcc_cmd, "-v"], nonZeroIsFatal=False) != 0:
mx.abort("Could not check the emcc version.")
if mx.run([gcc_cmd, "--version"], nonZeroIsFatal=False) != 0:
mx.abort("Could not check the gcc version.")
if not wabt_dir:
mx.abort("Set WABT_DIR if you want the binary to include .wat files.")
mx.log("Building files from the source dir: " + source_dir)
cc_flags = ["-O3", "-g2"]
include_flags = []
disable_test_api_flags = ["-DDISABLE_TEST_API"]
if hasattr(self.project, "includeset"):
include_flags = ["-I", os.path.join(_suite.dir, "includes", self.project.includeset)]
emcc_flags = cc_flags
if self.project.isBenchmarkProject():
emcc_flags = emcc_flags + ["-s", "EXPORTED_FUNCTIONS=" + str(benchmark_methods).replace("'", "\"") + ""]
subdir_program_names = defaultdict(lambda: [])
for root, filename in self.subject.getProgramSources():
subdir = os.path.relpath(root, self.subject.getSourceDir())
mx.ensure_dir_exists(os.path.join(output_dir, subdir))
basename = remove_extension(filename)
source_path = os.path.join(root, filename)
output_wasm_path = os.path.join(output_dir, subdir, basename + ".wasm")
timestampedSource = mx.TimeStampFile(source_path)
timestampedOutput = mx.TimeStampFile(output_wasm_path)
mustRebuild = timestampedSource.isNewerThan(timestampedOutput) or not timestampedOutput.exists()
# Step 1: build the .wasm binary.
if mustRebuild:
if filename.endswith(".c"):
# Step 1a: compile with the JS file, and store as files for running Node, if necessary.
output_js_path = os.path.join(output_dir, subdir, basename + ".js")
build_cmd_line = [emcc_cmd] + emcc_flags + disable_test_api_flags + [source_path, "-o", output_js_path] + include_flags
if mx.run(build_cmd_line, nonZeroIsFatal=False) != 0:
mx.abort("Could not build the JS output of " + filename + " with emcc.")
if self.subject.isBenchmarkProject():
node_dir = os.path.join(output_dir, subdir, NODE_BENCH_DIR)
mx.ensure_dir_exists(node_dir)
shutil.copyfile(output_js_path, os.path.join(node_dir, basename + ".js"))
shutil.copyfile(output_wasm_path, os.path.join(node_dir, basename + ".wasm"))
# Step 1b: extract the relevant information out of the JS file, and record it into an initialization file.
init_info = self.extractInitialization(output_js_path)
with open(os.path.join(output_dir, subdir, basename + ".init"), "w") as f:
f.write(init_info)
# Step 1c: compile to just a .wasm file, to avoid name mangling.
build_cmd_line = [emcc_cmd] + emcc_flags + ["-s", "ERROR_ON_UNDEFINED_SYMBOLS=0"] + [source_path, "-o", output_wasm_path] + include_flags
if mx.run(build_cmd_line, nonZeroIsFatal=False) != 0:
mx.abort("Could not build the wasm-only output of " + filename + " with emcc.")
elif filename.endswith(".wat"):
# Step 1: compile the .wat file to .wasm.
wat2wasm_cmd = os.path.join(wabt_dir, "wat2wasm")
build_cmd_line = [wat2wasm_cmd, "-o", output_wasm_path, source_path]
if mx.run(build_cmd_line, nonZeroIsFatal=False) != 0:
mx.abort("Could not translate " + filename + " to binary format.")
else:
mx.logv("skipping, file is up-to-date: " + source_path)
# Step 2: copy the result file if it exists.
result_path = os.path.join(root, basename + ".result")
if os.path.isfile(result_path):
result_output_path = os.path.join(output_dir, subdir, basename + ".result")
shutil.copyfile(result_path, result_output_path)
# Step 3: copy the opts file if it exists.
opts_path = os.path.join(root, basename + ".opts")
if os.path.isfile(opts_path):
opts_output_path = os.path.join(output_dir, subdir, basename + ".opts")
shutil.copyfile(opts_path, opts_output_path)
output_wat_path = os.path.join(output_dir, subdir, basename + ".wat")
if mustRebuild:
if filename.endswith(".c"):
# Step 4: produce the .wat files, for easier debugging.
wasm2wat_cmd = os.path.join(wabt_dir, "wasm2wat")
if mx.run([wasm2wat_cmd, "-o", output_wat_path, output_wasm_path], nonZeroIsFatal=False) != 0:
mx.abort("Could not compile .wat file for " + filename)
elif filename.endswith(".wat"):
# Step 4: copy the .wat file, for easier debugging.
wat_path = os.path.join(root, basename + ".wat")
shutil.copyfile(wat_path, output_wat_path)
# Step 5: if this is a benchmark project, create native binaries too.
if mustRebuild:
mx.ensure_dir_exists(os.path.join(output_dir, subdir, NATIVE_BENCH_DIR))
if filename.endswith(".c"):
output_path = os.path.join(output_dir, subdir, NATIVE_BENCH_DIR, mx.exe_suffix(basename))
link_flags = ["-lm"]
gcc_cmd_line = [gcc_cmd] + cc_flags + disable_test_api_flags + [source_path, "-o", output_path] + include_flags + link_flags
if mx.run(gcc_cmd_line, nonZeroIsFatal=False) != 0:
mx.abort("Could not build the native binary of " + filename + ".")
os.chmod(output_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
elif filename.endswith(".wat"):
mx.warn("The .wat files are not translated to native binaries: " + filename)
# Remember the source name.
subdir_program_names[subdir].append(basename)
for subdir in subdir_program_names:
with open(os.path.join(output_dir, subdir, "wasm_test_index"), "w") as f:
for name in subdir_program_names[subdir]:
f.write(name)
f.write("\n")
def build(self):
pre_ts = GraalNodeJsBuildTask._get_newest_ts(self.subject.getResults(),
fatalIfMissing=False)
build_env = os.environ.copy()
_setEnvVar(
'PATH', '%s%s%s' %
(join(_suite.mxDir, 'python2'), pathsep, build_env['PATH']),
build_env)
debug = ['--debug'] if self._debug_mode else []
shared_library = ['--enable-shared-library'] if hasattr(
self.args, 'sharedlibrary') and self.args.sharedlibrary else []
newest_config_file_ts = GraalNodeJsBuildTask._get_newest_ts(
_config_files, fatalIfMissing=True)
newest_generated_config_file_ts = GraalNodeJsBuildTask._get_newest_ts(
_generated_config_files, fatalIfMissing=False)
# Lazily generate config files only if `configure` and `configure.py` are older than the files they generate.
# If we don't do this, the `Makefile` always considers `config.gypi` out of date, triggering a second, unnecessary configure.
lazy_generator = ['--lazy-generator'
] if newest_generated_config_file_ts.isNewerThan(
newest_config_file_ts) else []
if _is_windows:
processDevkitRoot(env=build_env)
_setEnvVar(
'PATH',
pathsep.join([build_env['PATH']] + [
mx.library(lib_name).get_path(True)
for lib_name in ('NASM', 'NINJA')
]), build_env)
extra_flags = [
'--ninja', '--dest-cpu=x64', '--without-etw',
'--without-snapshot'
]
else:
extra_flags = []
_mxrun(python_cmd() + [
join(_suite.dir,
'configure'), '--partly-static', '--without-dtrace',
'--without-snapshot', '--without-node-snapshot',
'--without-node-code-cache', '--java-home',
_java_home(forBuild=True)
] + debug + shared_library + lazy_generator + extra_flags,
cwd=_suite.dir,
print_cmd=True,
env=build_env)
quiet_build = mx.is_continuous_integration(
) and not mx.get_opts().verbose
if _is_windows:
verbose = ['-v'] if mx.get_opts().verbose else []
# The custom env is not used to resolve the location of the executable
_mxrun([join(mx.library('NINJA').get_path(True), 'ninja.exe')] +
verbose +
['-j%d' % self.parallelism, '-C', self._build_dir],
print_cmd=True,
quiet_if_successful=quiet_build,
env=build_env)
else:
verbose = 'V={}'.format('1' if mx.get_opts().verbose else '')
_mxrun([mx.gmake_cmd(),
'-j%d' % self.parallelism, verbose],
cwd=_suite.dir,
print_cmd=True,
quiet_if_successful=quiet_build,
env=build_env)
# put headers for native modules into out/headers
_setEnvVar('HEADERS_ONLY', '1', build_env)
_mxrun(python_cmd() + [
join('tools', 'install.py'), 'install',
join('out', 'headers'), sep
],
quiet_if_successful=not mx.get_opts().verbose,
env=build_env)
post_ts = GraalNodeJsBuildTask._get_newest_ts(
self.subject.getResults(), fatalIfMissing=True)
mx.logv(
'Newest time-stamp before building: {}\nNewest time-stamp after building: {}\nHas built? {}'
.format(pre_ts, post_ts, post_ts.isNewerThan(pre_ts)))
built = post_ts.isNewerThan(pre_ts)
if built and _current_os == 'darwin':
nodePath = join(self._build_dir, 'node')
_mxrun([
'install_name_tool', '-add_rpath',
join(_java_home(), 'jre', 'lib'), '-add_rpath',
join(_java_home(), 'lib'), nodePath
],
print_cmd=True,
env=build_env)
return built
def testdownstream(suite, repoUrls, relTargetSuiteDir, mxCommands, branch=None):
"""
Tests a downstream repo against the current working directory state of `suite`.
:param mx.Suite suite: the suite to test against the downstream repo
:param list repoUrls: URLs of downstream repos to clone, the first of which is the repo being tested
:param str relTargetSuiteDir: directory of the downstream suite to test relative to the top level
directory of the downstream repo being tested
:param list mxCommands: argument lists for the mx commands run in downstream suite being tested
:param str branch: name of branch to look for in downstream repo(s)
"""
assert len(repoUrls) > 0
workDir = join(suite.get_output_root(), 'testdownstream')
# A mirror of each suites in the same repo as `suite` is created via copying
rel_mirror = os.path.relpath(suite.dir, mx.SuiteModel.siblings_dir(suite.dir))
in_subdir = os.sep in rel_mirror
suites_in_repo = [suite]
if in_subdir:
base = os.path.dirname(suite.dir)
for e in os.listdir(base):
candidate = join(base, e)
if candidate != suite.dir:
mxDir = mx._is_suite_dir(candidate)
if mxDir:
matches = [s for s in mx.suites() if s.dir == candidate]
if len(matches) == 0:
suites_in_repo.append(mx.SourceSuite(mxDir, primary=False, load=False))
else:
suites_in_repo.append(matches[0])
for suite_in_repo in suites_in_repo:
if suite_in_repo.vc_dir and suite_in_repo.dir != suite_in_repo.vc_dir:
mirror = join(workDir, basename(suite_in_repo.vc_dir), suite_in_repo.name)
else:
mirror = join(workDir, suite_in_repo.name)
if exists(mirror):
shutil.rmtree(mirror)
output_root = suite_in_repo.get_output_root()
def ignore_output_root(d, names):
mx.log('Copying ' + d)
if d == os.path.dirname(output_root):
mx.log('Omitting ' + output_root)
return [os.path.basename(output_root)]
return []
shutil.copytree(suite_in_repo.dir, mirror, ignore=ignore_output_root)
targetDir = None
for repoUrl in repoUrls:
# Deduce a target name from the target URL
url = urlparse(repoUrl)
targetName = url.path
if targetName.rfind('/') != -1:
targetName = targetName[targetName.rfind('/') + 1:]
if targetName.endswith('.git'):
targetName = targetName[0:-len('.git')]
repoWorkDir = join(workDir, targetName)
git = mx.GitConfig()
if exists(repoWorkDir):
git.pull(repoWorkDir)
else:
git.clone(repoUrl, repoWorkDir)
# See if there's a matching (non-master) branch and use it if there is
if not branch:
branch = git.git_command(suite.dir, ['rev-parse', '--abbrev-ref', 'HEAD']).strip()
if branch != 'master':
git.git_command(repoWorkDir, ['checkout', branch], abortOnError=False)
if not targetDir:
targetDir = repoWorkDir
assert not isabs(relTargetSuiteDir)
targetSuiteDir = join(targetDir, relTargetSuiteDir)
assert targetSuiteDir.startswith(targetDir)
mxpy = None if suite != mx._mx_suite else join(mirror, 'mx.py')
for command in mxCommands:
mx.logv('[running "mx ' + ' '.join(command) + '" in ' + targetSuiteDir + ']')
mx.run_mx(command, targetSuiteDir, mxpy=mxpy)
def testdownstream(suite, repoUrls, relTargetSuiteDir, mxCommands, branch=None):
"""
Tests a downstream repo against the current working directory state of `suite`.
:param mx.Suite suite: the suite to test against the downstream repo
:param list repoUrls: URLs of downstream repos to clone, the first of which is the repo being tested
:param str relTargetSuiteDir: directory of the downstream suite to test relative to the top level
directory of the downstream repo being tested
:param list mxCommands: argument lists for the mx commands run in downstream suite being tested
:param str branch: name of branch to look for in downstream repo(s)
"""
assert len(repoUrls) > 0
workDir = join(suite.get_output_root(), 'testdownstream')
# A mirror of the current suite is created with symlinks
mirror = join(workDir, suite.name)
if exists(mirror):
shutil.rmtree(mirror)
mx.ensure_dir_exists(mirror)
for f in os.listdir(suite.dir):
subDir = join(suite.dir, f)
if subDir == suite.get_output_root():
continue
src = join(suite.dir, f)
dst = join(mirror, f)
mx.logv('[Creating symlink from {} to {}]'.format(dst, src))
relsrc = os.path.relpath(src, os.path.dirname(dst))
os.symlink(relsrc, dst)
targetDir = None
for repoUrl in repoUrls:
# Deduce a target name from the target URL
url = urlparse(repoUrl)
targetName = url.path
if targetName.rfind('/') != -1:
targetName = targetName[targetName.rfind('/') + 1:]
if targetName.endswith('.git'):
targetName = targetName[0:-len('.git')]
repoWorkDir = join(workDir, targetName)
git = mx.GitConfig()
if exists(repoWorkDir):
git.pull(repoWorkDir)
else:
git.clone(repoUrl, repoWorkDir)
# See if there's a matching (non-master) branch downstream and use it if there is
if not branch:
branch = git.git_command(suite.dir, ['rev-parse', '--abbrev-ref', 'HEAD']).strip()
if branch != 'master':
git.git_command(repoWorkDir, ['checkout', branch], abortOnError=False)
if not targetDir:
targetDir = repoWorkDir
assert not isabs(relTargetSuiteDir)
targetSuiteDir = join(targetDir, relTargetSuiteDir)
assert targetSuiteDir.startswith(targetDir)
mxpy = None if suite != mx._mx_suite else join(mirror, 'mx.py')
for command in mxCommands:
mx.logv('[running "mx ' + ' '.join(command) + '" in ' + targetSuiteDir + ']')
mx.run_mx(command, targetSuiteDir, mxpy=mxpy)
def jdkartifactstats(args):
"""show stats about JDK deployed Graal artifacts"""
artifacts = {}
jdkDir = get_jvmci_jdk().home
def _getDeployedJars():
if JVMCI_VERSION < 9:
for root, _, filenames in os.walk(join(jdkDir, "jre", "lib")):
for f in filenames:
if f.endswith(".jar") and not f.endswith(".stripped.jar"):
yield join(root, f)
else:
for jdkDist in jdkDeployedDists:
dist = jdkDist.dist()
if isinstance(jdkDist, JvmciJDKDeployedDist):
yield dist.path
for jar in _getDeployedJars():
f = basename(jar)
if "truffle" in f:
if "enterprise" in f:
artifacts.setdefault("GraalEnterpriseTruffle", []).append(jar)
else:
artifacts.setdefault("GraalTruffle", []).append(jar)
elif "enterprise" in f:
artifacts.setdefault("GraalEnterprise", []).append(jar)
elif "jvmci" in f:
artifacts.setdefault("JVMCI", []).append(jar)
elif "graal" in f:
artifacts.setdefault("Graal", []).append(jar)
else:
mx.logv("ignored: " + jar)
print "{:>10} {:>10} {:>10} {}".format("All", "NoVars", "None", "Jar")
for category in sorted(artifacts.viewkeys()):
jars = artifacts[category]
if jars:
totals = (0, 0, 0)
print
for j in jars:
gSize = os.path.getsize(j)
stripped = j[: -len(".jar")] + ".stripped.jar"
mx.run(
[
mx.get_jdk().pack200,
"--repack",
"--quiet",
"-J-Djava.util.logging.config.file=",
"-DLocalVariableTypeTable=strip",
"-DLocalVariableTable=strip",
stripped,
j,
]
)
gLinesSourceSize = os.path.getsize(stripped)
mx.run(
[
mx.get_jdk().pack200,
"--repack",
"--quiet",
"-J-Djava.util.logging.config.file=",
"-G",
stripped,
j,
]
)
gNoneSize = os.path.getsize(stripped)
os.remove(stripped)
print "{:10,} {:10,} {:10,} {}:{}".format(gSize, gLinesSourceSize, gNoneSize, category, basename(j))
t1, t2, t3 = totals
totals = (t1 + gSize, t2 + gLinesSourceSize, t3 + gNoneSize)
t1, t2, t3 = totals
print "{:10,} {:10,} {:10,} {}".format(t1, t2, t3, category)
jvmLib = join(jdkDir, relativeVmLibDirInJdk(), get_vm(), mx.add_lib_suffix(mx.add_lib_prefix("jvm")))
print
if exists(jvmLib):
print "{:10,} {}".format(os.path.getsize(jvmLib), jvmLib)
else:
print "{:>10} {}".format("<missing>", jvmLib)