def java_base_unittest(args):
"""tests whether graal compiler runs on JDK9 with limited set of modules"""
jlink = mx.exe_suffix(join(jdk.home, 'bin', 'jlink'))
if not exists(jlink):
raise mx.JDKConfigException('jlink tool does not exist: ' + jlink)
basejdk_dir = join(_suite.get_output_root(), 'jdkbase')
basemodules = 'java.base,jdk.internal.vm.ci,jdk.unsupported'
if exists(basejdk_dir):
shutil.rmtree(basejdk_dir)
mx.run([jlink, '--output', basejdk_dir, '--add-modules', basemodules, '--module-path', join(jdk.home, 'jmods')])
jdwp = mx.add_lib_suffix(mx.add_lib_prefix('jdwp'))
shutil.copy(join(jdk.home, 'lib', jdwp), join(basejdk_dir, 'lib', jdwp))
dt_socket = mx.add_lib_suffix(mx.add_lib_prefix('dt_socket'))
shutil.copy(join(jdk.home, 'lib', dt_socket), join(basejdk_dir, 'lib', dt_socket))
if not args:
args = []
fakeJavac = join(basejdk_dir, 'bin', 'javac')
open(fakeJavac, 'a').close()
basejdk = mx.JDKConfig(basejdk_dir)
savedJava = jdk.java
try:
jdk.java = basejdk.java
if mx_gate.Task.verbose:
extra_args = ['--verbose', '--enable-timing']
else:
extra_args = []
mx_unittest.unittest(['--suite', 'compiler', '--fail-fast'] + extra_args + args)
finally:
jdk.java = savedJava
def dragonEggPath():
if 'DRAGONEGG' in os.environ:
return join(os.environ['DRAGONEGG'], mx.add_lib_suffix('dragonegg'))
if 'DRAGONEGG_GCC' in os.environ:
path = join(os.environ['DRAGONEGG_GCC'], 'lib', mx.add_lib_suffix('dragonegg'))
if os.path.exists(path):
return path
return None
def dragonEggPath():
if 'DRAGONEGG' in os.environ:
return join(os.environ['DRAGONEGG'], mx.add_lib_suffix('dragonegg'))
if 'DRAGONEGG_GCC' in os.environ:
path = join(os.environ['DRAGONEGG_GCC'], 'lib', mx.add_lib_suffix('dragonegg'))
if os.path.exists(path):
return path
return None
def hsdis(args, copyToDir=None):
"""download the hsdis library
This is needed to support HotSpot's assembly dumping features.
By default it downloads the Intel syntax version, use the 'att' argument to install AT&T syntax."""
flavor = 'intel'
if 'att' in args:
flavor = 'att'
if mx.get_arch() == "sparcv9":
flavor = "sparcv9"
lib = mx.add_lib_suffix('hsdis-' + mx.get_arch())
path = join(_suite.dir, 'lib', lib)
sha1s = {
'att/hsdis-amd64.dll' : 'bcbd535a9568b5075ab41e96205e26a2bac64f72',
'att/hsdis-amd64.so' : '58919ba085d4ef7a513f25bae75e7e54ee73c049',
'intel/hsdis-amd64.dll' : '6a388372cdd5fe905c1a26ced614334e405d1f30',
'intel/hsdis-amd64.so' : '844ed9ffed64fe9599638f29a8450c50140e3192',
'intel/hsdis-amd64.dylib' : 'fdb13ef0d7d23d93dacaae9c98837bea0d4fc5a2',
'sparcv9/hsdis-sparcv9.so': '970640a9af0bd63641f9063c11275b371a59ee60',
}
flavoredLib = flavor + "/" + lib
if flavoredLib not in sha1s:
mx.logv("hsdis not supported on this plattform or architecture")
return
if not exists(path):
sha1 = sha1s[flavoredLib]
sha1path = path + '.sha1'
mx.download_file_with_sha1('hsdis', path, ['https://lafo.ssw.uni-linz.ac.at/pub/hsdis/' + flavoredLib], sha1, sha1path, True, True, sources=False)
if copyToDir is not None and exists(copyToDir):
shutil.copy(path, copyToDir)
def get_output_binary_name(self):
if self.type == SVMImageProject.Type.SHARED_LIBRARY:
return mx.add_lib_suffix(self.get_image_builder_binary_name())
elif self.type == SVMImageProject.Type.EXECUTABLE:
return mx.exe_suffix(self.get_image_builder_binary_name())
else:
mx.abort("Unsupported type: " + str(self.type))
def getBuildEnv(self, replaceVar=mx_subst.path_substitutions):
env = super(SulongTestSuite, self).getBuildEnv(replaceVar=replaceVar)
env['VPATH'] = os.path.join(self.dir, self.name)
env['PROJECT'] = self.name
env['TESTS'] = ' '.join(self.getTests())
env['VARIANTS'] = ' '.join(self.getVariants())
env['BUILD_REF'] = '1' if self.buildRef else '0'
env['BUILD_SO'] = '1' if self.buildSharedObject else '0'
env['SO_EXT'] = mx.add_lib_suffix("")
env['CLANG'] = mx_sulong.findBundledLLVMProgram('clang')
env['CLANGXX'] = mx_sulong.findBundledLLVMProgram('clang++')
env['LLVM_OPT'] = mx_sulong.findBundledLLVMProgram('opt')
env['LLVM_AS'] = mx_sulong.findBundledLLVMProgram('llvm-as')
env['LLVM_LINK'] = mx_sulong.findBundledLLVMProgram('llvm-link')
env['LLVM_OBJCOPY'] = mx_sulong.findBundledLLVMProgram('llvm-objcopy')
env['GRAALVM_LLVM_HOME'] = mx_subst.path_substitutions.substitute("<path:SULONG_HOME>")
if SulongTestSuite.haveDragonegg():
env['DRAGONEGG'] = mx_sulong.dragonEggPath()
env['DRAGONEGG_GCC'] = mx_sulong.getGCC()
env['DRAGONEGG_LLVMAS'] = mx_sulong.findLLVMProgramForDragonegg("llvm-as")
env['DRAGONEGG_FC'] = mx_sulong.getGFortran()
env['FC'] = mx_sulong.getGFortran()
elif not self._is_needs_rebuild_call and getattr(self, 'requireDragonegg', False):
mx.abort('Could not find dragonegg, cannot build "{}" (requireDragonegg = True).'.format(self.name))
return env
def hsdis(args, copyToDir=None):
"""download the hsdis library
This is needed to support HotSpot's assembly dumping features.
By default it downloads the Intel syntax version, use the 'att' argument to install AT&T syntax."""
flavor = 'intel'
if 'att' in args:
flavor = 'att'
if mx.get_arch() == "sparcv9":
flavor = "sparcv9"
lib = mx.add_lib_suffix('hsdis-' + mx.get_arch())
path = join(_suite.dir, 'lib', lib)
sha1s = {
'att/hsdis-amd64.dll' : 'bcbd535a9568b5075ab41e96205e26a2bac64f72',
'att/hsdis-amd64.so' : '58919ba085d4ef7a513f25bae75e7e54ee73c049',
'intel/hsdis-amd64.dll' : '6a388372cdd5fe905c1a26ced614334e405d1f30',
'intel/hsdis-amd64.so' : '844ed9ffed64fe9599638f29a8450c50140e3192',
'intel/hsdis-amd64.dylib' : 'fdb13ef0d7d23d93dacaae9c98837bea0d4fc5a2',
'sparcv9/hsdis-sparcv9.so': '970640a9af0bd63641f9063c11275b371a59ee60',
}
flavoredLib = flavor + "/" + lib
if flavoredLib not in sha1s:
mx.logv("hsdis not supported on this plattform or architecture")
return
if not exists(path):
sha1 = sha1s[flavoredLib]
sha1path = path + '.sha1'
mx.download_file_with_sha1('hsdis', path, ['https://lafo.ssw.uni-linz.ac.at/pub/hsdis/' + flavoredLib], sha1, sha1path, True, True, sources=False)
if copyToDir is not None and exists(copyToDir):
shutil.copy(path, copyToDir)
def getBuildEnv(self, replaceVar=mx_subst.path_substitutions):
env = super(ExternalTestSuite, self).getBuildEnv(replaceVar=replaceVar)
roots = [d.get_path(resolve=True) for d in self.buildDependencies if d.isPackedResourceLibrary()]
env['PROJECT'] = self.name
env['VPATH'] = ':'.join(roots)
env['TESTFILE'] = self.getTestFile()
env['VARIANTS'] = ' '.join(self.getVariants())
env['BUILD_REF'] = '1' if self.buildRef else '0'
env['BUILD_SO'] = '1' if self.buildSharedObject else '0'
env['SO_EXT'] = mx.add_lib_suffix("")
env['CLANG'] = mx_sulong.findBundledLLVMProgram('clang')
env['CLANGXX'] = mx_sulong.findBundledLLVMProgram('clang++')
env['LLVM_OPT'] = mx_sulong.findBundledLLVMProgram('opt')
env['LLVM_AS'] = mx_sulong.findBundledLLVMProgram('llvm-as')
env['LLVM_DIS'] = mx_sulong.findBundledLLVMProgram('llvm-dis')
env['LLVM_LINK'] = mx_sulong.findBundledLLVMProgram('llvm-link')
env['LLVM_OBJCOPY'] = mx_sulong.findBundledLLVMProgram('llvm-objcopy')
env['GRAALVM_LLVM_HOME'] = mx_subst.path_substitutions.substitute("<path:SULONG_HOME>")
if 'OS' not in env:
env['OS'] = mx_subst.path_substitutions.substitute("<os>")
if DragonEggSupport.haveDragonegg():
env['DRAGONEGG'] = DragonEggSupport.pluginPath()
env['DRAGONEGG_GCC'] = DragonEggSupport.findGCCProgram('gcc', optional=False)
env['DRAGONEGG_LLVMAS'] = DragonEggSupport.findLLVMProgram("llvm-as")
env['DRAGONEGG_FC'] = DragonEggSupport.findGCCProgram('gfortran', optional=False)
env['FC'] = DragonEggSupport.findGCCProgram('gfortran', optional=False)
elif not self._is_needs_rebuild_call and getattr(self, 'requireDragonegg', False):
mx.abort('Could not find dragonegg, cannot build "{}" (requireDragonegg = True).'.format(self.name))
return env
def _prepare_svm_env():
import mx_vm
if hasattr(mx_vm, 'graalvm_home'):
graalvm_home = mx_vm.graalvm_home()
else:
import mx_sdk_vm_impl
graalvm_home = mx_sdk_vm_impl.graalvm_home()
libgraal_nodejs_filename = mx.add_lib_suffix(
mx.add_lib_prefix('graal-nodejs'))
candidates = [
join(graalvm_home, directory, libgraal_nodejs_filename)
for directory in [
join('jre', 'languages', 'nodejs', 'lib'),
join('languages', 'nodejs', 'lib')
]
]
libgraal_nodejs = None
for candidate in candidates:
if exists(candidate):
libgraal_nodejs = candidate
if libgraal_nodejs is None:
mx.abort(
"Cannot find graal-nodejs library in '{}'.\nDid you forget to build it (e.g., using 'mx --env svm build')?"
.format(candidates))
_setEnvVar('NODE_JVM_LIB', libgraal_nodejs)
_setEnvVar('ICU4J_DATA_PATH',
join(mx.suite('graal-js').dir, 'lib', 'icu4j', 'icudt'))
def jdkartifactstats(args):
"""show stats about JDK deployed Graal artifacts"""
artifacts = {}
jdkDir = get_jvmci_jdk().home
def _getDeployedJars():
if JVMCI_VERSION < 9:
for root, _, filenames in os.walk(join(jdkDir, 'jre', 'lib')):
for f in filenames:
if f.endswith('.jar') and not f.endswith('.stripped.jar'):
yield join(root, f)
else:
for jdkDist in jdkDeployedDists:
dist = jdkDist.dist()
if isinstance(jdkDist, JvmciJDKDeployedDist):
yield dist.path
for jar in _getDeployedJars():
f = basename(jar)
if 'truffle' in f:
if 'enterprise' in f:
artifacts.setdefault('GraalEnterpriseTruffle', []).append(jar)
else:
artifacts.setdefault('GraalTruffle', []).append(jar)
elif 'enterprise' in f:
artifacts.setdefault('GraalEnterprise', []).append(jar)
elif 'jvmci' in f:
artifacts.setdefault('JVMCI', []).append(jar)
elif 'graal' in f:
artifacts.setdefault('Graal', []).append(jar)
else:
mx.logv('ignored: ' + jar)
print '{:>10} {:>10} {:>10} {}'.format('All', 'NoVars', 'None', 'Jar')
for category in sorted(artifacts.viewkeys()):
jars = artifacts[category]
if jars:
totals = (0, 0, 0)
print
for j in jars:
gSize = os.path.getsize(j)
stripped = j[:-len('.jar')] + '.stripped.jar'
mx.run([mx.get_jdk().pack200, '--repack', '--quiet', '-J-Djava.util.logging.config.file=', '-DLocalVariableTypeTable=strip', '-DLocalVariableTable=strip', stripped, j])
gLinesSourceSize = os.path.getsize(stripped)
mx.run([mx.get_jdk().pack200, '--repack', '--quiet', '-J-Djava.util.logging.config.file=', '-G', stripped, j])
gNoneSize = os.path.getsize(stripped)
os.remove(stripped)
print '{:10,} {:10,} {:10,} {}:{}'.format(gSize, gLinesSourceSize, gNoneSize, category, basename(j))
t1, t2, t3 = totals
totals = (t1 + gSize, t2 + gLinesSourceSize, t3 + gNoneSize)
t1, t2, t3 = totals
print '{:10,} {:10,} {:10,} {}'.format(t1, t2, t3, category)
jvmLib = join(jdkDir, relativeVmLibDirInJdk(), get_vm(), mx.add_lib_suffix(mx.add_lib_prefix('jvm')))
print
if exists(jvmLib):
print '{:10,} {}'.format(os.path.getsize(jvmLib), jvmLib)
else:
print '{:>10} {}'.format('<missing>', jvmLib)
def jdkartifactstats(args):
"""show stats about JDK deployed Graal artifacts"""
artifacts = {}
jdkDir = get_jvmci_jdk().home
def _getDeployedJars():
if JVMCI_VERSION < 9:
for root, _, filenames in os.walk(join(jdkDir, 'jre', 'lib')):
for f in filenames:
if f.endswith('.jar') and not f.endswith('.stripped.jar'):
yield join(root, f)
else:
for jdkDist in jdkDeployedDists:
dist = jdkDist.dist()
if isinstance(jdkDist, JvmciJDKDeployedDist):
yield dist.path
for jar in _getDeployedJars():
f = basename(jar)
if 'truffle' in f:
if 'enterprise' in f:
artifacts.setdefault('GraalEnterpriseTruffle', []).append(jar)
else:
artifacts.setdefault('GraalTruffle', []).append(jar)
elif 'enterprise' in f:
artifacts.setdefault('GraalEnterprise', []).append(jar)
elif 'jvmci' in f:
artifacts.setdefault('JVMCI', []).append(jar)
elif 'graal' in f:
artifacts.setdefault('Graal', []).append(jar)
else:
mx.logv('ignored: ' + jar)
print '{:>10} {:>10} {:>10} {}'.format('All', 'NoVars', 'None', 'Jar')
for category in sorted(artifacts.viewkeys()):
jars = artifacts[category]
if jars:
totals = (0, 0, 0)
print
for j in jars:
gSize = os.path.getsize(j)
stripped = j[:-len('.jar')] + '.stripped.jar'
mx.run([mx.get_jdk().pack200, '--repack', '--quiet', '-J-Djava.util.logging.config.file=', '-DLocalVariableTypeTable=strip', '-DLocalVariableTable=strip', stripped, j])
gLinesSourceSize = os.path.getsize(stripped)
mx.run([mx.get_jdk().pack200, '--repack', '--quiet', '-J-Djava.util.logging.config.file=', '-G', stripped, j])
gNoneSize = os.path.getsize(stripped)
os.remove(stripped)
print '{:10,} {:10,} {:10,} {}:{}'.format(gSize, gLinesSourceSize, gNoneSize, category, basename(j))
t1, t2, t3 = totals
totals = (t1 + gSize, t2 + gLinesSourceSize, t3 + gNoneSize)
t1, t2, t3 = totals
print '{:10,} {:10,} {:10,} {}'.format(t1, t2, t3, category)
jvmLib = join(jdkDir, relativeVmLibDirInJdk(), get_vm(), mx.add_lib_suffix(mx.add_lib_prefix('jvm')))
print
if exists(jvmLib):
print '{:10,} {}'.format(os.path.getsize(jvmLib), jvmLib)
else:
print '{:>10} {}'.format('<missing>', jvmLib)
def hsdis(args, copyToDir=None):
"""download the hsdis library
This is needed to support HotSpot's assembly dumping features.
By default it downloads the Intel syntax version, use the 'att' argument to install AT&T syntax."""
flavor = None
if mx.get_arch() == "amd64":
flavor = mx.get_env('HSDIS_SYNTAX')
if flavor is None:
flavor = 'intel'
if 'att' in args:
flavor = 'att'
libpattern = mx.add_lib_suffix('hsdis-' + mx.get_arch() + '-' + mx.get_os() + '-%s')
sha1s = {
'att/hsdis-amd64-windows-%s.dll' : 'bcbd535a9568b5075ab41e96205e26a2bac64f72',
'att/hsdis-amd64-linux-%s.so' : '36a0b8e30fc370727920cc089f104bfb9cd508a0',
'att/hsdis-amd64-darwin-%s.dylib' : 'c1865e9a58ca773fdc1c5eea0a4dfda213420ffb',
'intel/hsdis-amd64-windows-%s.dll' : '6a388372cdd5fe905c1a26ced614334e405d1f30',
'intel/hsdis-amd64-linux-%s.so' : '0d031013db9a80d6c88330c42c983fbfa7053193',
'intel/hsdis-amd64-darwin-%s.dylib' : '67f6d23cbebd8998450a88b5bef362171f66f11a',
'hsdis-sparcv9-solaris-%s.so': '970640a9af0bd63641f9063c11275b371a59ee60',
'hsdis-sparcv9-linux-%s.so': '0c375986d727651dee1819308fbbc0de4927d5d9',
}
if flavor:
flavoredLib = flavor + "/" + libpattern
else:
flavoredLib = libpattern
if flavoredLib not in sha1s:
mx.warn("hsdis with flavor '{}' not supported on this plattform or architecture".format(flavor))
return
sha1 = sha1s[flavoredLib]
lib = flavoredLib % (sha1)
path = join(_suite.get_output_root(), lib)
if not exists(path):
sha1path = path + '.sha1'
mx.download_file_with_sha1('hsdis', path, ['https://lafo.ssw.uni-linz.ac.at/pub/hsdis/' + lib], sha1, sha1path, True, True, sources=False)
if copyToDir is not None and exists(copyToDir):
destFileName = mx.add_lib_suffix('hsdis-' + mx.get_arch())
mx.logv('Copying {} to {}'.format(path, copyToDir + os.sep + destFileName))
shutil.copy(path, copyToDir + os.sep + destFileName)
def getResults(self, replaceVar=mx_subst.results_substitutions):
if not self.results:
self.results = []
for t in self.getTests():
if self.buildRef:
self.results.append(os.path.join(t, 'ref.out'))
for v in self.getVariants():
result_file = mx.add_lib_suffix(v) if self.buildSharedObject else v + '.bc'
self.results.append(os.path.join(t, result_file))
return super(SulongTestSuite, self).getResults(replaceVar=replaceVar)
def _prepare_svm_env():
import mx_vm
libpolyglot = join(mx_vm.graalvm_home(), 'jre', 'lib', 'polyglot',
mx.add_lib_suffix(mx.add_lib_prefix('polyglot')))
if not exists(libpolyglot):
mx.abort(
"Cannot find polyglot library in '{}'.\nDid you forget to build it (e.g., using 'mx --env svm build')?"
.format(libpolyglot))
_setEnvVar('NODE_JVM_LIB', libpolyglot)
_setEnvVar('ICU4J_DATA_PATH',
join(mx.suite('graal-js').dir, 'lib', 'icu4j', 'icudt'))
def mx_register_dynamic_suite_constituents(register_project, register_distribution):
"""Conditionally creates the ESPRESSO_LLVM_SUPPORT distribution if a Java home with LLVM bitcode is provided.
:type register_project: (mx.Project) -> None
:type register_distribution: (mx.Distribution) -> None
"""
if LLVM_JAVA_HOME:
lib_prefix = mx.add_lib_prefix('')
lib_suffix = mx.add_lib_suffix('')
lib_path = join(LLVM_JAVA_HOME, 'lib')
libraries = [join(lib_path, name) for name in os.listdir(lib_path) if name.startswith(lib_prefix) and name.endswith(lib_suffix)]
register_distribution(mx.LayoutTARDistribution(_suite, 'ESPRESSO_LLVM_SUPPORT', [], {
"lib/llvm/default/":
["file:" + lib for lib in libraries] +
["file:{}/release".format(LLVM_JAVA_HOME)],
}, None, True, None))
def _helloworld(native_image, javac_command, path, args):
mkpath(path)
hello_file = os.path.join(path, 'HelloWorld.java')
envkey = 'HELLO_WORLD_MESSAGE'
output = 'Hello from native-image!'
with open(hello_file, 'w') as fp:
fp.write('public class HelloWorld { public static void main(String[] args) { System.out.println(System.getenv("' + envkey + '")); } }')
fp.flush()
mx.run(javac_command + [hello_file])
native_image(["-H:Path=" + path, '-H:+VerifyNamingConventions', '-cp', path, 'HelloWorld'] + args)
expected_output = [output + os.linesep]
actual_output = []
def _collector(x):
actual_output.append(x)
mx.log(x)
if '--shared' in args:
# If helloword got built into a shared library we use python to load the shared library
# and call its `run_main`. We are capturing the stdout during the call into an unnamed
# pipe so that we can use it in the actual vs. expected check below.
try:
import ctypes
so_name = mx.add_lib_suffix('helloworld')
lib = ctypes.CDLL(join(path, so_name))
stdout = os.dup(1) # save original stdout
pout, pin = os.pipe()
os.dup2(pin, 1) # connect stdout to pipe
run_main = 'run_main' if mx.get_os() != 'windows' else 'main'
os.environ[envkey] = output
getattr(lib, run_main)(1, 'dummy') # call run_main of shared lib
call_stdout = os.read(pout, 120) # get pipe contents
actual_output.append(call_stdout)
os.dup2(stdout, 1) # restore original stdout
mx.log("Stdout from calling {} in shared object {}:".format(run_main, so_name))
mx.log(call_stdout)
finally:
del os.environ[envkey]
os.close(pin)
os.close(pout)
else:
env = os.environ.copy()
env[envkey] = output
mx.run([join(path, 'helloworld')], out=_collector, env=env)
if actual_output != expected_output:
raise Exception('Unexpected output: ' + str(actual_output) + " != " + str(expected_output))
def getBuildEnv(self, replaceVar=mx_subst.path_substitutions):
env = super(GeneratedTestSuite, self).getBuildEnv(replaceVar=replaceVar)
env['VPATH'] = self.dir
env['PROJECT'] = self.name
env['TESTS'] = ' '.join(self.getTests())
env['VARIANTS'] = ' '.join(self.getVariants())
env['BUILD_REF'] = '1' if self.buildRef else '0'
env['BUILD_SO'] = '1' if self.buildSharedObject else '0'
env['SO_EXT'] = mx.add_lib_suffix("")
env['CLANG'] = mx_sulong.findBundledLLVMProgram('clang')
env['CLANGXX'] = mx_sulong.findBundledLLVMProgram('clang++')
env['LLVM_OPT'] = mx_sulong.findBundledLLVMProgram('opt')
env['LLVM_AS'] = mx_sulong.findBundledLLVMProgram('llvm-as')
env['LLVM_DIS'] = mx_sulong.findBundledLLVMProgram('llvm-dis')
env['LLVM_LINK'] = mx_sulong.findBundledLLVMProgram('llvm-link')
env['LLVM_OBJCOPY'] = mx_sulong.findBundledLLVMProgram('llvm-objcopy')
env['GRAALVM_LLVM_HOME'] = mx_subst.path_substitutions.substitute("<path:SULONG_HOME>")
return env
def getBuildEnv(self, replaceVar=mx_subst.path_substitutions):
env = super(SulongTestSuite, self).getBuildEnv(replaceVar=replaceVar)
env['VPATH'] = os.path.join(self.dir, self.name)
env['PROJECT'] = self.name
env['TESTS'] = ' '.join(self.getTests())
env['VARIANTS'] = ' '.join(self.getVariants())
env['BUILD_REF'] = '1' if self.buildRef else '0'
env['BUILD_SO'] = '1' if self.buildSharedObject else '0'
env['SO_EXT'] = mx.add_lib_suffix("")
env['SULONG_MAKE_CLANG_IMPLICIT_ARGS'] = mx_sulong.getClangImplicitArgs()
if SulongTestSuite.haveDragonegg():
env['DRAGONEGG'] = mx_sulong.dragonEggPath()
env['DRAGONEGG_GCC'] = mx_sulong.getGCC()
env['DRAGONEGG_LLVMAS'] = mx_sulong.findLLVMProgramForDragonegg("llvm-as")
env['DRAGONEGG_FC'] = mx_sulong.getGFortran()
env['FC'] = mx_sulong.getGFortran()
elif not self._is_needs_rebuild_call and getattr(self, 'requireDragonegg', False):
mx.abort('Could not find dragonegg, cannot build "{}" (requireDragonegg = True).'.format(self.name))
return env
def hsdis(args, copyToDir=None):
"""download the hsdis library
This is needed to support HotSpot's assembly dumping features.
By default it downloads the Intel syntax version, use the 'att' argument to install AT&T syntax."""
flavor = None
if mx.get_arch() == "amd64":
flavor = mx.get_env('HSDIS_SYNTAX')
if flavor is None:
flavor = 'intel'
if 'att' in args:
flavor = 'att'
libpattern = mx.add_lib_suffix('hsdis-' + mx.get_arch() + '-' +
mx.get_os() + '-%s')
sha1s = {
r'att\hsdis-amd64-windows-%s.dll':
'bcbd535a9568b5075ab41e96205e26a2bac64f72',
r'att/hsdis-amd64-linux-%s.so':
'36a0b8e30fc370727920cc089f104bfb9cd508a0',
r'att/hsdis-amd64-darwin-%s.dylib':
'c1865e9a58ca773fdc1c5eea0a4dfda213420ffb',
r'intel\hsdis-amd64-windows-%s.dll':
'6a388372cdd5fe905c1a26ced614334e405d1f30',
r'intel/hsdis-amd64-linux-%s.so':
'0d031013db9a80d6c88330c42c983fbfa7053193',
r'intel/hsdis-amd64-darwin-%s.dylib':
'67f6d23cbebd8998450a88b5bef362171f66f11a',
r'hsdis-sparcv9-solaris-%s.so':
'970640a9af0bd63641f9063c11275b371a59ee60',
r'hsdis-sparcv9-linux-%s.so':
'0c375986d727651dee1819308fbbc0de4927d5d9',
r'hsdis-aarch64-linux-%s.so':
'fcc9b70ac91c00db8a50b0d4345490a68e3743e1',
}
if flavor:
flavoredLib = join(flavor, libpattern)
else:
flavoredLib = libpattern
if flavoredLib not in sha1s:
mx.warn(
"hsdis with flavor '{}' not supported on this platform or architecture"
.format(flavor))
return
sha1 = sha1s[flavoredLib]
lib = flavoredLib % (sha1)
path = join(_suite.get_output_root(), lib)
if not exists(path):
sha1path = path + '.sha1'
mx.download_file_with_sha1(
'hsdis',
path, [
rewriteurl(
'https://lafo.ssw.uni-linz.ac.at/pub/graal-external-deps/hsdis/'
+ lib.replace(os.sep, '/'))
],
sha1,
sha1path,
True,
True,
sources=False)
overwrite = True
if copyToDir is None:
# Try install hsdis into JAVA_HOME
overwrite = False
jdk = mx.get_jdk()
base = jdk.home
if exists(join(base, 'jre')):
base = join(base, 'jre')
if mx.get_os() == 'darwin':
copyToDir = join(base, 'lib')
elif mx.get_os() == 'windows':
copyToDir = join(base, 'bin')
else:
if jdk.javaCompliance >= '11':
copyToDir = join(base, 'lib')
else:
copyToDir = join(base, 'lib', mx.get_arch())
if exists(copyToDir):
dest = join(copyToDir, mx.add_lib_suffix('hsdis-' + mx.get_arch()))
if exists(dest) and not overwrite:
import filecmp
# Only issue warning if existing lib is different
if filecmp.cmp(path, dest) is False:
mx.warn('Not overwriting existing {} with {}'.format(
dest, path))
else:
try:
shutil.copy(path, dest)
mx.log('Copied {} to {}'.format(path, dest))
except IOError as e:
mx.warn('Could not copy {} to {}: {}'.format(
path, dest, str(e)))
def mktemp_libfile():
return tempfile.NamedTemporaryFile(prefix=mx.add_lib_prefix(''), suffix=mx.add_lib_suffix(''))
def jdkartifactstats(args):
"""show stats about JDK deployed Graal artifacts"""
artifacts = {}
jdkDir = get_jvmci_jdk().home
def _getDeployedJars():
if JVMCI_VERSION < 9:
for root, _, filenames in os.walk(join(jdkDir, "jre", "lib")):
for f in filenames:
if f.endswith(".jar") and not f.endswith(".stripped.jar"):
yield join(root, f)
else:
for jdkDist in jdkDeployedDists:
dist = jdkDist.dist()
if isinstance(jdkDist, JvmciJDKDeployedDist):
yield dist.path
for jar in _getDeployedJars():
f = basename(jar)
if "truffle" in f:
if "enterprise" in f:
artifacts.setdefault("GraalEnterpriseTruffle", []).append(jar)
else:
artifacts.setdefault("GraalTruffle", []).append(jar)
elif "enterprise" in f:
artifacts.setdefault("GraalEnterprise", []).append(jar)
elif "jvmci" in f:
artifacts.setdefault("JVMCI", []).append(jar)
elif "graal" in f:
artifacts.setdefault("Graal", []).append(jar)
else:
mx.logv("ignored: " + jar)
print "{:>10} {:>10} {:>10} {}".format("All", "NoVars", "None", "Jar")
for category in sorted(artifacts.viewkeys()):
jars = artifacts[category]
if jars:
totals = (0, 0, 0)
print
for j in jars:
gSize = os.path.getsize(j)
stripped = j[: -len(".jar")] + ".stripped.jar"
mx.run(
[
mx.get_jdk().pack200,
"--repack",
"--quiet",
"-J-Djava.util.logging.config.file=",
"-DLocalVariableTypeTable=strip",
"-DLocalVariableTable=strip",
stripped,
j,
]
)
gLinesSourceSize = os.path.getsize(stripped)
mx.run(
[
mx.get_jdk().pack200,
"--repack",
"--quiet",
"-J-Djava.util.logging.config.file=",
"-G",
stripped,
j,
]
)
gNoneSize = os.path.getsize(stripped)
os.remove(stripped)
print "{:10,} {:10,} {:10,} {}:{}".format(gSize, gLinesSourceSize, gNoneSize, category, basename(j))
t1, t2, t3 = totals
totals = (t1 + gSize, t2 + gLinesSourceSize, t3 + gNoneSize)
t1, t2, t3 = totals
print "{:10,} {:10,} {:10,} {}".format(t1, t2, t3, category)
jvmLib = join(jdkDir, relativeVmLibDirInJdk(), get_vm(), mx.add_lib_suffix(mx.add_lib_prefix("jvm")))
print
if exists(jvmLib):
print "{:10,} {}".format(os.path.getsize(jvmLib), jvmLib)
else:
print "{:>10} {}".format("<missing>", jvmLib)
def hsdis(args, copyToDir=None):
"""download the hsdis library and copy it to a specific dir or to the current JDK
This is needed to support HotSpot's assembly dumping features.
On amd64 platforms, it downloads the Intel syntax version"""
parser = ArgumentParser(prog='hsdis')
args = parser.parse_args(args)
hsdis_syntax = mx.get_env('HSDIS_SYNTAX')
if hsdis_syntax:
mx.warn(
"The 'hsdis' function ignores the value of the 'HSDIS_SYNTAX' environment variable: "
+ hsdis_syntax)
hsdis_lib_name = 'HSDIS'
hsdis_lib = mx.library(hsdis_lib_name)
if hsdis_lib.optional:
mx.abort('hsdis is not supported on this platform or architecture')
hsdis_lib_path = hsdis_lib.get_path(resolve=True)
hsdis_lib_files = os.listdir(hsdis_lib_path)
if len(hsdis_lib_files) != 1:
mx.abort(
"hsdis library '{}' does not contain a single file: {}".format(
hsdis_lib_name, hsdis_lib_files))
hsdis_lib_file = join(hsdis_lib_path, hsdis_lib_files[0])
overwrite = True
if copyToDir is None:
# Try install hsdis into JAVA_HOME
overwrite = False
jdk = mx.get_jdk()
base = jdk.home
if exists(join(base, 'jre')):
base = join(base, 'jre')
if mx.get_os() == 'darwin':
copyToDir = join(base, 'lib')
elif mx.get_os() == 'windows':
copyToDir = join(base, 'bin')
else:
if jdk.javaCompliance >= '11':
copyToDir = join(base, 'lib')
else:
copyToDir = join(base, 'lib', mx.get_arch())
if exists(copyToDir):
dest = join(copyToDir, mx.add_lib_suffix('hsdis-' + mx.get_arch()))
if exists(dest) and not overwrite:
import filecmp
# Only issue warning if existing lib is different
if filecmp.cmp(hsdis_lib_file, dest) is False:
mx.warn('Not overwriting existing {} with {}'.format(
dest, hsdis_lib_file))
else:
try:
shutil.copy(hsdis_lib_file, dest)
mx.log('Copied {} to {}'.format(hsdis_lib_file, dest))
except IOError as e:
mx.warn('Could not copy {} to {}: {}'.format(
hsdis_lib_file, dest, str(e)))
def _prepare_svm_env():
setLibraryPath()
_setEnvVar('NODE_JVM_LIB', join(_suite.dir, mx.add_lib_suffix('nodejs')))
_setEnvVar('ICU4J_DATA_PATH',
join(mx.suite('graal-js').dir, 'lib', 'icu4j', 'icudt'))
def _prepare_svm_env():
setLibraryPath()
_svm = _import_substratevm()
_setEnvVar('NODE_JVM_LIB', join(_svm.svmbuild_dir(), mx.add_lib_suffix('nodejs')))
def _prepare_svm_env():
setLibraryPath()
_setEnvVar('NODE_JVM_LIB', join(_suite.dir, mx.add_lib_suffix('nodejs')))
def mktemp_libfile():
return tempfile.NamedTemporaryFile(prefix=mx.add_lib_prefix(''),
suffix=mx.add_lib_suffix(''))
class DefaultNativeProject(NinjaProject):
"""A NinjaProject that makes many assumptions when generating a build manifest.
It is assumed that:
#. Directory layout is fixed:
- `include` is a flat subdir containing public headers, and
- `src` subdir contains sources and private headers.
#. There is only one deliverable:
- Kind is the value of the `native` attribute, and
- Name is the value of the `deliverable` attribute if it is specified,
otherwise it is derived from the `name` of the project.
#. All source files are supported and necessary to build the deliverable.
#. All `include_dirs` and `libs` provided by build dependencies are necessary
to build the deliverable.
#. The deliverable and the public headers are intended for distribution.
Attributes
native : {'static_lib', 'shared_lib'}
Kind of the deliverable.
Depending on the value, the necessary flags will be prepended to `cflags`
and `ldflags` automatically.
deliverable : str, optional
Name of the deliverable. By default, it is derived from the `name` of the
project.
"""
include = 'include'
src = 'src'
_kinds = dict(
static_lib=dict(target=lambda name: mx.add_lib_prefix(name) +
('.lib' if mx.is_windows() else '.a'), ),
shared_lib=dict(
target=lambda name: mx.add_lib_suffix(mx.add_lib_prefix(name)), ),
executable=dict(target=mx.exe_suffix, ),
)
def __init__(self, suite, name, subDir, srcDirs, deps, workingSets, d,
kind, **kwargs):
self.deliverable = kwargs.pop('deliverable', name.split('.')[-1])
if srcDirs:
mx.abort(
'"sourceDirs" is not supported for default native projects')
srcDirs += [self.include, self.src]
super(DefaultNativeProject,
self).__init__(suite, name, subDir, srcDirs, deps, workingSets,
d, **kwargs)
try:
self._kind = self._kinds[kind]
except KeyError:
mx.abort('"native" should be one of {}, but "{}" is given'.format(
list(self._kinds.keys()), kind))
include_dir = mx.join(self.dir, self.include)
if next(os.walk(include_dir))[1]:
mx.abort('include directory must have a flat structure')
self.include_dirs = [include_dir]
if kind == 'static_lib':
self.libs = [mx.join(self.out_dir, mx.get_arch(), self._target)]
@property
def _target(self):
return self._kind['target'](self.deliverable)
@property
def cflags(self):
default_cflags = []
if self._kind == self._kinds['shared_lib']:
default_cflags += dict(windows=['-MD'
], ).get(mx.get_os(), ['-fPIC'])
if mx.is_linux() or mx.is_darwin():
# Do not leak host paths via dwarf debuginfo
def add_debug_prefix(prefix_dir):
return '-fdebug-prefix-map={}={}'.format(
prefix_dir, mx.basename(prefix_dir))
default_cflags += [add_debug_prefix(self.suite.vc_dir)]
default_cflags += [add_debug_prefix(NinjaProject.get_jdk().home)]
default_cflags += ['-gno-record-gcc-switches']
return default_cflags + super(DefaultNativeProject, self).cflags
@property
def ldflags(self):
default_ldflags = []
if self._kind == self._kinds['shared_lib']:
default_ldflags += dict(
darwin=['-dynamiclib', '-undefined', 'dynamic_lookup'],
windows=['-dll'],
).get(mx.get_os(), ['-shared', '-fPIC'])
return default_ldflags + super(DefaultNativeProject, self).ldflags
@property
def h_files(self):
return self._source['files'].get('.h', [])
@property
def c_files(self):
return self._source['files'].get('.c', [])
@property
def cxx_files(self):
return self._source['files'].get('.cc', [])
@property
def asm_sources(self):
return self._source['files'].get('.S', [])
def generate_manifest(self, path):
unsupported_source_files = list(
set(self._source['files'].keys()) - {'.h', '.c', '.cc', '.S'})
if unsupported_source_files:
mx.abort(
'{} source files are not supported by default native projects'.
format(unsupported_source_files))
with NinjaManifestGenerator(self, open(path, 'w')) as gen:
gen.comment('Project rules')
cc = cxx = asm = None
if self.c_files:
cc = gen.cc_rule()
if self.cxx_files:
cxx = gen.cc_rule(cxx=True)
if self.asm_sources:
asm = gen.asm_rule() if mx.is_windows(
) else cc if cc else gen.cc_rule()
ar = link = None
if self._kind == self._kinds['static_lib']:
ar = gen.ar_rule()
else:
link = gen.link_rule(cxx=bool(self.cxx_files))
gen.variables(
cflags=[
mx_subst.path_substitutions.substitute(cflag)
for cflag in self.cflags
],
ldflags=[
mx_subst.path_substitutions.substitute(ldflag)
for ldflag in self.ldflags
] if link else None,
ldlibs=self.ldlibs if link else None,
)
gen.include(
collections.OrderedDict.fromkeys(
# remove the duplicates while maintaining the ordering
[mx.dirname(h_file) for h_file in self.h_files] + list(
itertools.chain.from_iterable(
getattr(d, 'include_dirs', [])
for d in self.buildDependencies))).keys())
gen.comment('Compiled project sources')
object_files = [cc(f) for f in self.c_files]
gen.newline()
object_files += [cxx(f) for f in self.cxx_files]
gen.newline()
object_files += [asm(f) for f in self.asm_sources]
gen.newline()
gen.comment('Project deliverable')
if self._kind == self._kinds['static_lib']:
ar(self._target, object_files)
else:
link(
self._target, object_files + list(
itertools.chain.from_iterable(
getattr(d, 'libs', [])
for d in self.buildDependencies)))
def _archivable_results(self, target_arch, use_relpath, single):
def result(base_dir, file_path):
assert not mx.isabs(file_path)
archive_path = file_path if use_relpath else mx.basename(file_path)
return mx.join(base_dir, file_path), archive_path
yield result(mx.join(self.out_dir, target_arch), self._target)
if not single:
for header in os.listdir(mx.join(self.dir, self.include)):
yield result(self.dir, mx.join(self.include, header))
def makegraaljdk(args):
"""make a JDK with Graal as the default top level JIT"""
parser = ArgumentParser(prog='mx makegraaljdk')
parser.add_argument('-f', '--force', action='store_true', help='overwrite existing GraalJDK')
parser.add_argument('-a', '--archive', action='store', help='name of archive to create', metavar='<path>')
parser.add_argument('dest', help='destination directory for GraalJDK', metavar='<path>')
args = parser.parse_args(args)
if isJDK8:
dstJdk = os.path.abspath(args.dest)
srcJdk = jdk.home
if exists(dstJdk):
if args.force:
shutil.rmtree(dstJdk)
else:
mx.abort('Use --force to overwrite existing directory ' + dstJdk)
mx.log('Creating {} from {}'.format(dstJdk, srcJdk))
shutil.copytree(srcJdk, dstJdk)
bootDir = mx.ensure_dir_exists(join(dstJdk, 'jre', 'lib', 'boot'))
jvmciDir = join(dstJdk, 'jre', 'lib', 'jvmci')
assert exists(jvmciDir), jvmciDir + ' does not exist'
if mx.get_os() == 'darwin' or mx.get_os() == 'windows':
jvmlibDir = join(dstJdk, 'jre', 'lib', 'server')
else:
jvmlibDir = join(dstJdk, 'jre', 'lib', mx.get_arch(), 'server')
jvmlib = join(jvmlibDir, mx.add_lib_prefix(mx.add_lib_suffix('jvm')))
assert exists(jvmlib), jvmlib + ' does not exist'
with open(join(jvmciDir, 'compiler-name'), 'w') as fp:
print >> fp, 'graal'
vmName = 'Graal'
mapFiles = set()
for e in _jvmci_classpath:
src = basename(e.get_path())
mx.log('Copying {} to {}'.format(e.get_path(), jvmciDir))
candidate = e.get_path() + '.map'
if exists(candidate):
mapFiles.add(candidate)
with open(join(dstJdk, 'release'), 'a') as fp:
d = e.dist()
s = d.suite
print >> fp, '{}={}'.format(d.name, s.vc.parent(s.dir))
vmName = vmName + ':' + s.name + '_' + s.version()
shutil.copyfile(e.get_path(), join(jvmciDir, src))
for e in _bootclasspath_appends:
src = basename(e.classpath_repr())
mx.log('Copying {} to {}'.format(e.classpath_repr(), bootDir))
candidate = e.classpath_repr() + '.map'
if exists(candidate):
mapFiles.add(candidate)
with open(join(dstJdk, 'release'), 'a') as fp:
s = e.suite
print >> fp, '{}={}'.format(e.name, s.vc.parent(s.dir))
shutil.copyfile(e.classpath_repr(), join(bootDir, src))
out = mx.LinesOutputCapture()
mx.run([jdk.java, '-version'], err=out)
line = None
pattern = re.compile(r'(.* )(?:Server|Graal) VM \(build.*')
for line in out.lines:
m = pattern.match(line)
if m:
with open(join(jvmlibDir, 'vm.properties'), 'w') as fp:
# Modify VM name in `java -version` to be Graal along
# with a suffix denoting the commit of each Graal jar.
# For example:
# Java HotSpot(TM) 64-Bit Graal:compiler_88847fb25d1a62977a178331a5e78fa5f8fcbb1a (build 25.71-b01-internal-jvmci-0.34, mixed mode)
print >> fp, 'name=' + m.group(1) + vmName
line = True
break
if line is not True:
mx.abort('Could not find "{}" in output of `java -version`:\n{}'.format(pattern.pattern, os.linesep.join(out.lines)))
exe = join(dstJdk, 'bin', mx.exe_suffix('java'))
with StdoutUnstripping(args=[], out=None, err=None, mapFiles=mapFiles) as u:
mx.run([exe, '-XX:+BootstrapJVMCI', '-version'], out=u.out, err=u.err)
if args.archive:
mx.log('Archiving {}'.format(args.archive))
create_archive(dstJdk, args.archive, basename(args.dest) + '/')
else:
mx.abort('Can only make GraalJDK for JDK 8 currently')
def hsdis(args, copyToDir=None):
"""download the hsdis library
This is needed to support HotSpot's assembly dumping features.
By default it downloads the Intel syntax version, use the 'att' argument to install AT&T syntax."""
flavor = None
if mx.get_arch() == "amd64":
flavor = mx.get_env('HSDIS_SYNTAX')
if flavor is None:
flavor = 'intel'
if 'att' in args:
flavor = 'att'
libpattern = mx.add_lib_suffix('hsdis-' + mx.get_arch() + '-' +
mx.get_os() + '-%s')
sha1s = {
'att/hsdis-amd64-windows-%s.dll':
'bcbd535a9568b5075ab41e96205e26a2bac64f72',
'att/hsdis-amd64-linux-%s.so':
'36a0b8e30fc370727920cc089f104bfb9cd508a0',
'att/hsdis-amd64-darwin-%s.dylib':
'c1865e9a58ca773fdc1c5eea0a4dfda213420ffb',
'intel/hsdis-amd64-windows-%s.dll':
'6a388372cdd5fe905c1a26ced614334e405d1f30',
'intel/hsdis-amd64-linux-%s.so':
'0d031013db9a80d6c88330c42c983fbfa7053193',
'intel/hsdis-amd64-darwin-%s.dylib':
'67f6d23cbebd8998450a88b5bef362171f66f11a',
'hsdis-sparcv9-solaris-%s.so':
'970640a9af0bd63641f9063c11275b371a59ee60',
'hsdis-sparcv9-linux-%s.so':
'0c375986d727651dee1819308fbbc0de4927d5d9',
}
if flavor:
flavoredLib = flavor + "/" + libpattern
else:
flavoredLib = libpattern
if flavoredLib not in sha1s:
mx.warn(
"hsdis with flavor '{}' not supported on this plattform or architecture"
.format(flavor))
return
sha1 = sha1s[flavoredLib]
lib = flavoredLib % (sha1)
path = join(_suite.get_output_root(), lib)
if not exists(path):
sha1path = path + '.sha1'
mx.download_file_with_sha1(
'hsdis',
path, ['https://lafo.ssw.uni-linz.ac.at/pub/hsdis/' + lib],
sha1,
sha1path,
True,
True,
sources=False)
if copyToDir is not None and exists(copyToDir):
destFileName = mx.add_lib_suffix('hsdis-' + mx.get_arch())
mx.logv('Copying {} to {}'.format(path,
copyToDir + os.sep + destFileName))
shutil.copy(path, copyToDir + os.sep + destFileName)
