def hsdis(args, copyToDir=None):
"""download the hsdis library
This is needed to support HotSpot's assembly dumping features.
By default it downloads the Intel syntax version, use the 'att' argument to install AT&T syntax."""
flavor = 'intel'
if 'att' in args:
flavor = 'att'
if mx.get_arch() == "sparcv9":
flavor = "sparcv9"
lib = mx.add_lib_suffix('hsdis-' + mx.get_arch())
path = join(_suite.dir, 'lib', lib)
sha1s = {
'att/hsdis-amd64.dll' : 'bcbd535a9568b5075ab41e96205e26a2bac64f72',
'att/hsdis-amd64.so' : '58919ba085d4ef7a513f25bae75e7e54ee73c049',
'intel/hsdis-amd64.dll' : '6a388372cdd5fe905c1a26ced614334e405d1f30',
'intel/hsdis-amd64.so' : '844ed9ffed64fe9599638f29a8450c50140e3192',
'intel/hsdis-amd64.dylib' : 'fdb13ef0d7d23d93dacaae9c98837bea0d4fc5a2',
'sparcv9/hsdis-sparcv9.so': '970640a9af0bd63641f9063c11275b371a59ee60',
}
flavoredLib = flavor + "/" + lib
if flavoredLib not in sha1s:
mx.logv("hsdis not supported on this plattform or architecture")
return
if not exists(path):
sha1 = sha1s[flavoredLib]
sha1path = path + '.sha1'
mx.download_file_with_sha1('hsdis', path, ['https://lafo.ssw.uni-linz.ac.at/pub/hsdis/' + flavoredLib], sha1, sha1path, True, True, sources=False)
if copyToDir is not None and exists(copyToDir):
shutil.copy(path, copyToDir)
def hsdis(args, copyToDir=None):
"""download the hsdis library
This is needed to support HotSpot's assembly dumping features.
By default it downloads the Intel syntax version, use the 'att' argument to install AT&T syntax."""
flavor = 'intel'
if 'att' in args:
flavor = 'att'
if mx.get_arch() == "sparcv9":
flavor = "sparcv9"
lib = mx.add_lib_suffix('hsdis-' + mx.get_arch())
path = join(_suite.dir, 'lib', lib)
sha1s = {
'att/hsdis-amd64.dll' : 'bcbd535a9568b5075ab41e96205e26a2bac64f72',
'att/hsdis-amd64.so' : '58919ba085d4ef7a513f25bae75e7e54ee73c049',
'intel/hsdis-amd64.dll' : '6a388372cdd5fe905c1a26ced614334e405d1f30',
'intel/hsdis-amd64.so' : '844ed9ffed64fe9599638f29a8450c50140e3192',
'intel/hsdis-amd64.dylib' : 'fdb13ef0d7d23d93dacaae9c98837bea0d4fc5a2',
'sparcv9/hsdis-sparcv9.so': '970640a9af0bd63641f9063c11275b371a59ee60',
}
flavoredLib = flavor + "/" + lib
if flavoredLib not in sha1s:
mx.logv("hsdis not supported on this plattform or architecture")
return
if not exists(path):
sha1 = sha1s[flavoredLib]
sha1path = path + '.sha1'
mx.download_file_with_sha1('hsdis', path, ['https://lafo.ssw.uni-linz.ac.at/pub/hsdis/' + flavoredLib], sha1, sha1path, True, True, sources=False)
if copyToDir is not None and exists(copyToDir):
shutil.copy(path, copyToDir)
def __init__(self, args, project, *otherargs, **kwargs):
super(CMakeBuildTaskMixin, self).__init__(args, project, *otherargs, **kwargs)
self._cmake_config_file = os.path.join(project.suite.get_mx_output_dir(), 'cmakeConfig',
mx.get_os() + '-' + mx.get_arch() if project.isPlatformDependent() else '',
type(project).__name__,
project._extra_artifact_discriminant(),
self.name)
def pullLLVMBinaries(args=None):
"""downloads the LLVM binaries"""
toolDir = join(_toolDir, "llvm")
mx.ensure_dir_exists(toolDir)
osStr = mx.get_os()
arch = mx.get_arch()
if osStr == 'windows':
mx.log_error('windows currently only supported with cygwin!')
return
elif osStr == 'linux':
if arch == 'amd64':
urls = [
'https://lafo.ssw.uni-linz.ac.at/pub/sulong-deps/clang+llvm-3.2-x86_64-linux-ubuntu-12.04.tar.gz'
]
else:
urls = [
'https://lafo.ssw.uni-linz.ac.at/pub/sulong-deps/clang+llvm-3.2-x86-linux-ubuntu-12.04.tar.gz'
]
elif osStr == 'darwin':
urls = [
'https://lafo.ssw.uni-linz.ac.at/pub/sulong-deps/clang+llvm-3.2-x86_64-apple-darwin11.tar.gz'
]
elif osStr == 'cygwin':
urls = [
'https://lafo.ssw.uni-linz.ac.at/pub/sulong-deps/clang+llvm-3.2-x86-mingw32-EXPERIMENTAL.tar.gz'
]
else:
mx.log_error("{0} {1} not supported!".format(osStr, arch))
localPath = pullsuite(toolDir, urls)
tar(localPath, toolDir, stripLevels=1)
os.remove(localPath)
def _get_openjdk_cpu():
cpu = mx.get_arch()
if cpu == 'amd64':
cpu = 'x86_64'
elif cpu == 'sparcv9':
cpu = 'sparcv9'
return cpu
def relativeVmLibDirInJdk():
mxos = mx.get_os()
if mxos == 'darwin':
return join('lib')
if mxos == 'windows' or mxos == 'cygwin':
return join('bin')
return join('lib', mx.get_arch())
def getArchivableResults(self, use_relpath=True, single=False):
for target_arch in self.multiarch if self._use_multiarch else [mx.get_arch()]:
toolchain = _Toolchain.for_(target_arch)
target_arch_path = toolchain.target if self.multiarch else ''
if toolchain.is_native or not single:
for file_path, archive_path in self._archivable_results(target_arch, use_relpath, single):
yield file_path, mx.join(target_arch_path, archive_path)
def __init__(self, suite, name, subDir, srcDirs, deps, workingSets, d,
kind, **kwargs):
self.deliverable = kwargs.pop('deliverable', name.split('.')[-1])
self.toolchain = kwargs.pop('toolchain', 'mx:DEFAULT_NINJA_TOOLCHAIN')
if srcDirs:
raise mx.abort(
'"sourceDirs" is not supported for default native projects')
srcDirs += [self.include, self.src]
super(DefaultNativeProject,
self).__init__(suite, name, subDir, srcDirs, deps, workingSets,
d, **kwargs)
try:
self._kind = self._kinds[kind]
except KeyError:
raise mx.abort(
'"native" should be one of {}, but "{}" is given'.format(
list(self._kinds.keys()), kind))
include_dir = mx.join(self.dir, self.include)
if next(os.walk(include_dir))[1]:
raise mx.abort('include directory must have a flat structure')
self.include_dirs = [include_dir]
if kind == 'static_lib':
self.libs = [mx.join(self.out_dir, mx.get_arch(), self._target)]
self.buildDependencies.append(self.toolchain)
def pullLLVMBinaries(args=None):
"""downloads the LLVM binaries"""
toolDir = join(_toolDir, "tools/llvm")
mx.ensure_dir_exists(toolDir)
osStr = mx.get_os()
arch = mx.get_arch()
if osStr == 'windows':
print 'windows currently only supported with cygwin!'
return
elif osStr == 'linux':
if arch == 'amd64':
urls = ['http://lafo.ssw.uni-linz.ac.at/sulong-deps/clang+llvm-3.2-x86_64-linux-ubuntu-12.04.tar.gz',
'http://llvm.org/releases/3.2/clang+llvm-3.2-x86_64-linux-ubuntu-12.04.tar.gz']
else:
urls = ['http://lafo.ssw.uni-linz.ac.at/sulong-deps/clang+llvm-3.2-x86-linux-ubuntu-12.04.tar.gz',
'http://llvm.org/releases/3.2/clang+llvm-3.2-x86-linux-ubuntu-12.04.tar.gz']
elif osStr == 'darwin':
urls = ['http://lafo.ssw.uni-linz.ac.at/sulong-deps/clang+llvm-3.2-x86_64-apple-darwin11.tar.gz',
'http://llvm.org/releases/3.2/clang+llvm-3.2-x86_64-apple-darwin11.tar.gz']
elif osStr == 'cygwin':
urls = ['http://lafo.ssw.uni-linz.ac.at/sulong-deps/clang+llvm-3.2-x86-mingw32-EXPERIMENTAL.tar.gz',
'http://llvm.org/releases/3.2/clang+llvm-3.2-x86-mingw32-EXPERIMENTAL.tar.gz']
else:
print osStr, arch, "not supported!"
localPath = pullsuite(toolDir, urls)
tar(localPath, toolDir, stripLevels=1)
os.remove(localPath)
def collectExcludes(path):
def _collect(path, skip=None):
for root, _, files in os.walk(path):
if skip and skip(os.path.relpath(root, path)):
continue
for f in files:
if f.endswith('.exclude'):
for line in open(os.path.join(root, f)):
yield line.strip()
# use `yield from` in python 3.3
for x in _collect(path, lambda p: p.startswith('os_arch')):
yield x
os_arch_root = os.path.join(path, 'os_arch')
if os.path.exists(os_arch_root):
try:
os_path = next(x for x in (os.path.join(os_arch_root, os_dir)
for os_dir in [mx.get_os(), 'others'])
if os.path.exists(x))
os_arch_path = next(
x for x in (os.path.join(os_path, arch_dir)
for arch_dir in [mx.get_arch(), 'others'])
if os.path.exists(x))
# use `yield from` in python 3.3
for x in _collect(os_arch_path):
yield x
except StopIteration:
pass
def _get_openjdk_cpu():
cpu = mx.get_arch()
if cpu == 'amd64':
cpu = 'x86_64'
elif cpu == 'sparcv9':
cpu = 'sparcv9'
return cpu
def relativeVmLibDirInJdk():
mxos = mx.get_os()
if mxos == 'darwin':
return join('lib')
if mxos == 'windows' or mxos == 'cygwin':
return join('bin')
return join('lib', mx.get_arch())
def copylib(args):
'''
This supports a configuration where no explicit setting (e.g. LD_LIBRARY_PATH) is
required at runtime for the libraries that are required by FastR, e.g. pcre.
The easy case is when the libraries are already installed with the correct versions
in one of the directories, e.g./usr/lib, that is searched by default by the system linker -
in which case no configuration is required.
Otherwise, since systems vary considerably in where such libraries are located, the general solution
is to copy libraries located in non-system locations into the FastR 'lib' directory. N.B. GNU R is
even more picky than FastR about library versions and depends on a larger set, so the local
copy embedded in FastR is built using PKG_LDFLAGS_OVERRIDE to specify the location of necessary external
libraries. However, the result of this analysis isn't captured anywhere, so we re-analyze here.
If PKG_LDFLAGS_OVERRIDE is unset, we assume the libraries are located in the system directories
and do nothing.
'''
if 'PKG_LDFLAGS_OVERRIDE' in os.environ:
partsVar = os.environ['PKG_LDFLAGS_OVERRIDE']
if 'PKG_LDFLAGS_GCC' in os.environ:
partsVar += ' ' + os.environ['PKG_LDFLAGS_GCC']
parts = partsVar.split(' ')
ext = 'dylib' if platform.system() == 'Darwin' else 'so'
lib_prefix = 'lib' + args[0] + '.'
plain_libpath_base = lib_prefix + ext
ver_env_key = 'FASTR_LIB' + args[0].upper() + '_VER'
if ver_env_key in os.environ:
plain_libpath_base += '.' + os.environ[ver_env_key]
for part in parts:
path = part.strip('"').lstrip('-L')
if os.path.exists(path):
for f in os.listdir(path):
if f.startswith(lib_prefix):
if os.path.exists(os.path.join(path, plain_libpath_base)):
f = plain_libpath_base
target_dir = args[1]
target = os.path.join(path, f)
if not os.path.exists(os.path.join(target_dir, f)):
if os.path.islink(target):
link_target = os.path.join(path, os.readlink(target))
mx.log('link target: ' + link_target)
if link_target == '/usr/lib/libSystem.B.dylib' or link_target == '/usr/lib/libSystem.dylib':
# simply copy over the link to the system library
os.symlink(link_target, os.path.join(target_dir, plain_libpath_base))
return 0
_copylib(args[0], target, plain_libpath_base, args[1])
return 0
if 'FASTR_RELEASE' in os.environ:
# if args[0] == 'quadmath' and (mx.get_arch() == 'sparcv9' or mx.get_os() == 'solaris'):
if mx.get_arch() == 'sparcv9' or mx.get_os() == 'solaris':
return 0
if os.environ.get('FASTR_RELEASE') == 'dev':
mx.log(args[0] + ' not found in PKG_LDFLAGS_OVERRIDE, but required with FASTR_RELEASE')
mx.log('the resulting FastR release build will not be portable to another system')
else:
mx.abort(args[0] + ' not found in PKG_LDFLAGS_OVERRIDE, but required with FASTR_RELEASE')
mx.log(args[0] + ' not found in PKG_LDFLAGS_OVERRIDE, assuming system location')
return 0
def patches(self):
"""A list of patches that will be applied during a build."""
os_arch_dir = mx.join(self.source_dirs()[0], '{}-{}'.format(mx.get_os(), mx.get_arch()))
if mx.exists(os_arch_dir):
return [mx.join(os_arch_dir, patch) for patch in os.listdir(os_arch_dir)]
others_dir = mx.join(self.source_dirs()[0], 'others')
return [mx.join(others_dir, patch) for patch in os.listdir(others_dir)]
class MultiVariantBuildTask(mx.Buildable, mx.TaskSequence):
subtasks = [VariantCMakeNinjaBuildTask(args, self, target_arch=mx.get_arch(), ninja_targets=self._ninja_targets, variant=variant) for variant in _variant()]
def execute(self):
super(MultiVariantBuildTask, self).execute()
self.built = any(t.built for t in self.subtasks)
def newestOutput(self):
return mx.TimeStampFile.newest(t.newestOutput().path for t in self.subtasks)
def __init__(self, suite, name, subDir, srcDirs, deps, workingSets, d,
**kwargs):
context = 'project ' + name
if 'multiarch' in kwargs:
multiarch = mx.Suite._pop_list(kwargs, 'multiarch', context)
self.multiarch = list(
set(
mx_subst.results_substitutions.substitute(arch)
for arch in multiarch))
if mx.get_arch() not in self.multiarch:
mx.abort(
'"multiarch" must contain the host architecture "{}"'.
format(mx.get_arch()), context)
else:
self.multiarch = []
super(MultiarchProject, self).__init__(suite, name, subDir, srcDirs,
deps, workingSets, d, **kwargs)
self.out_dir = self.get_output_root()
def hsdis(args, copyToDir=None):
"""download the hsdis library
This is needed to support HotSpot's assembly dumping features.
By default it downloads the Intel syntax version, use the 'att' argument to install AT&T syntax."""
flavor = None
if mx.get_arch() == "amd64":
flavor = mx.get_env('HSDIS_SYNTAX')
if flavor is None:
flavor = 'intel'
if 'att' in args:
flavor = 'att'
libpattern = mx.add_lib_suffix('hsdis-' + mx.get_arch() + '-' + mx.get_os() + '-%s')
sha1s = {
'att/hsdis-amd64-windows-%s.dll' : 'bcbd535a9568b5075ab41e96205e26a2bac64f72',
'att/hsdis-amd64-linux-%s.so' : '36a0b8e30fc370727920cc089f104bfb9cd508a0',
'att/hsdis-amd64-darwin-%s.dylib' : 'c1865e9a58ca773fdc1c5eea0a4dfda213420ffb',
'intel/hsdis-amd64-windows-%s.dll' : '6a388372cdd5fe905c1a26ced614334e405d1f30',
'intel/hsdis-amd64-linux-%s.so' : '0d031013db9a80d6c88330c42c983fbfa7053193',
'intel/hsdis-amd64-darwin-%s.dylib' : '67f6d23cbebd8998450a88b5bef362171f66f11a',
'hsdis-sparcv9-solaris-%s.so': '970640a9af0bd63641f9063c11275b371a59ee60',
'hsdis-sparcv9-linux-%s.so': '0c375986d727651dee1819308fbbc0de4927d5d9',
}
if flavor:
flavoredLib = flavor + "/" + libpattern
else:
flavoredLib = libpattern
if flavoredLib not in sha1s:
mx.warn("hsdis with flavor '{}' not supported on this plattform or architecture".format(flavor))
return
sha1 = sha1s[flavoredLib]
lib = flavoredLib % (sha1)
path = join(_suite.get_output_root(), lib)
if not exists(path):
sha1path = path + '.sha1'
mx.download_file_with_sha1('hsdis', path, ['https://lafo.ssw.uni-linz.ac.at/pub/hsdis/' + lib], sha1, sha1path, True, True, sources=False)
if copyToDir is not None and exists(copyToDir):
destFileName = mx.add_lib_suffix('hsdis-' + mx.get_arch())
mx.logv('Copying {} to {}'.format(path, copyToDir + os.sep + destFileName))
shutil.copy(path, copyToDir + os.sep + destFileName)
def __init__(self,
args,
project,
target_arch=mx.get_arch(),
ninja_targets=None):
super(NinjaBuildTask, self).__init__(args, project, target_arch)
self._reason = None
self._manifest = mx.join(self.out_dir, Ninja.default_manifest)
self.ninja = Ninja(self.out_dir,
self.parallelism,
targets=ninja_targets)
def _get_jdk_bundle_arches():
"""
Gets a list of names that will be the part of a JDK bundle's file name denoting the architecture.
The first element in the list is the canonical name. Symlinks should be created for the
remaining names.
"""
cpu = mx.get_arch()
if cpu == 'amd64':
return ['x64', 'x86_64', 'amd64']
elif cpu == 'sparcv9':
return ['sparcv9']
mx.abort('Unsupported JDK bundle arch: ' + cpu)
def __init__(self, version):
JdkDistribution.__init__(self, version)
machine = mx.get_os() + '-' + mx.get_arch()
self._filename = "labsjdk-{}-{}".format(version, machine)
self._short_version = re.sub(r".*(jvmci.*)", "\\1", version)
self._archive = "{}.tar.gz".format(self._filename)
self._url = ("{GITHUB_URL}/graalvm/labs-openjdk-16/"
"{GITHUB_RELEASES}/{short_version}/{archive}").format(
GITHUB_URL=GITHUB_URL,
GITHUB_RELEASES=GITHUB_RELEASES,
short_version=self._short_version,
archive=self._archive)
def _get_jdk_bundle_arches():
"""
Gets a list of names that will be the part of a JDK bundle's file name denoting the architecture.
The first element in the list is the canonical name. Symlinks should be created for the
remaining names.
"""
cpu = mx.get_arch()
if cpu == 'amd64':
return ['x64', 'x86_64', 'amd64']
elif cpu == 'sparcv9':
return ['sparcv9']
mx.abort('Unsupported JDK bundle arch: ' + cpu)
def testv8(args, nonZeroIsFatal=True):
"""run the testV8 conformance suite"""
_location = join(_suite.dir, 'lib', 'testv8')
_stack_size = '2m' if mx.get_arch() == 'sparcv9' else '1m'
_run_test_suite(
location=_location,
library_names=['TESTV8'],
custom_args=args,
default_vm_args=[],
max_heap='8g',
stack_size=_stack_size,
main_class='com.oracle.truffle.js.test.external.testv8.TestV8',
nonZeroIsFatal=nonZeroIsFatal,
cwd=_suite.dir)
def python_svm(args):
mx.run_mx([
"--dynamicimports", "/substratevm,/vm", "build",
"--force-deprecation-as-warning", "--dependencies",
"GRAAL_MANAGEMENT,graalpython.image"
],
nonZeroIsFatal=True)
vmdir = os.path.join(mx.suite("truffle").dir, "..", "vm")
svm_image = os.path.join(vmdir, "mxbuild",
"-".join([mx.get_os(), mx.get_arch()]),
"graalpython.image", "graalpython")
shutil.copy(svm_image, os.path.join(_suite.dir, "graalpython-svm"))
mx.run([svm_image] + args)
return svm_image
def runWrk2(self, wrkFlags):
distro = self.getOS()
arch = mx.get_arch()
wrkDirectory = mx.library('WRK2_MULTIARCH', True).get_path(True)
wrkPath = os.path.join(wrkDirectory, "wrk-{os}-{arch}".format(os=distro, arch=arch))
if not os.path.exists(wrkPath):
raise ValueError("Unsupported OS or arch. Binary doesn't exist: {}".format(wrkPath))
runWrkCmd = [wrkPath] + wrkFlags
mx.log("Running Wrk2: {0}".format(runWrkCmd))
output = mx.TeeOutputCapture(mx.OutputCapture())
mx.run(runWrkCmd, out=output, err=output)
return output.underlying.data
def test262(args, nonZeroIsFatal=True):
"""run the test262 conformance suite"""
_default_vm_args = []
_stack_size = '2m' if mx.get_arch() in ('aarch64', 'sparcv9') else '1m'
_fetch_test262()
return _run_test_suite(
custom_args=args,
default_vm_args=_default_vm_args,
max_heap='4g',
stack_size=_stack_size,
main_class='com.oracle.truffle.js.test.external.test262.Test262',
nonZeroIsFatal=nonZeroIsFatal,
cwd=_suite.dir
)
def testnashorn(args, nonZeroIsFatal=True):
"""run the testNashorn conformance suite"""
_location = join(_suite.dir, 'lib', 'testnashorn')
_default_vm_args = []
_stack_size = '2m' if mx.get_arch() in ('aarch64', 'sparcv9') else '1m'
_run_test_suite(
location=_location,
library_names=['TESTNASHORN', 'TESTNASHORN_EXTERNAL'],
custom_args=args,
default_vm_args=_default_vm_args,
max_heap='2g',
stack_size=_stack_size,
main_class='com.oracle.truffle.js.test.external.nashorn.TestNashorn',
nonZeroIsFatal=nonZeroIsFatal,
cwd=_location)
def getBuildTask(self, args):
if self._use_multiarch:
class MultiarchBuildTask(mx.Buildable, mx.TaskSequence):
subtasks = [self._build_task(target_arch, args) for target_arch in self.multiarch]
def execute(self):
super(MultiarchBuildTask, self).execute()
self.built = any(t.built for t in self.subtasks)
def newestOutput(self):
return mx.TimeStampFile.newest(t.newestOutput() for t in self.subtasks)
return MultiarchBuildTask(self, args)
else:
return self._build_task(mx.get_arch(), args)
def __init__(self,
args,
project,
target_arch=mx.get_arch(),
ninja_targets=None,
variant=None):
self.variant = variant
# abuse target_arch to inject variant
super(VariantCMakeNinjaBuildTask,
self).__init__(args,
project,
target_arch=os.path.join("build", variant),
ninja_targets=ninja_targets)
# set correct target_arch
self.target_arch = target_arch
def test262(args, nonZeroIsFatal=True):
"""run the test262 conformance suite"""
_location = join(_suite.dir, 'lib', 'test262')
_default_vm_args = []
_stack_size = '2m' if mx.get_arch() in ('aarch64', 'sparcv9') else '1m'
return _run_test_suite(
location=_location,
library_names=['TEST262'],
custom_args=args,
default_vm_args=_default_vm_args,
max_heap='4g',
stack_size=_stack_size,
main_class='com.oracle.truffle.js.test.external.test262.Test262',
nonZeroIsFatal=nonZeroIsFatal,
cwd=_suite.dir)
def _get_hotspot_build_dir(jvmVariant=None, debugLevel=None):
"""
Gets the directory in which a particular HotSpot configuration is built
(e.g., <JDK_REPO_ROOT>/build/macosx-x86_64-normal-server-release/hotspot/bsd_amd64_compiler2)
"""
if jvmVariant is None:
jvmVariant = _vm.jvmVariant
os = mx.get_os()
if os == 'darwin':
os = 'bsd'
arch = mx.get_arch()
buildname = {'client': 'compiler1', 'server': 'compiler2'}.get(jvmVariant, jvmVariant)
name = '{}_{}_{}'.format(os, arch, buildname)
return join(_get_jdk_build_dir(debugLevel=debugLevel), 'hotspot', name)
def _get_hotspot_build_dir(jvmVariant=None, debugLevel=None):
"""
Gets the directory in which a particular HotSpot configuration is built
(e.g., <JDK_REPO_ROOT>/build/macosx-x86_64-normal-server-release/hotspot/bsd_amd64_compiler2)
"""
if jvmVariant is None:
jvmVariant = _vm.jvmVariant
os = mx.get_os()
if os == 'darwin':
os = 'bsd'
arch = mx.get_arch()
buildname = {'client': 'compiler1', 'server': 'compiler2'}.get(jvmVariant, jvmVariant)
name = '{}_{}_{}'.format(os, arch, buildname)
return join(_get_jdk_build_dir(debugLevel=debugLevel), 'hotspot', name)
def __init__(self, jdk_id, version, filename, url, source):
self._jdk_id = jdk_id
self._version = version
self._filename_template = filename
self._url_template = url
self._source = source
platform = mx.get_os() + '-' + mx.get_arch()
keywords = {'version': version, 'platform': platform}
self._filename = _instantiate(filename, keywords, source)
keywords['filename'] = self._filename
self._folder_name = "{}-{}".format(
jdk_id, _instantiate('{version|jvmci-tag}', keywords, source))
self._url = _instantiate(url,
{k: quote(v)
for k, v in keywords.items()}, source)
self._archive = self._url[self._url.rfind(self._filename):]
def copylib(args):
'''
This supports a configuration where no explicit setting (e.g. LD_LIBRARY_PATH) is
required at runtime for the libraries that are required by FastR, e.g. pcre.
The easy case is when the libraries are already installed with the correct versions
in one of the directories, e.g./usr/lib, that is searched by default by the system linker -
in which case no configuration is required.
Otherwise, since systems vary considerably in where such libraries are located, the general solution
is to copy libraries located in non-system locations into the FastR 'lib' directory. N.B. GNU R is
even more picky than FastR about library versions and depends on a larger set, so the local
copy embedded in FastR is built using PKG_LDFLAGS_OVERRIDE to specify the location of necessary external
libraries. However, the result of this analysis isn't captured anywhere, so we re-analyze here.
If PKG_LDFLAGS_OVERRIDE is unset, we assume the libraries are located in the system directories
and do nothing.
'''
if os.environ.has_key('PKG_LDFLAGS_OVERRIDE'):
parts = os.environ['PKG_LDFLAGS_OVERRIDE'].split(' ')
ext = '.dylib' if platform.system() == 'Darwin' else '.so'
lib_prefix = 'lib' + args[0]
plain_libpath_base = lib_prefix + ext
for part in parts:
path = part.strip('"').lstrip('-L')
if os.path.exists(path):
for f in os.listdir(path):
if f.startswith(lib_prefix):
if os.path.exists(
os.path.join(path, plain_libpath_base)):
f = plain_libpath_base
target_dir = args[1]
if not os.path.exists(os.path.join(target_dir, f)):
_copylib(args[0], os.path.join(path, f),
plain_libpath_base, args[1])
return 0
if os.environ.has_key('FASTR_RELEASE'):
# if args[0] == 'quadmath' and (mx.get_arch() == 'sparcv9' or mx.get_os() == 'solaris'):
if mx.get_arch() == 'sparcv9' or mx.get_os() == 'solaris':
return 0
mx.abort(
args[0] +
' not found in PKG_LDFLAGS_OVERRIDE, but required with FASTR_RELEASE'
)
mx.log(args[0] +
' not found in PKG_LDFLAGS_OVERRIDE, assuming system location')
return 0
def patches(self):
"""A list of patches that will be applied during a build."""
def patch_dir(d):
return mx.join(self.source_dirs()[0], d)
def get_patches(patchdir):
for patch in os.listdir(patchdir):
yield mx.join(patchdir, patch)
for p in get_patches(patch_dir('common')):
yield p
os_arch_dir = patch_dir('{}-{}'.format(mx.get_os(), mx.get_arch()))
if mx.exists(os_arch_dir):
for p in get_patches(os_arch_dir):
yield p
else:
for p in get_patches(patch_dir('others')):
yield p
def copylib(args):
'''
This supports a configuration where no explicit setting (e.g. LD_LIBRARY_PATH) is
required at runtime for the libraries that are required by FastR, e.g. pcre.
The easy case is when the libraries are already installed with the correct versions
in one of the directories, e.g./usr/lib, that is searched by default by the system linker -
in which case no configuration is required.
Otherwise, since systems vary considerably in where such libraries are located, the general solution
is to copy libraries located in non-system locations into the FastR 'lib' directory. N.B. GNU R is
even more picky than FastR about library versions and depends on a larger set, so the local
copy embedded in FastR is built using PKG_LDFLAGS_OVERRIDE to specify the location of necessary external
libraries. However, the result of this analysis isn't captured anywhere, so we re-analyze here.
If PKG_LDFLAGS_OVERRIDE is unset, we assume the libraries are located in the system directories
and do nothing.
'''
if os.environ.has_key('PKG_LDFLAGS_OVERRIDE'):
parts = os.environ['PKG_LDFLAGS_OVERRIDE'].split(' ')
ext = '.dylib' if platform.system() == 'Darwin' else '.so'
lib_prefix = 'lib' + args[0]
plain_libpath = lib_prefix + ext
for part in parts:
path = part.strip('"').lstrip('-L')
if os.path.exists(path):
for f in os.listdir(path):
if f.startswith(lib_prefix):
if os.path.exists(os.path.join(path, plain_libpath)):
f = plain_libpath
target_dir = args[1]
if not os.path.exists(os.path.join(target_dir, f)):
_copylib(args[0], os.path.join(path, f), args[1])
return 0
if os.environ.has_key('FASTR_RELEASE'):
if args[0] == 'quadmath' and (mx.get_arch() == 'sparcv9' or mx.get_os() == 'solaris'):
return 0
mx.abort(args[0] + ' not found in PKG_LDFLAGS_OVERRIDE, but required with FASTR_RELEASE')
mx.log(args[0] + ' not found in PKG_LDFLAGS_OVERRIDE, assuming system location')
return 0
def pullLLVMBinaries(args=None):
"""downloads the LLVM binaries"""
toolDir = join(_toolDir, "tools/llvm")
mx.ensure_dir_exists(toolDir)
osStr = mx.get_os()
arch = mx.get_arch()
if osStr == "windows":
print "windows currently only supported with cygwin!"
return
elif osStr == "linux":
if arch == "amd64":
urls = ["https://lafo.ssw.uni-linz.ac.at/pub/sulong-deps/clang+llvm-3.2-x86_64-linux-ubuntu-12.04.tar.gz"]
else:
urls = ["https://lafo.ssw.uni-linz.ac.at/pub/sulong-deps/clang+llvm-3.2-x86-linux-ubuntu-12.04.tar.gz"]
elif osStr == "darwin":
urls = ["https://lafo.ssw.uni-linz.ac.at/pub/sulong-deps/clang+llvm-3.2-x86_64-apple-darwin11.tar.gz"]
elif osStr == "cygwin":
urls = ["https://lafo.ssw.uni-linz.ac.at/pub/sulong-deps/clang+llvm-3.2-x86-mingw32-EXPERIMENTAL.tar.gz"]
else:
print osStr, arch, "not supported!"
localPath = pullsuite(toolDir, urls)
tar(localPath, toolDir, stripLevels=1)
os.remove(localPath)
def machineArch(self):
return mx.get_arch()
# ----------------------------------------------------------------------------------------------------
import mx, mx_gate, mx_subst, mx_sdk, mx_sdk_vm, mx_graal_js, os, tarfile, tempfile, subprocess, sys
import mx_graal_nodejs_benchmark
from mx import BinarySuite, TimeStampFile
from mx_gate import Task
from argparse import ArgumentParser
from os.path import exists, join, isdir, pathsep, sep
from mx_graal_js import get_jdk
_suite = mx.suite('graal-nodejs')
_current_os = mx.get_os()
_is_windows = _current_os == 'windows'
_current_arch = mx.get_arch()
_config_files = [join(_suite.dir, f) for f in ('configure', 'configure.py')]
_generated_config_files = [
join(_suite.dir, f)
for f in ('config.gypi', 'config.status', 'configure.pyc', 'config.mk',
'icu_config.gypi')
]
class GraalNodeJsTags:
allTests = 'all'
unitTests = 'unit'
jniProfilerTests = 'jniprofiler'
windows = 'windows' # we cannot run `node-gyp` in our CI unless we install the "Visual Studio Build Tools" (using the "Visual C++ build tools" workload)
def hsdis(args, copyToDir=None):
"""download the hsdis library
This is needed to support HotSpot's assembly dumping features.
By default it downloads the Intel syntax version, use the 'att' argument to install AT&T syntax."""
flavor = None
if mx.get_arch() == "amd64":
flavor = mx.get_env('HSDIS_SYNTAX')
if flavor is None:
flavor = 'intel'
if 'att' in args:
flavor = 'att'
libpattern = mx.add_lib_suffix('hsdis-' + mx.get_arch() + '-' +
mx.get_os() + '-%s')
sha1s = {
r'att\hsdis-amd64-windows-%s.dll':
'bcbd535a9568b5075ab41e96205e26a2bac64f72',
r'att/hsdis-amd64-linux-%s.so':
'36a0b8e30fc370727920cc089f104bfb9cd508a0',
r'att/hsdis-amd64-darwin-%s.dylib':
'c1865e9a58ca773fdc1c5eea0a4dfda213420ffb',
r'intel\hsdis-amd64-windows-%s.dll':
'6a388372cdd5fe905c1a26ced614334e405d1f30',
r'intel/hsdis-amd64-linux-%s.so':
'0d031013db9a80d6c88330c42c983fbfa7053193',
r'intel/hsdis-amd64-darwin-%s.dylib':
'67f6d23cbebd8998450a88b5bef362171f66f11a',
r'hsdis-sparcv9-solaris-%s.so':
'970640a9af0bd63641f9063c11275b371a59ee60',
r'hsdis-sparcv9-linux-%s.so':
'0c375986d727651dee1819308fbbc0de4927d5d9',
r'hsdis-aarch64-linux-%s.so':
'fcc9b70ac91c00db8a50b0d4345490a68e3743e1',
}
if flavor:
flavoredLib = join(flavor, libpattern)
else:
flavoredLib = libpattern
if flavoredLib not in sha1s:
mx.warn(
"hsdis with flavor '{}' not supported on this platform or architecture"
.format(flavor))
return
sha1 = sha1s[flavoredLib]
lib = flavoredLib % (sha1)
path = join(_suite.get_output_root(), lib)
if not exists(path):
sha1path = path + '.sha1'
mx.download_file_with_sha1(
'hsdis',
path, [
rewriteurl(
'https://lafo.ssw.uni-linz.ac.at/pub/graal-external-deps/hsdis/'
+ lib.replace(os.sep, '/'))
],
sha1,
sha1path,
True,
True,
sources=False)
overwrite = True
if copyToDir is None:
# Try install hsdis into JAVA_HOME
overwrite = False
jdk = mx.get_jdk()
base = jdk.home
if exists(join(base, 'jre')):
base = join(base, 'jre')
if mx.get_os() == 'darwin':
copyToDir = join(base, 'lib')
elif mx.get_os() == 'windows':
copyToDir = join(base, 'bin')
else:
if jdk.javaCompliance >= '11':
copyToDir = join(base, 'lib')
else:
copyToDir = join(base, 'lib', mx.get_arch())
if exists(copyToDir):
dest = join(copyToDir, mx.add_lib_suffix('hsdis-' + mx.get_arch()))
if exists(dest) and not overwrite:
import filecmp
# Only issue warning if existing lib is different
if filecmp.cmp(path, dest) is False:
mx.warn('Not overwriting existing {} with {}'.format(
dest, path))
else:
try:
shutil.copy(path, dest)
mx.log('Copied {} to {}'.format(path, dest))
except IOError as e:
mx.warn('Could not copy {} to {}: {}'.format(
path, dest, str(e)))
# ensure benchmark counters still work
with JVMCIMode('jit'):
with Task('DaCapo_pmd:BenchmarkCounters:product', tasks, tags=[GraalTags.fulltest]) as t:
if t: dacapo(_noneAsEmptyList(extraVMarguments) + ['-G:+LIRProfileMoves', '-G:+GenericDynamicCounters', '-XX:JVMCICounterSize=10', 'pmd'])
# ensure -Xcomp still works
with JVMCIMode('jit'):
with Task('XCompMode:product', tasks, tags=[GraalTags.fulltest]) as t:
if t: run_vm(_noneAsEmptyList(extraVMarguments) + ['-Xcomp', '-version'])
graal_unit_test_runs = [
UnitTestRun('UnitTests', [], tags=[GraalTags.test]),
]
_registers = 'o0,o1,o2,o3,f8,f9,d32,d34' if mx.get_arch() == 'sparcv9' else 'rbx,r11,r10,r14,xmm3,xmm11,xmm14'
graal_bootstrap_tests = [
BootstrapTest('BootstrapWithSystemAssertions', 'fastdebug', ['-esa', '-G:+VerifyGraalGraphs', '-G:+VerifyGraalGraphEdges'], tags=[GraalTags.bootstrap]),
BootstrapTest('BootstrapWithSystemAssertionsNoCoop', 'fastdebug', ['-esa', '-XX:-UseCompressedOops', '-G:+ExitVMOnException'], tags=[GraalTags.fulltest]),
BootstrapTest('BootstrapWithGCVerification', 'product', ['-XX:+UnlockDiagnosticVMOptions', '-XX:+VerifyBeforeGC', '-XX:+VerifyAfterGC', '-G:+ExitVMOnException'], tags=[GraalTags.fulltest], suppress=['VerifyAfterGC:', 'VerifyBeforeGC:']),
BootstrapTest('BootstrapWithG1GCVerification', 'product', ['-XX:+UnlockDiagnosticVMOptions', '-XX:-UseSerialGC', '-XX:+UseG1GC', '-XX:+VerifyBeforeGC', '-XX:+VerifyAfterGC', '-G:+ExitVMOnException'], tags=[GraalTags.fulltest], suppress=['VerifyAfterGC:', 'VerifyBeforeGC:']),
BootstrapTest('BootstrapEconomyWithSystemAssertions', 'fastdebug', ['-esa', '-Djvmci.Compiler=graal-economy', '-G:+ExitVMOnException'], tags=[GraalTags.fulltest]),
BootstrapTest('BootstrapWithExceptionEdges', 'fastdebug', ['-esa', '-G:+StressInvokeWithExceptionNode', '-G:+ExitVMOnException'], tags=[GraalTags.fulltest]),
BootstrapTest('BootstrapWithRegisterPressure', 'product', ['-esa', '-G:RegisterPressure=' + _registers, '-G:+ExitVMOnException', '-G:+LIRUnlockBackendRestart'], tags=[GraalTags.fulltest]),
BootstrapTest('BootstrapTraceRAWithRegisterPressure', 'product', ['-esa', '-G:+TraceRA', '-G:RegisterPressure=' + _registers, '-G:+ExitVMOnException', '-G:+LIRUnlockBackendRestart'], tags=[GraalTags.fulltest]),
BootstrapTest('BootstrapWithImmutableCode', 'product', ['-esa', '-G:+ImmutableCode', '-G:+VerifyPhases', '-G:+ExitVMOnException'], tags=[GraalTags.fulltest]),
]
def _graal_gate_runner(args, tasks):
def platform_name():
return mx.get_os() + "-" + mx.get_arch()
if t:
dacapo(
_noneAsEmptyList(extraVMarguments)
+ ["-G:+LIRProfileMoves", "-G:+GenericDynamicCounters", "-XX:JVMCICounterSize=10", "pmd"]
)
# ensure -Xcomp still works
with JVMCIMode("jit"):
with Task("XCompMode:product", tasks, tags=[GraalTags.fulltest]) as t:
if t:
run_vm(_noneAsEmptyList(extraVMarguments) + ["-Xcomp", "-version"])
graal_unit_test_runs = [UnitTestRun("UnitTests", [], tags=[GraalTags.test])]
_registers = "o0,o1,o2,o3,f8,f9,d32,d34" if mx.get_arch() == "sparcv9" else "rbx,r11,r10,r14,xmm3,xmm11,xmm14"
graal_bootstrap_tests = [
BootstrapTest("BootstrapWithSystemAssertions", "fastdebug", ["-esa"], tags=[GraalTags.bootstrap]),
BootstrapTest(
"BootstrapWithSystemAssertionsNoCoop",
"fastdebug",
["-esa", "-XX:-UseCompressedOops", "-G:+ExitVMOnException"],
tags=[GraalTags.fulltest],
),
BootstrapTest(
"BootstrapWithGCVerification",
"product",
["-XX:+UnlockDiagnosticVMOptions", "-XX:+VerifyBeforeGC", "-XX:+VerifyAfterGC", "-G:+ExitVMOnException"],
tags=[GraalTags.fulltest],
suppress=["VerifyAfterGC:", "VerifyBeforeGC:"],
