def collect_unique_dependencies(self, path, benchmark):
deps = super(ScalaDaCapoNativeImageBenchmarkSuite,
self).collect_dependencies(path)
# if there are more versions of the same jar, we choose one and omit remaining from the classpath
if benchmark in _scala_daCapo_exclude_lib:
for lib in _scala_daCapo_exclude_lib[benchmark]:
lib_path = mx.join(path, lib)
if lib_path in deps:
deps.remove(mx.join(path, lib))
return deps
def collect_group_dependencies(self, group, scala_version):
if group == 'harness':
if scala_version == 12:
unpacked_renaissance = RenaissanceNativeImageBenchmarkSuite.renaissance_unpacked(self.suite)
path = mx.join(unpacked_renaissance, 'renaissance-harness')
else:
path = RenaissanceNativeImageBenchmarkSuite.harness_path(self.suite)
else:
unpacked_renaissance = RenaissanceNativeImageBenchmarkSuite.renaissance_unpacked(self.suite)
path = mx.join(unpacked_renaissance, 'benchmarks', group)
return self.get_dependencies(path, group)
def collect_group_dependencies(self, group, requires_recompiled_harness):
if group == 'harness':
if requires_recompiled_harness:
path = RenaissanceNativeImageBenchmarkSuite.harness_path(self.suite)
else:
unpacked_renaissance = RenaissanceNativeImageBenchmarkSuite.renaissance_unpacked(self.suite)
path = mx.join(unpacked_renaissance, 'renaissance-harness')
else:
unpacked_renaissance = RenaissanceNativeImageBenchmarkSuite.renaissance_unpacked(self.suite)
path = mx.join(unpacked_renaissance, 'benchmarks', group)
return self.get_dependencies(path, group)
def _archivable_results(self, target_arch, use_relpath, single):
def result(base_dir, file_path):
assert not mx.isabs(file_path)
archive_path = file_path if use_relpath else mx.basename(file_path)
return mx.join(base_dir, file_path), archive_path
yield result(mx.join(self.out_dir, target_arch), self._target)
if not single:
for header in os.listdir(mx.join(self.dir, self.include)):
yield result(self.dir, mx.join(self.include, header))
def _archivable_results(self, target_arch, use_relpath, single):
out_dir_arch = self._install_dir
for file_path in self.getResults():
assert not mx.isabs(file_path)
abs_path = mx.join(out_dir_arch, file_path)
archive_path = file_path if use_relpath else mx.basename(file_path)
# if test.skip exists the test should be skipped
if mx.exists(mx.join(mx.dirname(abs_path), "test.skip")):
continue
yield abs_path, archive_path
def patches(self):
"""A list of patches that will be applied during a build."""
os_arch_dir = mx.join(self.source_dirs()[0],
'{}-{}'.format(mx.get_os(), mx.get_arch()))
if mx.exists(os_arch_dir):
return [
mx.join(os_arch_dir, patch)
for patch in os.listdir(os_arch_dir)
]
others_dir = mx.join(self.source_dirs()[0], 'others')
return [mx.join(others_dir, patch) for patch in os.listdir(others_dir)]
def create_dacapo_classpath(self, dacapo_path, benchmark):
dacapo_nested_resources = []
dacapo_dat_resources = []
dacapo_extracted = self.extract_dacapo(dacapo_path)
benchmark_resources = self.benchmark_resources(benchmark)
if benchmark_resources:
for resource in benchmark_resources:
dacapo_dat_resource = extract_archive(mx.join(dacapo_extracted, resource), benchmark)
dat_resource_name = os.path.splitext(os.path.basename(resource))[0]
dacapo_dat_resources.append(mx.join(dacapo_dat_resource, dat_resource_name))
#collects nested jar files and classes directories
dacapo_nested_resources += self.collect_nested_dependencies(dacapo_dat_resource)
return dacapo_extracted, dacapo_dat_resources, dacapo_nested_resources
def getArchivableResults(self, use_relpath=True, single=False):
for target_arch in self.multiarch if self._use_multiarch else [mx.get_arch()]:
toolchain = _Toolchain.for_(target_arch)
target_arch_path = toolchain.target if self.multiarch else ''
if toolchain.is_native or not single:
for file_path, archive_path in self._archivable_results(target_arch, use_relpath, single):
yield file_path, mx.join(target_arch_path, archive_path)
def get_dependencies(self, group):
deps = []
for jar in list_jars(group):
deps.append(
RenaissanceNativeImageBenchmarkSuite.RenaissanceDependency(
os.path.basename(jar), mx.join(group, jar)))
return deps
def _build_aot_images(benchmarks):
print('---------------- BEGIN BUILDING AOT IMAGES ----------------')
output_dir = mx.join(_suite.get_output_root(), 'bin')
mx.ensure_dir_exists(output_dir)
for bench_name, bench in benchmarks.items():
witness = mx.TimeStampFile(mx.join(output_dir, bench_name))
if not witness.exists() or witness.isOlderThan(bench.src_file):
cmd = [
jdk.aot_image, '-cp', bench.classpath,
'-H:Name={}'.format(bench_name),
'-H:Class={}'.format(bench.class_file)
]
cmd.extend(bench.build_args)
print(' '.join(cmd))
mx.run(cmd, cwd=output_dir)
print('---------------- END BUILDING AOT IMAGES ----------------')
def aot_benchmark(args):
for bench in args:
if bench not in _benchmarks:
raise ValueError("'{}' is not an AOT benchmark.".format(bench))
project = _get_project(_project_name)
benchmarks = {bench: _benchmarks[bench]
for bench in args} if args else _benchmarks
class_files = _build_aot_images(project, benchmarks)
print('---------------- BEGIN RUNNING AOT BENCHMARKS ----------------')
for bench in benchmarks:
common_args = ['-XX:+PrintGC']
print('------------ BEGIN {} BENCHMARK ------------'.format(
bench.upper()))
print('-------- BEGIN GRAAL RUN --------')
graal_cmd = [jdk.java] + common_args
graal_cmd.append(class_files[bench])
graal_cmd.extend(benchmarks[bench][0])
print(' '.join(graal_cmd))
mx.run(graal_cmd, cwd=project.output_dir())
print('-------- END GRAAL RUN --------')
print('-------- BEGIN SVM RUN --------')
svm_cmd = [mx.join('.', bench)] + common_args
svm_cmd.append('10') # set fixed number of warm up rounds
svm_cmd.extend(benchmarks[bench][0][1:])
print(' '.join(svm_cmd))
mx.run(svm_cmd, cwd=project.output_dir())
print('-------- END SVM RUN --------')
print('------------ END {} BENCHMARK ------------'.format(
bench.upper()))
print('---------------- END RUNNING AOT BENCHMARKS ----------------')
def extract_archive(path, extracted_name):
extracted_archive = mx.join(mx.dirname(path), extracted_name)
if not mx.exists(extracted_archive):
# There can be multiple processes doing this so be atomic about it
with mx.SafeDirectoryUpdater(extracted_archive, create=True) as sdu:
with zipfile.ZipFile(path, 'r') as zf:
zf.extractall(sdu.directory)
return extracted_archive
def extract_archive(path, extracted_name):
extracted_archive = mx.join(mx.dirname(path), extracted_name)
if not mx.exists(extracted_archive):
os.makedirs(extracted_archive)
arc = zipfile.ZipFile(path, 'r')
arc.extractall(extracted_archive)
arc.close()
return extracted_archive
def __init__(self, suite, name, subDir, srcDirs, deps, workingSets, d, kind, **kwargs):
self.deliverable = kwargs.pop('deliverable', name.split('.')[-1])
if srcDirs:
mx.abort('"sourceDirs" is not supported for default native projects')
srcDirs += [self.include, self.src]
super(DefaultNativeProject, self).__init__(suite, name, subDir, srcDirs, deps, workingSets, d, **kwargs)
try:
self._kind = self._kinds[kind]
except KeyError:
mx.abort('"native" should be one of {}, but "{}" is given'.format(list(self._kinds.keys()), kind))
include_dir = mx.join(self.dir, self.include)
if next(os.walk(include_dir))[1]:
mx.abort('include directory must have a flat structure')
self.include_dirs = [include_dir]
if kind == 'static_lib':
self.libs = [mx.join(self.out_dir, mx.get_arch(), self._target)]
def get_dependencies(self, group):
deps = []
for f in os.listdir(group):
f_path = mx.join(group, f)
if os.path.isfile(f_path) and f.endswith('.jar'):
deps.append(
RenaissanceNativeImageBenchmarkSuite.
RenaissanceDependency(os.path.basename(f), f_path))
return deps
def renaissance_unpacked(self):
renaissance_unpacked = mx.join(mx.dirname(self.renaissancePath()),
'renaissance.extracted')
if not mx.exists(renaissance_unpacked):
os.makedirs(renaissance_unpacked)
arc = zipfile.ZipFile(self.renaissancePath(), 'r')
arc.extractall(renaissance_unpacked)
arc.close()
return renaissance_unpacked
def _archivable_results(self, target_arch, use_relpath, single):
def result(base_dir, file_path):
assert not mx.isabs(file_path)
archive_path = file_path if use_relpath else mx.basename(file_path)
return mx.join(base_dir, file_path), archive_path
out_dir_arch = mx.join(self.out_dir, target_arch)
for _result in self.getResults():
yield result(out_dir_arch, _result)
def __init__(self,
args,
project,
target_arch=mx.get_arch(),
ninja_targets=None):
super(NinjaBuildTask, self).__init__(args, project, target_arch)
self._reason = None
self._manifest = mx.join(self.out_dir, Ninja.default_manifest)
self.ninja = Ninja(self.out_dir,
self.parallelism,
targets=ninja_targets)
def get_dependencies(self, path, group):
deps = []
for jar in list_jars(path):
deps.append(
RenaissanceNativeImageBenchmarkSuite.RenaissanceDependency(
os.path.basename(jar), mx.join(path, jar)))
if group in _renaissance_exclude_lib:
for lib in _renaissance_exclude_lib[group]:
lib_dep = RenaissanceNativeImageBenchmarkSuite.RenaissanceDependency(
lib, mx.join(path, lib))
if lib_dep in deps:
deps.remove(lib_dep)
if group in _renaissance_additional_lib:
for lib in _renaissance_additional_lib[group]:
lib_path = RenaissanceNativeImageBenchmarkSuite.renaissance_additional_lib(
self.suite, lib)
deps.append(
RenaissanceNativeImageBenchmarkSuite.
RenaissanceDependency(os.path.basename(lib_path),
lib_path))
return deps
class LibtoolNativeProject(mx.NativeProject, # pylint: disable=too-many-ancestors
mx_native.NativeDependency):
include_dirs = property(lambda self: [mx.join(self.getOutput(), 'include')])
libs = property(lambda self: [next(self.getArchivableResults(single=True))[0]])
def getArchivableResults(self, use_relpath=True, single=False):
for file_path, archive_path in super(LibtoolNativeProject, self).getArchivableResults(use_relpath):
path_in_lt_objdir = mx.basename(mx.dirname(file_path)) == '.libs'
yield file_path, mx.basename(archive_path) if path_in_lt_objdir else archive_path
if single:
assert path_in_lt_objdir, 'the first build result must be from LT_OBJDIR'
break
def generate_manifest(self, path):
unsupported_source_files = list(
set(self._source['files'].keys()) - {'.h', '.c', '.cc', '.S'})
if unsupported_source_files:
mx.abort(
'{} source files are not supported by default native projects'.
format(unsupported_source_files))
with NinjaManifestGenerator(self, open(path, 'w')) as gen:
gen.comment("Toolchain configuration")
gen.include(mx.join(self.toolchain.get_output(),
'toolchain.ninja'))
gen.newline()
gen.variables(cflags=[
mx_subst.path_substitutions.substitute(cflag)
for cflag in self.cflags
])
if self._kind != self._kinds['static_lib']:
gen.variables(
ldflags=[
mx_subst.path_substitutions.substitute(ldflag)
for ldflag in self.ldflags
],
ldlibs=self.ldlibs,
)
gen.include_dirs(
collections.OrderedDict.fromkeys(
# remove the duplicates while maintaining the ordering
[mx.dirname(h_file) for h_file in self.h_files] + list(
itertools.chain.from_iterable(
getattr(d, 'include_dirs', [])
for d in self.buildDependencies))).keys())
gen.comment('Compiled project sources')
object_files = [gen.cc(f) for f in self.c_files]
gen.newline()
object_files += [gen.cxx(f) for f in self.cxx_files]
gen.newline()
object_files += [gen.asm(f) for f in self.asm_sources]
gen.newline()
gen.comment('Project deliverable')
if self._kind == self._kinds['static_lib']:
gen.ar(self._target, object_files)
else:
link = gen.linkxx if self.cxx_files else gen.link
dep_libs = list(
itertools.chain.from_iterable(
getattr(d, 'libs', [])
for d in self.buildDependencies))
link(self._target, object_files + dep_libs)
def get_dependencies(self, path, group):
deps = []
for jar in list_jars(path):
deps.append(
RenaissanceNativeImageBenchmarkSuite.RenaissanceDependency(
os.path.basename(jar), mx.join(path, jar)))
if self.suite.version() in ["0.9.0", "0.10.0", "0.11.0"]:
if group == 'apache-spark':
# breeze jar is replaced with a patched jar because of IncompatibleClassChange errors due to a bug in the Scala compiler
invalid_bytecode_jar = 'breeze_2.11-0.11.2.jar'
lib_dep = RenaissanceNativeImageBenchmarkSuite.RenaissanceDependency(
invalid_bytecode_jar,
mx.join(path, invalid_bytecode_jar))
if lib_dep in deps:
deps.remove(lib_dep)
lib_path = RenaissanceNativeImageBenchmarkSuite.renaissance_additional_lib(
self.suite, 'SPARK_BREEZE_PATCHED')
deps.append(
RenaissanceNativeImageBenchmarkSuite.
RenaissanceDependency(os.path.basename(lib_path),
lib_path))
return deps
def _ninja_deps(cls): # pylint: disable=no-self-argument
deps = []
try:
subprocess.check_output(['ninja', '--version'], stderr=subprocess.STDOUT)
except OSError:
dep = mx.library('NINJA', False)
if dep:
deps.append(dep.qualifiedName())
Ninja.binary = mx.join(dep.get_path(False), 'ninja')
else:
# necessary until GR-13214 is resolved
mx.warn('Make `ninja` binary available via PATH to build native projects.')
try:
import ninja_syntax # pylint: disable=unused-variable, unused-import
except ImportError:
dep = mx.library('NINJA_SYNTAX')
deps.append(dep.qualifiedName())
module_path = mx.join(dep.get_path(False), 'ninja_syntax-{}'.format(dep.version))
mx.ensure_dir_exists(module_path) # otherwise, import machinery will ignore it
sys.path.append(module_path)
return deps
def _source(self):
source_tree = []
source_files = collections.defaultdict(list)
for source_dir in self.source_dirs():
for root, _, files in os.walk(source_dir):
rel_root = os.path.relpath(root, self.dir)
source_tree.append(rel_root)
# group files by extension
grouping = collections.defaultdict(list)
for f in files:
grouping[os.path.splitext(f)[1]].append(mx.join(rel_root, f))
for ext in grouping.keys():
source_files[ext] += grouping[ext]
return dict(tree=source_tree, files=source_files)
def create_classpath(self, benchmark):
dacapo_nested_resources = []
dacapo_dat_resources = []
dacapo_extracted = self.extract_dacapo()
benchmark_resources = _dacapo_resources[benchmark]
if benchmark_resources:
for resource in benchmark_resources:
dacapo_dat_resource = extract_archive(
mx.join(dacapo_extracted, resource), benchmark)
dacapo_dat_resources.append(dacapo_dat_resource)
#collects nested jar files and classes directories
dacapo_nested_resources += self.collect_nested_dependencies(
dacapo_dat_resource)
dacapo_jars = self.collect_dependencies(
os.path.join(dacapo_extracted, 'jar'))
cp = ':'.join([dacapo_extracted] + dacapo_jars + dacapo_dat_resources +
dacapo_nested_resources)
return cp
def __init__(self,
suite,
name,
deps,
workingSets,
subDir,
buildRef=True,
buildSharedObject=False,
bundledLLVMOnly=False,
ninja_install_targets=None,
max_jobs=8,
cmakeConfigVariant=None,
**args):
if bundledLLVMOnly and mx.get_env('CLANG_CC', None):
self.ignore = "Environment variable 'CLANG_CC' is set but project specifies 'bundledLLVMOnly'"
if 'buildDependencies' not in args:
args['buildDependencies'] = []
if 'sdk:LLVM_TOOLCHAIN' not in args['buildDependencies']:
args['buildDependencies'].append('sdk:LLVM_TOOLCHAIN')
self.buildRef = buildRef
self.buildSharedObject = buildSharedObject
self.current_variant = None
self.cmake_config_variant = cmakeConfigVariant or {}
super(SulongCMakeTestSuite,
self).__init__(suite,
name,
deps,
workingSets,
subDir,
ninja_install_targets=ninja_install_targets
or ["install"],
max_jobs=max_jobs,
**args)
self._install_dir = mx.join(self.out_dir, "result")
# self._ninja_targets = self.getResults()
_config = self._cmake_config_raw
_module_path = mx.Suite._pop_list(_config, 'CMAKE_MODULE_PATH', self)
_module_path.append('<path:com.oracle.truffle.llvm.tests.cmake>')
_config['CMAKE_MODULE_PATH'] = ';'.join(_module_path)
_config[
'SULONG_MODULE_PATH'] = '<path:com.oracle.truffle.llvm.tests.cmake>'
self._init_cmake = False
def _build_aot_images(project, benchmarks):
print('---------------- BEGIN BUILDING AOT IMAGES ----------------')
output_dir = project.output_dir()
class_files = {}
for bench in benchmarks:
bench_class, (bench_src, _) = project.find_classes_with_annotations(
None, ['@AOTBench("{}")'.format(bench)]).popitem()
class_files[bench] = bench_class
witness = mx.TimeStampFile(mx.join(output_dir, bench))
if not witness.exists() or witness.isOlderThan(bench_src):
cmd = [
jdk.aot_image, '-cp', '.', '-H:Name={}'.format(bench),
'-H:Class={}'.format(bench_class)
]
cmd.extend(benchmarks[bench][1])
print(' '.join(cmd))
mx.run(cmd, cwd=output_dir)
print('---------------- END BUILDING AOT IMAGES ----------------')
return class_files
def __init__(self, vm, bm_suite, args):
self.bmSuite = bm_suite
self.benchmark_suite_name = bm_suite.benchSuiteName(args) if len(inspect.getargspec(bm_suite.benchSuiteName).args) > 1 else bm_suite.benchSuiteName() # pylint: disable=deprecated-method
self.benchmark_name = bm_suite.benchmarkName()
self.extra_image_build_arguments = bm_suite.extra_image_build_argument(self.benchmark_name, args)
self.extra_run_args = bm_suite.extra_run_arg(self.benchmark_name, args)
self.extra_agent_run_args = bm_suite.extra_agent_run_arg(self.benchmark_name, args)
self.extra_profile_run_args = bm_suite.extra_profile_run_arg(self.benchmark_name, args)
self.extra_agent_profile_run_args = bm_suite.extra_agent_profile_run_arg(self.benchmark_name, args)
self.benchmark_output_dir = bm_suite.benchmark_output_dir(self.benchmark_name, args)
self.pgo_iteration_num = None
self.params = ['extra-image-build-argument', 'extra-run-arg', 'extra-agent-run-arg', 'extra-profile-run-arg',
'extra-agent-profile-run-arg', 'benchmark-output-dir', 'stages', 'skip-agent-assertions']
self.profile_file_extension = '.iprof'
self.stages = bm_suite.stages(args)
self.last_stage = self.stages[-1]
self.skip_agent_assertions = bm_suite.skip_agent_assertions(self.benchmark_name, args)
self.root_dir = self.benchmark_output_dir if self.benchmark_output_dir else mx.join(mx.suite('vm').dir, 'mxbuild')
self.executable_suffix = ('-' + self.benchmark_name) if self.benchmark_name else ''
self.executable, self.classpath_arguments, self.system_properties, self.image_run_args = NativeImageVM.extract_benchmark_arguments(args)
self.executable_name = (os.path.splitext(os.path.basename(self.executable[1]))[0] + self.executable_suffix if self.executable[0] == '-jar' else self.executable[0] + self.executable_suffix).lower()
self.final_image_name = self.executable_name + '-' + vm.config_name()
self.output_dir = mx.join(os.path.abspath(self.root_dir), 'native-image-bench-' + self.executable_name + '-' + vm.config_name())
self.profile_path_no_extension = os.path.join(self.output_dir, self.executable_name)
self.latest_profile_path = self.profile_path_no_extension + '-latest' + self.profile_file_extension
self.config_dir = os.path.join(self.output_dir, 'config')
self.log_dir = self.output_dir
self.base_image_build_args = [os.path.join(mx_sdk_vm_impl.graalvm_home(fatalIfMissing=True), 'bin', 'native-image')]
self.base_image_build_args += ['--no-fallback', '--no-server', '-g', '--allow-incomplete-classpath']
self.base_image_build_args += ['-J-ea', '-J-esa', '-H:+VerifyGraalGraphs', '-H:+VerifyPhases'] if vm.is_gate else []
self.base_image_build_args += self.system_properties
self.base_image_build_args += self.classpath_arguments
self.base_image_build_args += self.executable
self.base_image_build_args += ['-H:Path=' + self.output_dir]
self.base_image_build_args += ['-H:ConfigurationFileDirectories=' + self.config_dir]
if vm.is_llvm:
self.base_image_build_args += ['-H:CompilerBackend=llvm', '-H:Features=org.graalvm.home.HomeFinderFeature', '-H:DeadlockWatchdogInterval=0']
self.base_image_build_args += self.extra_image_build_arguments
def _run(benchmarks):
print('---------------- BEGIN RUNNING AOT BENCHMARKS ----------------')
for bench_name, bench in benchmarks.items():
common_args = ['-XX:+PrintGC']
print('------------ BEGIN {} BENCHMARK ------------'.format(
bench_name.upper()))
print('-------- BEGIN GRAAL RUN --------')
graal_cmd = [jdk.java, '-cp', bench.classpath] + common_args
graal_cmd.append(bench.class_file)
graal_cmd.extend(bench.runtime_args)
print(' '.join(graal_cmd))
mx.run(graal_cmd, cwd=_suite.get_output_root())
print('-------- END GRAAL RUN --------')
print('-------- BEGIN SVM RUN --------')
svm_cmd = [mx.join('bin', bench_name)] + common_args
svm_cmd.append('10') # set fixed number of warm up rounds
svm_cmd.extend(bench.runtime_args[1:])
print(' '.join(svm_cmd))
mx.run(svm_cmd, cwd=_suite.get_output_root())
print('-------- END SVM RUN --------')
print('------------ END {} BENCHMARK ------------'.format(
bench_name.upper()))
print('---------------- END RUNNING AOT BENCHMARKS ----------------')
def collect_dependencies(path):
deps = []
for f in list_jars(path):
deps.append(mx.join(path, f))
return deps
