def compiler_gate_runner(suites, unit_test_runs, bootstrap_tests, tasks, extraVMarguments=None):
# Build server-hosted-jvmci now so we can run the unit tests
with Task('BuildHotSpotGraalHosted: product', tasks, tags=[GraalTags.test, GraalTags.fulltest]) as t:
if t: buildvms(['--vms', 'server', '--builds', 'product'])
with VM('server', 'product'):
# Run unit tests on server-hosted-jvmci
for r in unit_test_runs:
r.run(suites, tasks, extraVMarguments)
# Run microbench on server-hosted-jvmci (only for testing the JMH setup)
with VM('server', 'product'):
for r in [MicrobenchRun('Microbench', ['TestJMH'], tags=[GraalTags.fulltest])]:
r.run(tasks, extraVMarguments)
# Run ctw against rt.jar on server-hosted-jvmci
with VM('server', 'product'):
with Task('CTW:hosted-product', tasks, tags=[GraalTags.fulltest]) as t:
if t: ctw(['--ctwopts', '-Inline +ExitVMOnException', '-esa', '-G:+CompileTheWorldMultiThreaded', '-G:-InlineDuringParsing', '-G:-CompileTheWorldVerbose', '-XX:ReservedCodeCacheSize=400m'], _noneAsEmptyList(extraVMarguments))
# Build the jvmci VMs so we can run the other tests
with Task('BuildHotSpotGraalJVMCI: fastdebug', tasks, tags=[GraalTags.bootstrap, GraalTags.fulltest]) as t:
if t: buildvms(['--vms', 'jvmci', '--builds', 'fastdebug'])
with Task('BuildHotSpotGraalJVMCI: product', tasks, tags=[GraalTags.fulltest]) as t:
if t: buildvms(['--vms', 'jvmci', '--builds', 'product'])
# bootstrap tests
for b in bootstrap_tests:
b.run(tasks, extraVMarguments)
# run dacapo sanitychecks
for vmbuild in ['fastdebug', 'product']:
for test in sanitycheck.getDacapos(level=sanitycheck.SanityCheckLevel.Gate, gateBuildLevel=vmbuild, extraVmArguments=extraVMarguments) \
+ sanitycheck.getScalaDacapos(level=sanitycheck.SanityCheckLevel.Gate, gateBuildLevel=vmbuild, extraVmArguments=extraVMarguments):
with Task(str(test) + ':' + vmbuild, tasks, tags=[GraalTags.fulltest]) as t:
if t and not test.test('jvmci'):
t.abort(test.name + ' Failed')
# ensure -Xbatch still works
with VM('jvmci', 'product'):
with Task('DaCapo_pmd:BatchMode:product', tasks, tags=[GraalTags.fulltest]) as t:
if t: dacapo(_noneAsEmptyList(extraVMarguments) + ['-Xbatch', 'pmd'])
# ensure benchmark counters still work
with VM('jvmci', 'product'):
with Task('DaCapo_pmd:BenchmarkCounters:product', tasks, tags=[GraalTags.fulltest]) as t:
if t: dacapo(_noneAsEmptyList(extraVMarguments) + ['-G:+LIRProfileMoves', '-G:+GenericDynamicCounters', '-XX:JVMCICounterSize=10', 'pmd'])
# ensure -Xcomp still works
with VM('jvmci', 'product'):
with Task('XCompMode:product', tasks, tags=[GraalTags.fulltest]) as t:
if t: run_vm(_noneAsEmptyList(extraVMarguments) + ['-Xcomp', '-version'])
def compiler_gate_runner(suites, unit_test_runs, bootstrap_tests, tasks, extraVMarguments=None):
# Build server-hosted-jvmci now so we can run the unit tests
with Task('BuildHotSpotGraalHosted: product', tasks, tags=[GraalTags.test, GraalTags.fulltest]) as t:
if t: buildvms(['--vms', 'server', '--builds', 'product'])
with VM('server', 'product'):
# Run unit tests on server-hosted-jvmci
for r in unit_test_runs:
r.run(suites, tasks, extraVMarguments)
# Run microbench on server-hosted-jvmci (only for testing the JMH setup)
with VM('server', 'product'):
for r in [MicrobenchRun('Microbench', ['TestJMH'], tags=[GraalTags.fulltest])]:
r.run(tasks, extraVMarguments)
# Run ctw against rt.jar on server-hosted-jvmci
with VM('server', 'product'):
with Task('CTW:hosted-product', tasks, tags=[GraalTags.fulltest]) as t:
if t: ctw(['--ctwopts', '-Inline +ExitVMOnException', '-esa', '-G:+CompileTheWorldMultiThreaded', '-G:-InlineDuringParsing', '-G:-CompileTheWorldVerbose', '-XX:ReservedCodeCacheSize=400m'], _noneAsEmptyList(extraVMarguments))
# Build the jvmci VMs so we can run the other tests
with Task('BuildHotSpotGraalJVMCI: fastdebug', tasks, tags=[GraalTags.bootstrap, GraalTags.fulltest]) as t:
if t: buildvms(['--vms', 'jvmci', '--builds', 'fastdebug'])
with Task('BuildHotSpotGraalJVMCI: product', tasks, tags=[GraalTags.fulltest]) as t:
if t: buildvms(['--vms', 'jvmci', '--builds', 'product'])
# bootstrap tests
for b in bootstrap_tests:
b.run(tasks, extraVMarguments)
# run dacapo sanitychecks
for vmbuild in ['fastdebug', 'product']:
for test in sanitycheck.getDacapos(level=sanitycheck.SanityCheckLevel.Gate, gateBuildLevel=vmbuild, extraVmArguments=extraVMarguments) \
+ sanitycheck.getScalaDacapos(level=sanitycheck.SanityCheckLevel.Gate, gateBuildLevel=vmbuild, extraVmArguments=extraVMarguments):
with Task(str(test) + ':' + vmbuild, tasks, tags=[GraalTags.fulltest]) as t:
if t and not test.test('jvmci'):
t.abort(test.name + ' Failed')
# ensure -Xbatch still works
with VM('jvmci', 'product'):
with Task('DaCapo_pmd:BatchMode:product', tasks, tags=[GraalTags.fulltest]) as t:
if t: dacapo(_noneAsEmptyList(extraVMarguments) + ['-Xbatch', 'pmd'])
# ensure benchmark counters still work
with VM('jvmci', 'product'):
with Task('DaCapo_pmd:BenchmarkCounters:product', tasks, tags=[GraalTags.fulltest]) as t:
if t: dacapo(_noneAsEmptyList(extraVMarguments) + ['-G:+LIRProfileMoves', '-G:+GenericDynamicCounters', '-XX:JVMCICounterSize=10', 'pmd'])
# ensure -Xcomp still works
with VM('jvmci', 'product'):
with Task('XCompMode:product', tasks, tags=[GraalTags.fulltest]) as t:
if t: run_vm(_noneAsEmptyList(extraVMarguments) + ['-Xcomp', '-version'])
def compiler_gate_runner(suites,
unit_test_runs,
bootstrap_tests,
tasks,
extraVMarguments=None):
# Run unit tests in hosted mode
with JVMCIMode('hosted'):
for r in unit_test_runs:
r.run(suites, tasks, extraVMarguments)
# Run ctw against rt.jar on server-hosted-jvmci
with JVMCIMode('hosted'):
with Task('CTW:hosted', tasks) as t:
if t:
ctw([
'--ctwopts', '-Inline +ExitVMOnException', '-esa',
'-G:+CompileTheWorldMultiThreaded',
'-G:-InlineDuringParsing', '-G:-CompileTheWorldVerbose',
'-XX:ReservedCodeCacheSize=300m'
], _noneAsEmptyList(extraVMarguments))
# bootstrap tests
for b in bootstrap_tests:
b.run(tasks, extraVMarguments)
# run dacapo sanitychecks
for test in sanitycheck.getDacapos(level=sanitycheck.SanityCheckLevel.Gate, gateBuildLevel='release', extraVmArguments=extraVMarguments) \
+ sanitycheck.getScalaDacapos(level=sanitycheck.SanityCheckLevel.Gate, gateBuildLevel='release', extraVmArguments=extraVMarguments):
with Task(str(test) + ':' + 'release', tasks) as t:
if t and not test.test('jvmci'):
t.abort(test.name + ' Failed')
# ensure -Xbatch still works
with JVMCIMode('jit'):
with Task('DaCapo_pmd:BatchMode', tasks) as t:
if t:
dacapo(_noneAsEmptyList(extraVMarguments) + ['-Xbatch', 'pmd'])
# ensure -Xcomp still works
with JVMCIMode('jit'):
with Task('XCompMode:product', tasks) as t:
if t:
run_vm(
_noneAsEmptyList(extraVMarguments) +
['-Xcomp', '-version'])
def compiler_gate_runner(suites, unit_test_runs, bootstrap_tests, tasks, extraVMarguments=None):
# Run unit tests in hosted mode
with JVMCIMode('hosted'):
for r in unit_test_runs:
r.run(suites, tasks, extraVMarguments)
# Run microbench in hosted mode (only for testing the JMH setup)
with JVMCIMode('hosted'):
for r in [MicrobenchRun('Microbench', ['TestJMH'])]:
r.run(tasks, extraVMarguments)
# Run ctw against rt.jar on server-hosted-jvmci
with JVMCIMode('hosted'):
with Task('CTW:hosted', tasks) as t:
if t: ctw(['--ctwopts', '-Inline +ExitVMOnException', '-esa', '-G:+CompileTheWorldMultiThreaded', '-G:-InlineDuringParsing', '-G:-CompileTheWorldVerbose', '-XX:ReservedCodeCacheSize=300m'], _noneAsEmptyList(extraVMarguments))
# bootstrap tests
for b in bootstrap_tests:
b.run(tasks, extraVMarguments)
# run dacapo sanitychecks
for test in sanitycheck.getDacapos(level=sanitycheck.SanityCheckLevel.Gate, gateBuildLevel='release', extraVmArguments=extraVMarguments) \
+ sanitycheck.getScalaDacapos(level=sanitycheck.SanityCheckLevel.Gate, gateBuildLevel='release', extraVmArguments=extraVMarguments):
with Task(str(test) + ':' + 'release', tasks) as t:
if t and not test.test('jvmci'):
t.abort(test.name + ' Failed')
# ensure -Xbatch still works
with JVMCIMode('jit'):
with Task('DaCapo_pmd:BatchMode', tasks) as t:
if t: dacapo(_noneAsEmptyList(extraVMarguments) + ['-Xbatch', 'pmd'])
# ensure benchmark counters still work
with JVMCIMode('jit'):
with Task('DaCapo_pmd:BenchmarkCounters:product', tasks) as t:
if t: dacapo(_noneAsEmptyList(extraVMarguments) + ['-G:+LIRProfileMoves', '-G:+GenericDynamicCounters', '-XX:JVMCICounterSize=10', 'pmd'])
# ensure -Xcomp still works
with JVMCIMode('jit'):
with Task('XCompMode:product', tasks) as t:
if t: run_vm(_noneAsEmptyList(extraVMarguments) + ['-Xcomp', '-version'])
def bench(args):
"""run benchmarks and parse their output for results
Results are JSON formated : {group : {benchmark : score}}."""
resultFile = None
if '-resultfile' in args:
index = args.index('-resultfile')
if index + 1 < len(args):
resultFile = args[index + 1]
del args[index]
del args[index]
else:
mx.abort('-resultfile must be followed by a file name')
resultFileCSV = None
if '-resultfilecsv' in args:
index = args.index('-resultfilecsv')
if index + 1 < len(args):
resultFileCSV = args[index + 1]
del args[index]
del args[index]
else:
mx.abort('-resultfilecsv must be followed by a file name')
vm = mx_graal_core.get_vm()
if len(args) is 0:
args = ['all']
vmArgs = [arg for arg in args if arg.startswith('-')]
def benchmarks_in_group(group):
prefix = group + ':'
return [a[len(prefix):] for a in args if a.startswith(prefix)]
results = {}
benchmarks = []
# DaCapo
if 'dacapo' in args or 'all' in args:
benchmarks += sanitycheck.getDacapos(level=sanitycheck.SanityCheckLevel.Benchmark)
else:
dacapos = benchmarks_in_group('dacapo')
for dacapo in dacapos:
if dacapo not in sanitycheck.dacapoSanityWarmup.keys():
mx.abort('Unknown DaCapo : ' + dacapo)
iterations = sanitycheck.dacapoSanityWarmup[dacapo][sanitycheck.SanityCheckLevel.Benchmark]
if iterations > 0:
benchmarks += [sanitycheck.getDacapo(dacapo, ['-n', str(iterations)])]
if 'scaladacapo' in args or 'all' in args:
benchmarks += sanitycheck.getScalaDacapos(level=sanitycheck.SanityCheckLevel.Benchmark)
else:
scaladacapos = benchmarks_in_group('scaladacapo')
for scaladacapo in scaladacapos:
if scaladacapo not in sanitycheck.dacapoScalaSanityWarmup.keys():
mx.abort('Unknown Scala DaCapo : ' + scaladacapo)
iterations = sanitycheck.dacapoScalaSanityWarmup[scaladacapo][sanitycheck.SanityCheckLevel.Benchmark]
if iterations > 0:
benchmarks += [sanitycheck.getScalaDacapo(scaladacapo, ['-n', str(iterations)])]
# Bootstrap
if 'bootstrap' in args or 'all' in args:
benchmarks += sanitycheck.getBootstraps()
# SPECjvm2008
if 'specjvm2008' in args or 'all' in args:
benchmarks += [sanitycheck.getSPECjvm2008(['-ikv', '-wt', '120', '-it', '120'])]
else:
specjvms = benchmarks_in_group('specjvm2008')
for specjvm in specjvms:
benchmarks += [sanitycheck.getSPECjvm2008(['-ikv', '-wt', '120', '-it', '120', specjvm])]
if 'specjbb2005' in args or 'all' in args:
benchmarks += [sanitycheck.getSPECjbb2005()]
if 'specjbb2013' in args: # or 'all' in args //currently not in default set
benchmarks += [sanitycheck.getSPECjbb2013()]
if 'ctw-full' in args:
benchmarks.append(sanitycheck.getCTW(vm, sanitycheck.CTWMode.Full))
if 'ctw-noinline' in args:
benchmarks.append(sanitycheck.getCTW(vm, sanitycheck.CTWMode.NoInline))
for f in extraBenchmarks:
f(args, vm, benchmarks)
for test in benchmarks:
for (groupName, res) in test.bench(vm, extraVmOpts=vmArgs).items():
group = results.setdefault(groupName, {})
group.update(res)
mx.log(json.dumps(results))
if resultFile:
with open(resultFile, 'w') as f:
f.write(json.dumps(results))
if resultFileCSV:
with open(resultFileCSV, 'w') as f:
for key1, value1 in results.iteritems():
f.write('%s;\n' % (str(key1)))
for key2, value2 in sorted(value1.iteritems()):
f.write('%s; %s;\n' % (str(key2), str(value2)))
def bench(args):
"""run benchmarks and parse their output for results
Results are JSON formated : {group : {benchmark : score}}."""
resultFile = None
if '-resultfile' in args:
index = args.index('-resultfile')
if index + 1 < len(args):
resultFile = args[index + 1]
del args[index]
del args[index]
else:
mx.abort('-resultfile must be followed by a file name')
resultFileCSV = None
if '-resultfilecsv' in args:
index = args.index('-resultfilecsv')
if index + 1 < len(args):
resultFileCSV = args[index + 1]
del args[index]
del args[index]
else:
mx.abort('-resultfilecsv must be followed by a file name')
vm = mx_graal_core.get_vm()
if len(args) is 0:
args = ['all']
vmArgs = [arg for arg in args if arg.startswith('-')]
def benchmarks_in_group(group):
prefix = group + ':'
return [a[len(prefix):] for a in args if a.startswith(prefix)]
results = {}
benchmarks = []
# DaCapo
if 'dacapo' in args or 'all' in args:
benchmarks += sanitycheck.getDacapos(
level=sanitycheck.SanityCheckLevel.Benchmark)
else:
dacapos = benchmarks_in_group('dacapo')
for dacapo in dacapos:
if dacapo not in sanitycheck.dacapoSanityWarmup.keys():
mx.abort('Unknown DaCapo : ' + dacapo)
iterations = sanitycheck.dacapoSanityWarmup[dacapo][
sanitycheck.SanityCheckLevel.Benchmark]
if iterations > 0:
benchmarks += [
sanitycheck.getDacapo(dacapo, ['-n', str(iterations)])
]
if 'scaladacapo' in args or 'all' in args:
benchmarks += sanitycheck.getScalaDacapos(
level=sanitycheck.SanityCheckLevel.Benchmark)
else:
scaladacapos = benchmarks_in_group('scaladacapo')
for scaladacapo in scaladacapos:
if scaladacapo not in sanitycheck.dacapoScalaSanityWarmup.keys():
mx.abort('Unknown Scala DaCapo : ' + scaladacapo)
iterations = sanitycheck.dacapoScalaSanityWarmup[scaladacapo][
sanitycheck.SanityCheckLevel.Benchmark]
if iterations > 0:
benchmarks += [
sanitycheck.getScalaDacapo(scaladacapo,
['-n', str(iterations)])
]
# Bootstrap
if 'bootstrap' in args or 'all' in args:
benchmarks += sanitycheck.getBootstraps()
# SPECjvm2008
if 'specjvm2008' in args or 'all' in args:
benchmarks += [
sanitycheck.getSPECjvm2008(['-ikv', '-wt', '120', '-it', '120'])
]
else:
specjvms = benchmarks_in_group('specjvm2008')
for specjvm in specjvms:
benchmarks += [
sanitycheck.getSPECjvm2008(
['-ikv', '-wt', '120', '-it', '120', specjvm])
]
if 'specjbb2005' in args or 'all' in args:
benchmarks += [sanitycheck.getSPECjbb2005()]
if 'specjbb2013' in args: # or 'all' in args //currently not in default set
benchmarks += [sanitycheck.getSPECjbb2013()]
if 'ctw-full' in args:
benchmarks.append(sanitycheck.getCTW(vm, sanitycheck.CTWMode.Full))
if 'ctw-noinline' in args:
benchmarks.append(sanitycheck.getCTW(vm, sanitycheck.CTWMode.NoInline))
for f in extraBenchmarks:
f(args, vm, benchmarks)
for test in benchmarks:
for (groupName, res) in test.bench(vm, extraVmOpts=vmArgs).items():
group = results.setdefault(groupName, {})
group.update(res)
mx.log(json.dumps(results))
if resultFile:
with open(resultFile, 'w') as f:
f.write(json.dumps(results))
if resultFileCSV:
with open(resultFileCSV, 'w') as f:
for key1, value1 in results.iteritems():
f.write('%s;\n' % (str(key1)))
for key2, value2 in sorted(value1.iteritems()):
f.write('%s; %s;\n' % (str(key2), str(value2)))
def compiler_gate_runner(suites, unit_test_runs, bootstrap_tests, tasks, extraVMarguments=None):
# Run unit tests in hosted mode
with JVMCIMode("hosted"):
for r in unit_test_runs:
r.run(suites, tasks, extraVMarguments)
# Run microbench in hosted mode (only for testing the JMH setup)
with JVMCIMode("hosted"):
for r in [MicrobenchRun("Microbench", ["TestJMH"], tags=[GraalTags.fulltest])]:
r.run(tasks, extraVMarguments)
# Run ctw against rt.jar on server-hosted-jvmci
with JVMCIMode("hosted"):
with Task("CTW:hosted", tasks, tags=[GraalTags.fulltest]) as t:
if t:
ctw(
[
"--ctwopts",
"-Inline +ExitVMOnException",
"-esa",
"-G:+CompileTheWorldMultiThreaded",
"-G:-InlineDuringParsing",
"-G:-CompileTheWorldVerbose",
"-XX:ReservedCodeCacheSize=300m",
],
_noneAsEmptyList(extraVMarguments),
)
# bootstrap tests
for b in bootstrap_tests:
b.run(tasks, extraVMarguments)
# run dacapo sanitychecks
for test in sanitycheck.getDacapos(
level=sanitycheck.SanityCheckLevel.Gate, gateBuildLevel="release", extraVmArguments=extraVMarguments
) + sanitycheck.getScalaDacapos(
level=sanitycheck.SanityCheckLevel.Gate, gateBuildLevel="release", extraVmArguments=extraVMarguments
):
with Task(str(test) + ":" + "release", tasks, tags=[GraalTags.fulltest]) as t:
if t and not test.test("jvmci"):
t.abort(test.name + " Failed")
# ensure -Xbatch still works
with JVMCIMode("jit"):
with Task("DaCapo_pmd:BatchMode", tasks, tags=[GraalTags.fulltest]) as t:
if t:
dacapo(_noneAsEmptyList(extraVMarguments) + ["-Xbatch", "pmd"])
# ensure benchmark counters still work
with JVMCIMode("jit"):
with Task("DaCapo_pmd:BenchmarkCounters:product", tasks, tags=[GraalTags.fulltest]) as t:
if t:
dacapo(
_noneAsEmptyList(extraVMarguments)
+ ["-G:+LIRProfileMoves", "-G:+GenericDynamicCounters", "-XX:JVMCICounterSize=10", "pmd"]
)
# ensure -Xcomp still works
with JVMCIMode("jit"):
with Task("XCompMode:product", tasks, tags=[GraalTags.fulltest]) as t:
if t:
run_vm(_noneAsEmptyList(extraVMarguments) + ["-Xcomp", "-version"])