def c1visualizer(args):
"""run the Cl Compiler Visualizer"""
libpath = join(_suite.dir, 'lib')
if mx.get_os() == 'windows':
executable = join(libpath, 'c1visualizer', 'bin', 'c1visualizer.exe')
else:
executable = join(libpath, 'c1visualizer', 'bin', 'c1visualizer')
# Check whether the current C1Visualizer installation is the up-to-date
if exists(executable) and not exists(mx.library('C1VISUALIZER_DIST').get_path(resolve=False)):
mx.log('Updating C1Visualizer')
shutil.rmtree(join(libpath, 'c1visualizer'))
archive = mx.library('C1VISUALIZER_DIST').get_path(resolve=True)
if not exists(executable):
zf = zipfile.ZipFile(archive, 'r')
zf.extractall(libpath)
if not exists(executable):
mx.abort('C1Visualizer binary does not exist: ' + executable)
if mx.get_os() != 'windows':
# Make sure that execution is allowed. The zip file does not always specfiy that correctly
os.chmod(executable, 0777)
mx.run([executable])
def _run_netbeans_app(app_name, env=None, args=None):
args = [] if args is None else args
dist = app_name.upper() + '_DIST'
name = app_name.lower()
extractPath = join(_suite.get_output_root())
if mx.get_os() == 'windows':
executable = join(extractPath, name, 'bin', name + '.exe')
else:
executable = join(extractPath, name, 'bin', name)
# Check whether the current installation is up-to-date
if exists(executable) and not exists(mx.library(dist).get_path(resolve=False)):
mx.log('Updating ' + app_name)
shutil.rmtree(join(extractPath, name))
archive = mx.library(dist).get_path(resolve=True)
if not exists(executable):
zf = zipfile.ZipFile(archive, 'r')
zf.extractall(extractPath)
if not exists(executable):
mx.abort(app_name + ' binary does not exist: ' + executable)
if mx.get_os() != 'windows':
# Make sure that execution is allowed. The zip file does not always specfiy that correctly
os.chmod(executable, 0777)
mx.run([executable]+args, env=env)
def _parseVmArgs(jdk, args, addDefaultArgs=True):
args = mx.expand_project_in_args(args, insitu=False)
jacocoArgs = mx_gate.get_jacoco_agent_args()
if jacocoArgs:
args = jacocoArgs + args
# Support for -G: options
def translateGOption(arg):
if arg.startswith('-G:+'):
if '=' in arg:
mx.abort('Mixing + and = in -G: option specification: ' + arg)
arg = '-Dgraal.' + arg[len('-G:+'):] + '=true'
elif arg.startswith('-G:-'):
if '=' in arg:
mx.abort('Mixing - and = in -G: option specification: ' + arg)
arg = '-Dgraal.' + arg[len('-G:+'):] + '=false'
elif arg.startswith('-G:'):
if '=' not in arg:
mx.abort('Missing "=" in non-boolean -G: option specification: ' + arg)
arg = '-Dgraal.' + arg[len('-G:'):]
return arg
args = map(translateGOption, args)
if '-G:+PrintFlags' in args and '-Xcomp' not in args:
mx.warn('Using -G:+PrintFlags may have no effect without -Xcomp as Graal initialization is lazy')
bcp = []
if _jvmciModes[_vm.jvmciMode]:
if _add_jvmci_to_bootclasspath:
bcp.append(mx.library('JVMCI').classpath_repr())
bcp.extend([d.get_classpath_repr() for d in _bootClasspathDists])
if bcp:
args = ['-Xbootclasspath/p:' + os.pathsep.join(bcp)] + args
# Remove JVMCI from class path. It's only there to support compilation.
cpIndex, cp = mx.find_classpath_arg(args)
if cp:
jvmciLib = mx.library('JVMCI').path
cp = os.pathsep.join([e for e in cp.split(os.pathsep) if e != jvmciLib])
args[cpIndex] = cp
# Set the default JVMCI compiler
jvmciCompiler = _compilers[-1]
args = ['-Djvmci.compiler=' + jvmciCompiler] + args
if '-version' in args:
ignoredArgs = args[args.index('-version') + 1:]
if len(ignoredArgs) > 0:
mx.log("Warning: The following options will be ignored by the vm because they come after the '-version' argument: " + ' '.join(ignoredArgs))
return jdk.processArgs(args, addDefaultArgs=addDefaultArgs)
def igv(args):
"""run the Ideal Graph Visualizer"""
logFile = '.ideal_graph_visualizer.log'
with open(join(_suite.dir, logFile), 'w') as fp:
mx.logv('[Ideal Graph Visualizer log is in ' + fp.name + ']')
nbplatform = join(_suite.dir, 'src', 'share', 'tools', 'IdealGraphVisualizer', 'nbplatform')
# Remove NetBeans platform if it is earlier than the current supported version
if exists(nbplatform):
updateTrackingFile = join(nbplatform, 'platform', 'update_tracking', 'org-netbeans-core.xml')
if not exists(updateTrackingFile):
mx.log('Could not find \'' + updateTrackingFile + '\', removing NetBeans platform')
shutil.rmtree(nbplatform)
else:
dom = xml.dom.minidom.parse(updateTrackingFile)
currentVersion = mx.VersionSpec(dom.getElementsByTagName('module_version')[0].getAttribute('specification_version'))
supportedVersion = mx.VersionSpec('3.43.1')
if currentVersion < supportedVersion:
mx.log('Replacing NetBeans platform version ' + str(currentVersion) + ' with version ' + str(supportedVersion))
shutil.rmtree(nbplatform)
elif supportedVersion < currentVersion:
mx.log('Supported NetBeans version in igv command should be updated to ' + str(currentVersion))
if not exists(nbplatform):
mx.logv('[This execution may take a while as the NetBeans platform needs to be downloaded]')
env = _igvBuildEnv()
# make the jar for Batik 1.7 available.
env['IGV_BATIK_JAR'] = mx.library('BATIK').get_path(True)
if mx.run(['ant', '-f', mx._cygpathU2W(join(_suite.dir, 'src', 'share', 'tools', 'IdealGraphVisualizer', 'build.xml')), '-l', mx._cygpathU2W(fp.name), 'run'], env=env, nonZeroIsFatal=False):
mx.abort("IGV ant build & launch failed. Check '" + logFile + "'. You can also try to delete 'src/share/tools/IdealGraphVisualizer/nbplatform'.")
def igv(args):
"""run the Ideal Graph Visualizer"""
env = dict(os.environ)
# make the jar for Batik 1.7 available.
env['IGV_BATIK_JAR'] = mx.library('BATIK').get_path(True)
env['jdkhome'] = _igvJdk()
_run_netbeans_app('IdealGraphVisualizer', env, args)
def jacocoreport(args):
"""create a JaCoCo coverage report
Creates the report from the 'jacoco.exec' file in the current directory.
Default output directory is 'coverage', but an alternative can be provided as an argument."""
jacocoreport = mx.library("JACOCOREPORT", True)
out = 'coverage'
if len(args) == 1:
out = args[0]
elif len(args) > 1:
mx.abort('jacocoreport takes only one argument : an output directory')
includes = list(_jacoco_includes)
for p in mx.projects():
projsetting = getattr(p, 'jacoco', '')
if projsetting == 'include' or projsetting == '':
includes.append(p.name)
includedirs = set()
for p in mx.projects():
projsetting = getattr(p, 'jacoco', '')
if projsetting == 'exclude':
continue
for include in includes:
if include in p.dir:
includedirs.add(p.dir)
for i in includedirs:
bindir = i + '/bin'
mx.ensure_dir_exists(bindir)
mx.run_java(['-jar', jacocoreport.get_path(True), '--in', 'jacoco.exec', '--out', out] + sorted(includedirs))
def getDacapo(name, dacapoArgs=None, extraVmArguments=None):
dacapo = mx.get_env('DACAPO_CP')
if dacapo is None:
l = mx.library('DACAPO', False)
if l is not None:
dacapo = l.get_path(True)
else:
mx.abort('DaCapo 9.12 jar file must be specified with DACAPO_CP environment variable or as DACAPO library')
if not isfile(dacapo) or not dacapo.endswith('.jar'):
mx.abort('Specified DaCapo jar file does not exist or is not a jar file: ' + dacapo)
dacapoSuccess = re.compile(r"^===== DaCapo 9\.12 ([a-zA-Z0-9_]+) PASSED in ([0-9]+) msec =====", re.MULTILINE)
dacapoFail = re.compile(r"^===== DaCapo 9\.12 ([a-zA-Z0-9_]+) FAILED (warmup|) =====", re.MULTILINE)
dacapoTime = re.compile(r"===== DaCapo 9\.12 (?P<benchmark>[a-zA-Z0-9_]+) PASSED in (?P<time>[0-9]+) msec =====")
dacapoTime1 = re.compile(r"===== DaCapo 9\.12 (?P<benchmark>[a-zA-Z0-9_]+) completed warmup 1 in (?P<time>[0-9]+) msec =====")
dacapoMatcher = ValuesMatcher(dacapoTime, {'group' : 'DaCapo', 'name' : '<benchmark>', 'score' : '<time>'})
dacapoMatcher1 = ValuesMatcher(dacapoTime1, {'group' : 'DaCapo-1stRun', 'name' : '<benchmark>', 'score' : '<time>'})
# Use ipv4 stack for dacapos; tomcat+solaris+ipv6_interface fails (see also: JDK-8072384)
return Test("DaCapo-" + name, ['-jar', mx._cygpathU2W(dacapo), name] + _noneAsEmptyList(dacapoArgs), [dacapoSuccess], [dacapoFail],
[dacapoMatcher, dacapoMatcher1],
['-Xms2g', '-XX:+' + gc, '-XX:-UseCompressedOops', "-Djava.net.preferIPv4Stack=true", '-G:+ExitVMOnException'] +
_noneAsEmptyList(extraVmArguments))
def dacapoPath(self):
dacapo = mx.get_env("DACAPO_CP")
if dacapo:
return dacapo
lib = mx.library("DACAPO", False)
if lib:
return lib.get_path(True)
return None
def daCapoPath(self):
dacapo = mx.get_env(self.daCapoClasspathEnvVarName())
if dacapo:
return dacapo
lib = mx.library(self.daCapoLibraryName(), False)
if lib:
return lib.get_path(True)
return None
def pullTestSuite(library, destDir, **kwargs):
"""downloads and unpacks a test suite"""
mx.ensure_dir_exists(destDir)
localPath = mx.library(library).get_path(True)
mx_sulong.tar(localPath, destDir, **kwargs)
os.remove(localPath)
sha1Path = localPath + '.sha1'
if os.path.exists(sha1Path):
os.remove(sha1Path)
def jol(args):
"""Java Object Layout"""
joljar = mx.library('JOL_INTERNALS').get_path(resolve=True)
candidates = mx.findclass(args, logToConsole=False, matcher=lambda s, classname: s == classname or classname.endswith('.' + s) or classname.endswith('$' + s))
if len(candidates) > 0:
candidates = mx.select_items(sorted(candidates))
else:
# mx.findclass can be mistaken, don't give up yet
candidates = args
mx.run_java(['-javaagent:' + joljar, '-cp', os.pathsep.join([mx.classpath(jdk=mx.get_jdk()), joljar]), "org.openjdk.jol.MainObjectInternals"] + candidates)
def _sigtest_check(checktype, args, suite=None, projects=None):
"""run sigtest against Java projects with API"""
sigtestlib = mx.library('SIGTEST').get_path(resolve=True)
nonTestProjects = [p for p in mx.projects() if _should_test_project(p)]
if not nonTestProjects:
return 1
javaCompliance = max([p.javaCompliance for p in nonTestProjects])
class OutputCapture:
def __init__(self):
self.data = ""
def __call__(self, data):
self.data += data
failed = None
for p in nonTestProjects:
sigtestResults = p.dir + os.sep + 'snapshot.sigtest'
if not os.path.exists(sigtestResults):
continue
jdk = mx.get_jdk(javaCompliance)
cmd = ['-cp', mx._cygpathU2W(sigtestlib), 'com.sun.tdk.signaturetest.SignatureTest',
'-Static', '-Mode', 'bin', '-FileName', sigtestResults,
'-ClassPath', mx.classpath(p, jdk=jdk) + os.pathsep + jdk.bootclasspath(),
]
if checktype != 'all':
cmd.append('-b')
for pkg in mx._find_packages(p):
cmd = cmd + ['-PackageWithoutSubpackages', pkg]
out = OutputCapture()
print 'Checking ' + checktype + ' signature changes against ' + sigtestResults
exitcode = mx.run_java(cmd, nonZeroIsFatal=False, jdk=mx.get_jdk(javaCompliance), out=out, err=out)
mx.ensure_dir_exists(p.get_output_root())
with open(p.get_output_root() + os.path.sep + 'sigtest-junit.xml', 'w') as f:
f.write('<?xml version="1.0" encoding="UTF-8" ?>\n')
f.write('<testsuite tests="1" name="' + p.name + '.sigtest.' + checktype + '">\n')
f.write('<testcase classname="' + p.name + '" name="sigtest.' + checktype + '">\n')
if exitcode != 95:
print out.data
failed = sigtestResults
f.write('<failure type="SignatureCheck"><![CDATA[\n')
f.write(out.data)
f.write(']]></failure>')
else:
f.write('<system-err><![CDATA[\n')
f.write(out.data)
f.write(']]></system-err>')
f.write('</testcase>\n')
f.write('</testsuite>\n')
if failed:
mx.abort('Signature error in ' + failed)
else:
print 'OK.'
return 0
def _parseVmArgs(jdk, args, addDefaultArgs=True):
args = mx.expand_project_in_args(args, insitu=False)
jacocoArgs = mx_gate.get_jacoco_agent_args()
if jacocoArgs:
args = jacocoArgs + args
# Support for -G: options
def translateGOption(arg):
if arg.startswith("-G:+"):
if "=" in arg:
mx.abort("Mixing + and = in -G: option specification: " + arg)
arg = "-Dgraal." + arg[len("-G:+") :] + "=true"
elif arg.startswith("-G:-"):
if "=" in arg:
mx.abort("Mixing - and = in -G: option specification: " + arg)
arg = "-Dgraal." + arg[len("-G:+") :] + "=false"
elif arg.startswith("-G:"):
if "=" not in arg:
mx.abort('Missing "=" in non-boolean -G: option specification: ' + arg)
arg = "-Dgraal." + arg[len("-G:") :]
return arg
args = map(translateGOption, args)
if "-G:+PrintFlags" in args and "-Xcomp" not in args:
mx.warn("Using -G:+PrintFlags may have no effect without -Xcomp as Graal initialization is lazy")
bcp = [mx.distribution("truffle:TRUFFLE_API").classpath_repr()]
if _jvmciModes[_vm.jvmciMode]:
bcp.extend([d.get_classpath_repr() for d in _bootClasspathDists])
args = ["-Xbootclasspath/p:" + os.pathsep.join(bcp)] + args
# Remove JVMCI from class path. It's only there to support compilation.
cpIndex, cp = mx.find_classpath_arg(args)
if cp:
jvmciLib = mx.library("JVMCI").path
cp = os.pathsep.join([e for e in cp.split(os.pathsep) if e != jvmciLib])
args[cpIndex] = cp
# Set the default JVMCI compiler
jvmciCompiler = _compilers[-1]
args = ["-Djvmci.compiler=" + jvmciCompiler] + args
if "-version" in args:
ignoredArgs = args[args.index("-version") + 1 :]
if len(ignoredArgs) > 0:
mx.log(
"Warning: The following options will be ignored by the vm because they come after the '-version' argument: "
+ " ".join(ignoredArgs)
)
return jdk.processArgs(args, addDefaultArgs=addDefaultArgs)
def hcfdis(args):
"""disassemble HexCodeFiles embedded in text files
Run a tool over the input files to convert all embedded HexCodeFiles
to a disassembled format."""
parser = ArgumentParser(prog='mx hcfdis')
parser.add_argument('-m', '--map', help='address to symbol map applied to disassembler output')
parser.add_argument('files', nargs=REMAINDER, metavar='files...')
args = parser.parse_args(args)
path = mx.library('HCFDIS').get_path(resolve=True)
mx.run_java(['-cp', path, 'com.oracle.max.hcfdis.HexCodeFileDis'] + args.files)
if args.map is not None:
addressRE = re.compile(r'0[xX]([A-Fa-f0-9]+)')
with open(args.map) as fp:
lines = fp.read().splitlines()
symbols = dict()
for l in lines:
addressAndSymbol = l.split(' ', 1)
if len(addressAndSymbol) == 2:
address, symbol = addressAndSymbol
if address.startswith('0x'):
address = long(address, 16)
symbols[address] = symbol
for f in args.files:
with open(f) as fp:
lines = fp.read().splitlines()
updated = False
for i in range(0, len(lines)):
l = lines[i]
for m in addressRE.finditer(l):
sval = m.group(0)
val = long(sval, 16)
sym = symbols.get(val)
if sym:
l = l.replace(sval, sym)
updated = True
lines[i] = l
if updated:
mx.log('updating ' + f)
with open('new_' + f, "w") as fp:
for l in lines:
print >> fp, l
def get_jacoco_agent_args():
"""
Gets the args to be added to a VM command line for injecting the JaCoCo agent
if use of JaCoCo has been requested otherwise returns None.
"""
if _jacoco == "on" or _jacoco == "append":
jacocoagent = mx.library("JACOCOAGENT", True)
includes = list(_jacoco_includes)
baseExcludes = []
for p in mx.projects():
projsetting = getattr(p, "jacoco", "")
if projsetting == "exclude":
baseExcludes.append(p.name)
if projsetting == "include":
includes.append(p.name + ".*")
def _filter(l):
# filter out specific classes which are already covered by a baseExclude package
return [clazz for clazz in l if not any([clazz.startswith(package) for package in baseExcludes])]
excludes = []
for p in mx.projects():
if p.isJavaProject():
excludes += _filter(
p.find_classes_with_annotations(None, _jacoco_excluded_annotations, includeInnerClasses=True).keys()
)
excludes += _filter(
p.find_classes_with_matching_source_line(
None, lambda line: "JaCoCo Exclude" in line, includeInnerClasses=True
).keys()
)
excludes += [package + ".*" for package in baseExcludes]
agentOptions = {
"append": "true" if _jacoco == "append" else "false",
"bootclasspath": "true",
"includes": ":".join(includes),
"excludes": ":".join(excludes),
"destfile": "jacoco.exec",
}
return [
"-javaagent:" + jacocoagent.get_path(True) + "=" + ",".join([k + "=" + v for k, v in agentOptions.items()])
]
return None
def findbugs(args, fbArgs=None, suite=None, projects=None):
"""run FindBugs against non-test Java projects"""
findBugsHome = mx.get_env('FINDBUGS_HOME', None)
if suite is None:
suite = mx._primary_suite
if findBugsHome:
findbugsJar = join(findBugsHome, 'lib', 'findbugs.jar')
else:
findbugsLib = join(mx._mx_suite.get_output_root(), 'findbugs-3.0.0')
if not exists(findbugsLib):
tmp = tempfile.mkdtemp(prefix='findbugs-download-tmp', dir=mx._mx_suite.dir)
try:
findbugsDist = mx.library('FINDBUGS_DIST').get_path(resolve=True)
with zipfile.ZipFile(findbugsDist) as zf:
candidates = [e for e in zf.namelist() if e.endswith('/lib/findbugs.jar')]
assert len(candidates) == 1, candidates
libDirInZip = os.path.dirname(candidates[0])
zf.extractall(tmp)
shutil.copytree(join(tmp, libDirInZip), findbugsLib)
finally:
shutil.rmtree(tmp)
findbugsJar = join(findbugsLib, 'findbugs.jar')
assert exists(findbugsJar)
nonTestProjects = [p for p in mx.projects() if _should_test_project(p)]
if not nonTestProjects:
return 0
outputDirs = map(mx._cygpathU2W, [p.output_dir() for p in nonTestProjects])
javaCompliance = max([p.javaCompliance for p in nonTestProjects])
jdk = mx.get_jdk(javaCompliance)
if jdk.javaCompliance >= "1.9":
mx.log('FindBugs does not yet support JDK9 - skipping')
return 0
findbugsResults = join(suite.dir, 'findbugs.results')
if fbArgs is None:
fbArgs = defaultFindbugsArgs()
cmd = ['-jar', mx._cygpathU2W(findbugsJar)] + fbArgs
cmd = cmd + ['-auxclasspath', mx._separatedCygpathU2W(mx.classpath([p.name for p in nonTestProjects], jdk=jdk)), '-output', mx._cygpathU2W(findbugsResults), '-exitcode'] + args + outputDirs
exitcode = mx.run_java(cmd, nonZeroIsFatal=False, jdk=jdk)
if exitcode != 0:
with open(findbugsResults) as fp:
mx.log(fp.read())
os.unlink(findbugsResults)
return exitcode
def getScalaDacapo(name, dacapoArgs=[]):
dacapo = mx.get_env('DACAPO_SCALA_CP')
if dacapo is None:
l = mx.library('DACAPO_SCALA', False)
if l is not None:
dacapo = l.get_path(True)
else:
mx.abort('Scala DaCapo 0.1.0 jar file must be specified with DACAPO_SCALA_CP environment variable or as DACAPO_SCALA library')
if not isfile(dacapo) or not dacapo.endswith('.jar'):
mx.abort('Specified Scala DaCapo jar file does not exist or is not a jar file: ' + dacapo)
dacapoSuccess = re.compile(r"^===== DaCapo 0\.1\.0(-SNAPSHOT)? ([a-zA-Z0-9_]+) PASSED in ([0-9]+) msec =====$", re.MULTILINE)
dacapoFail = re.compile(r"^===== DaCapo 0\.1\.0(-SNAPSHOT)? ([a-zA-Z0-9_]+) FAILED (warmup|) =====$", re.MULTILINE)
dacapoTime = re.compile(r"===== DaCapo 0\.1\.0(-SNAPSHOT)? (?P<benchmark>[a-zA-Z0-9_]+) PASSED in (?P<time>[0-9]+) msec =====")
dacapoMatcher = ValuesMatcher(dacapoTime, {'group' : "Scala-DaCapo", 'name' : '<benchmark>', 'score' : '<time>'})
return Test("Scala-DaCapo-" + name, ['-jar', dacapo, name] + dacapoArgs, [dacapoSuccess], [dacapoFail], [dacapoMatcher], ['-Xms2g', '-XX:+'+gc, '-XX:-UseCompressedOops'])
def jackpot(args, suite=None, nonZeroIsFatal=False):
"""run Jackpot 3.0 against non-test Java projects"""
jackpotHome = mx.get_env('JACKPOT_HOME', None)
if jackpotHome:
jackpotJar = join(jackpotHome, 'jackpot.jar')
else:
jackpotJar = mx.library('JACKPOT').get_path(resolve=True)
assert exists(jackpotJar)
if suite is None:
suite = mx._primary_suite
nonTestProjects = [p for p in mx.projects() if _should_test_project(p)]
if not nonTestProjects:
return 0
groups = []
for p in nonTestProjects:
javacClasspath = []
deps = []
p.walk_deps(visit=lambda dep, edge: deps.append(dep) if dep.isLibrary() or dep.isJavaProject() else None)
annotationProcessorOnlyDeps = []
if len(p.annotation_processors()) > 0:
for apDep in p.annotation_processors():
if not apDep in deps:
deps.append(apDep)
annotationProcessorOnlyDeps.append(apDep)
for dep in deps:
if dep == p:
continue
if dep in annotationProcessorOnlyDeps:
continue
javacClasspath.append(dep.classpath_repr(resolve=True))
javaCompliance = p.javaCompliance
groups = groups + ['--group', "--classpath " + mx._separatedCygpathU2W(_escape_string(os.pathsep.join(javacClasspath))) + " --source " + str(p.javaCompliance) + " " + " ".join([_escape_string(d) for d in p.source_dirs()])]
cmd = ['-classpath', mx._cygpathU2W(jackpotJar), 'org.netbeans.modules.jackpot30.cmdline.Main']
cmd = cmd + ['--fail-on-warnings', '--progress'] + args + groups
return mx.run_java(cmd, nonZeroIsFatal=nonZeroIsFatal, jdk=mx.get_jdk(javaCompliance))
def getDacapo(name, dacapoArgs=None):
dacapo = mx.get_env('DACAPO_CP')
if dacapo is None:
l = mx.library('DACAPO', False)
if l is not None:
dacapo = l.get_path(True)
else:
mx.abort('DaCapo 9.12 jar file must be specified with DACAPO_CP environment variable or as DACAPO library')
if not isfile(dacapo) or not dacapo.endswith('.jar'):
mx.abort('Specified DaCapo jar file does not exist or is not a jar file: ' + dacapo)
dacapoSuccess = re.compile(r"^===== DaCapo 9\.12 ([a-zA-Z0-9_]+) PASSED in ([0-9]+) msec =====$", re.MULTILINE)
dacapoFail = re.compile(r"^===== DaCapo 9\.12 ([a-zA-Z0-9_]+) FAILED (warmup|) =====$", re.MULTILINE)
dacapoTime = re.compile(r"===== DaCapo 9\.12 (?P<benchmark>[a-zA-Z0-9_]+) PASSED in (?P<time>[0-9]+) msec =====")
dacapoTime1 = re.compile(r"===== DaCapo 9\.12 (?P<benchmark>[a-zA-Z0-9_]+) completed warmup 1 in (?P<time>[0-9]+) msec =====")
dacapoMatcher = ValuesMatcher(dacapoTime, {'group' : 'DaCapo', 'name' : '<benchmark>', 'score' : '<time>'})
dacapoMatcher1 = ValuesMatcher(dacapoTime1, {'group' : 'DaCapo-1stRun', 'name' : '<benchmark>', 'score' : '<time>'})
return Test("DaCapo-" + name, ['-jar', dacapo, name] + _noneAsEmptyList(dacapoArgs), [dacapoSuccess], [dacapoFail], [dacapoMatcher, dacapoMatcher1], ['-Xms2g', '-XX:+' + gc, '-XX:-UseCompressedOops'])
def _alldeps(args):
parser = ArgumentParser(prog='mx mxt-alldeps')
parser.add_argument('--kind', action='store', help='project, dist or library', default='project')
parser.add_argument('--name', action='store', help='name of entity', required=True)
parser.add_argument('--includeLibs', action='store_true', help='includeLibs')
parser.add_argument('--ignoreSelf', action='store_true', help='ignoreSelf')
args = parser.parse_args(args)
entity = None
if args.kind == 'project':
entity = mx.project(args.name)
elif args.kind == 'library':
entity = mx.library(args.name)
elif args.kind == 'dist':
entity = mx.distribution(args.name)
else:
mx.abort('illegal kind: ' + args.kind)
in_deps = []
deps = entity.all_deps(in_deps, includeLibs=args.includeLibs, includeSelf=not args.ignoreSelf)
print 'alldeps:'
for d in deps:
print d.__class__.__name__, ":", d.name
def _sigtest_generate(args, suite=None, projects=None):
"""run sigtest generator for Java projects with API"""
sigtestlib = mx.library('SIGTEST').get_path(resolve=True)
nonTestProjects = [p for p in mx.projects() if _should_test_project(p)]
if not nonTestProjects:
return 0
javaCompliance = max([p.javaCompliance for p in nonTestProjects])
for p in nonTestProjects:
sigtestResults = p.dir + os.sep + 'snapshot.sigtest'
cmd = ['-cp', mx._cygpathU2W(sigtestlib), 'com.sun.tdk.signaturetest.Setup',
'-Static', '-FileName', sigtestResults,
'-ClassPath', mx.classpath(p) + os.pathsep + mx.get_jdk(javaCompliance).bootclasspath(),
]
for pkg in mx.find_packages(p):
cmd = cmd + ['-PackageWithoutSubpackages', pkg]
exitcode = mx.run_java(cmd, nonZeroIsFatal=False, jdk=mx.get_jdk(javaCompliance))
if exitcode != 95:
mx.abort('Exit code was ' + str(exitcode) + ' while generating ' + sigtestResults)
if not exists(sigtestResults):
mx.abort('Cannot generate ' + sigtestResults)
mx.log("Sigtest snapshot generated to " + sigtestResults)
return 0
def get_jacoco_agent_args():
'''
Gets the args to be added to a VM command line for injecting the JaCoCo agent
if use of JaCoCo has been requested otherwise returns None.
'''
if _jacoco == 'on' or _jacoco == 'append':
jacocoagent = mx.library("JACOCOAGENT", True)
includes = list(_jacoco_includes)
baseExcludes = []
for p in mx.projects():
projsetting = getattr(p, 'jacoco', '')
if projsetting == 'exclude':
baseExcludes.append(p.name)
if projsetting == 'include':
includes.append(p.name + '.*')
def _filter(l):
# filter out specific classes which are already covered by a baseExclude package
return [clazz for clazz in l if not any([clazz.startswith(package) for package in baseExcludes])]
excludes = []
for p in mx.projects():
if p.isJavaProject():
excludes += _filter(p.find_classes_with_annotations(None, _jacoco_excluded_annotations, includeInnerClasses=True).keys())
excludes += _filter(p.find_classes_with_matching_source_line(None, lambda line: 'JaCoCo Exclude' in line, includeInnerClasses=True).keys())
excludes += [package + '.*' for package in baseExcludes]
agentOptions = {
'append' : 'true' if _jacoco == 'append' else 'false',
'bootclasspath' : 'true',
'includes' : ':'.join(includes),
'excludes' : ':'.join(excludes),
'destfile' : 'jacoco.exec'
}
return ['-javaagent:' + jacocoagent.get_path(True) + '=' + ','.join([k + '=' + v for k, v in agentOptions.items()])]
return None
def gate_body(args, tasks):
with Task('Vm: GraalVM dist names', tasks, tags=['names']) as t:
if t:
mx_sdk_vm.verify_graalvm_configs(suites=['vm', 'vm-enterprise'])
with Task('Vm: Basic GraalVM Tests', tasks,
tags=[VmGateTasks.compiler]) as t:
if t and mx_sdk_vm_impl.has_component('GraalVM compiler'):
# 1. the build must be a GraalVM
# 2. the build must be JVMCI-enabled since the 'GraalVM compiler' component is registered
mx_sdk_vm_impl.check_versions(
mx_sdk_vm_impl.graalvm_output(),
graalvm_version_regex=mx_sdk_vm_impl.graalvm_version_regex,
expect_graalvm=True,
check_jvmci=True)
if mx_sdk_vm_impl.has_component('LibGraal'):
libgraal_location = mx_sdk_vm_impl.get_native_image_locations(
'LibGraal', 'jvmcicompiler')
if libgraal_location is None:
mx.warn(
"Skipping libgraal tests: no library enabled in the LibGraal component"
)
else:
extra_vm_arguments = [
'-XX:+UseJVMCICompiler', '-XX:+UseJVMCINativeLibrary',
'-XX:JVMCILibPath=' + dirname(libgraal_location)
]
if args.extra_vm_argument:
extra_vm_arguments += args.extra_vm_argument
import mx_compiler
# run avrora on the GraalVM binary itself
with Task('LibGraal Compiler:GraalVM DaCapo-avrora',
tasks,
tags=[VmGateTasks.libgraal]) as t:
if t:
java_exe = join(mx_sdk_vm_impl.graalvm_home(), 'bin',
'java')
mx.run([
java_exe, '-XX:+UseJVMCICompiler',
'-XX:+UseJVMCINativeLibrary', '-jar',
mx.library('DACAPO').get_path(True), 'avrora', '-n',
'1'
])
# Ensure that fatal errors in libgraal route back to HotSpot
vmargs = [
'-XX:+UseJVMCICompiler', '-XX:+UseJVMCINativeLibrary',
'-XX:+PrintFlagsFinal',
'-Dlibgraal.CrashAt=length,hashCode',
'-Dlibgraal.CrashAtIsFatal=true'
]
cmd = ["dacapo:avrora", "--tracker=none", "--"
] + vmargs + ["--", "--preserve"]
out = mx.OutputCapture()
exitcode, bench_suite, _ = mx_benchmark.gate_mx_benchmark(
cmd, out=out, err=out, nonZeroIsFatal=False)
if exitcode == 0:
if 'CrashAtIsFatal: no fatalError function pointer installed' in out.data:
# Executing a VM that does not configure fatal errors handling
# in libgraal to route back through the VM.
pass
else:
mx.abort(
'Expected following benchmark to result in non-zero exit code: '
+ ' '.join(cmd))
else:
if len(bench_suite.scratchDirs()) == 0:
mx.abort(
"No scratch dir found despite error being expected!"
)
latest_scratch_dir = bench_suite.scratchDirs()[-1]
seen_libjvmci_log = False
hs_errs = glob.glob(
join(latest_scratch_dir, 'hs_err_pid*.log'))
if not hs_errs:
mx.abort(
'Expected a file starting with "hs_err_pid" in test directory. Entries found='
+ str(listdir(latest_scratch_dir)))
for hs_err in hs_errs:
mx.log("Verifying content of {}".format(
join(latest_scratch_dir, hs_err)))
with open(join(latest_scratch_dir, hs_err)) as fp:
contents = fp.read()
if 'libjvmci' in hs_err:
seen_libjvmci_log = True
if 'Fatal error: Forced crash' not in contents:
mx.abort(
'Expected "Fatal error: Forced crash" to be in contents of '
+ hs_err + ':' + linesep + contents)
else:
if 'Fatal error in JVMCI' not in contents:
mx.abort(
'Expected "Fatal error in JVMCI" to be in contents of '
+ hs_err + ':' + linesep + contents)
if 'JVMCINativeLibraryErrorFile' in out.data and not seen_libjvmci_log:
mx.abort(
'Expected a file matching "hs_err_pid*_libjvmci.log" in test directory. Entries found='
+ str(listdir(latest_scratch_dir)))
# Only clean up scratch dir on success
for scratch_dir in bench_suite.scratchDirs():
mx.log(
"Cleaning up scratch dir after gate task completion: {}"
.format(scratch_dir))
mx.rmtree(scratch_dir)
with Task('LibGraal Compiler:CTW',
tasks,
tags=[VmGateTasks.libgraal]) as t:
if t:
mx_compiler.ctw([
'-DCompileTheWorld.Config=Inline=false ' + ' '.join(
mx_compiler._compiler_error_options(prefix='')),
'-esa',
'-XX:+EnableJVMCI',
'-DCompileTheWorld.MultiThreaded=true',
'-Dgraal.InlineDuringParsing=false',
'-Dgraal.TrackNodeSourcePosition=true',
'-DCompileTheWorld.Verbose=false',
'-DCompileTheWorld.HugeMethodLimit=4000',
'-DCompileTheWorld.MaxCompiles=150000',
'-XX:ReservedCodeCacheSize=300m',
], extra_vm_arguments)
mx_compiler.compiler_gate_benchmark_runner(
tasks, extra_vm_arguments, prefix='LibGraal Compiler:')
with Task('LibGraal Truffle:unittest',
tasks,
tags=[VmGateTasks.libgraal]) as t:
if t:
def _unittest_config_participant(config):
vmArgs, mainClass, mainClassArgs = config
def is_truffle_fallback(arg):
fallback_args = [
"-Dtruffle.TruffleRuntime=com.oracle.truffle.api.impl.DefaultTruffleRuntime",
"-Dgraalvm.ForcePolyglotInvalid=true"
]
return arg in fallback_args
newVmArgs = [
arg for arg in vmArgs
if not is_truffle_fallback(arg)
]
return (newVmArgs, mainClass, mainClassArgs)
mx_unittest.add_config_participant(
_unittest_config_participant)
excluded_tests = environ.get("TEST_LIBGRAAL_EXCLUDE")
if excluded_tests:
with NamedTemporaryFile(prefix='blacklist.',
mode='w',
delete=False) as fp:
fp.file.writelines(
[l + '\n' for l in excluded_tests.split()])
unittest_args = ["--blacklist", fp.name]
else:
unittest_args = []
unittest_args = unittest_args + [
"--enable-timing", "--verbose"
]
compiler_log_file = "graal-compiler.log"
mx_unittest.unittest(unittest_args + extra_vm_arguments + [
"-Dpolyglot.engine.AllowExperimentalOptions=true",
"-Dpolyglot.engine.CompileImmediately=true",
"-Dpolyglot.engine.BackgroundCompilation=false",
"-Dpolyglot.engine.TraceCompilation=true",
"-Dpolyglot.log.file={0}".format(compiler_log_file),
"-Dgraalvm.locatorDisabled=true", "truffle"
])
if exists(compiler_log_file):
remove(compiler_log_file)
else:
mx.warn("Skipping libgraal tests: component not enabled")
gate_substratevm(tasks)
gate_sulong(tasks)
gate_python(tasks)
gate_svm_sl_tck(tasks)
gate_svm_truffle_tck_js(tasks)
def _sigtest_check(checktype, args, suite=None, projects=None):
"""run sigtest against Java projects with API"""
sigtestlib = mx.library('SIGTEST').get_path(resolve=True)
nonTestProjects = [p for p in mx.projects() if _should_test_project(p)]
if not nonTestProjects:
return 1
javaCompliance = max([p.javaCompliance for p in nonTestProjects])
class OutputCapture:
def __init__(self):
self.data = ""
def __call__(self, data):
self.data += data
failed = None
for p in nonTestProjects:
sigtestResults = p.dir + os.sep + 'snapshot.sigtest'
if not os.path.exists(sigtestResults):
continue
jdk = mx.get_jdk(javaCompliance)
cmd = [
'-cp',
mx._cygpathU2W(sigtestlib),
'com.sun.tdk.signaturetest.SignatureTest',
'-BootCP',
'-Static',
'-Mode',
'bin',
'-FileName',
sigtestResults,
'-ClassPath',
mx.classpath(p, jdk=jdk),
]
if args.human:
cmd.append('-H')
if checktype != 'all':
cmd.append('-b')
for pkg in mx._find_packages(p):
cmd = cmd + ['-PackageWithoutSubpackages', pkg]
out = OutputCapture()
print('Checking ' + checktype + ' signature changes against ' +
sigtestResults)
exitcode = mx.run_java(cmd,
nonZeroIsFatal=False,
jdk=mx.get_jdk(javaCompliance),
out=out,
err=out)
mx.ensure_dir_exists(p.get_output_root())
with open(p.get_output_root() + os.path.sep + 'sigtest-junit.xml',
'w') as f:
f.write('<?xml version="1.0" encoding="UTF-8" ?>\n')
f.write('<testsuite tests="1" name="' + p.name + '.sigtest.' +
checktype + '">\n')
f.write('<testcase classname="' + p.name + '" name="sigtest.' +
checktype + '">\n')
if exitcode != 95:
print(out.data)
failed = sigtestResults
f.write('<failure type="SignatureCheck"><![CDATA[\n')
f.write(out.data)
f.write(']]></failure>')
else:
f.write('<system-err><![CDATA[\n')
f.write(out.data)
f.write(']]></system-err>')
f.write('</testcase>\n')
f.write('</testsuite>\n')
if failed:
print(
'\nThe signature check detected changes in the API by comparing it with previous signature files.'
)
print(
'To fix this restore the original API or regenerate the signature files with:'
)
print('mx sigtest --generate')
mx.abort('Signature error in ' + failed)
else:
print('OK.')
return 0
def harness_path(self):
lib = mx.library(self.renaissance_harness_lib_name())
if lib:
return lib.get_path(True)
return None
def _get_lib_path(_lib_name):
return mx.library(_lib_name).get_path(resolve=True)
def get_jacoco_agent_path(resolve):
return mx.library('JACOCOAGENT_0.8.4', True).get_path(resolve)
def _gate_dacapo(name, iterations, extraVMarguments=None):
vmargs = ['-Xms2g', '-XX:+UseSerialGC', '-XX:-UseCompressedOops', '-Djava.net.preferIPv4Stack=true', '-Dgraal.ExitVMOnException=true'] + _remove_empty_entries(extraVMarguments)
dacapoJar = mx.library('DACAPO').get_path(True)
_gate_java_benchmark(vmargs + ['-jar', dacapoJar, name, '-n', str(iterations)], r'^===== DaCapo 9\.12 ([a-zA-Z0-9_]+) PASSED in ([0-9]+) msec =====')
def _gate_scala_dacapo(name, iterations, extraVMarguments=None):
vmargs = ['-Xms2g', '-XX:+UseSerialGC', '-XX:-UseCompressedOops', '-Dgraal.ExitVMOnException=true'] + _noneAsEmptyList(extraVMarguments)
scalaDacapoJar = mx.library('DACAPO_SCALA').get_path(True)
_gate_java_benchmark(vmargs + ['-jar', scalaDacapoJar, name, '-n', str(iterations)], r'^===== DaCapo 0\.1\.0(-SNAPSHOT)? ([a-zA-Z0-9_]+) PASSED in ([0-9]+) msec =====')
def build(self):
pre_ts = GraalNodeJsBuildTask._get_newest_ts(self.subject.getResults(),
fatalIfMissing=False)
build_env = os.environ.copy()
_setEnvVar(
'PATH', '%s%s%s' %
(join(_suite.mxDir, 'python2'), pathsep, build_env['PATH']),
build_env)
debug = ['--debug'] if self._debug_mode else []
shared_library = ['--enable-shared-library'] if hasattr(
self.args, 'sharedlibrary') and self.args.sharedlibrary else []
newest_config_file_ts = GraalNodeJsBuildTask._get_newest_ts(
_config_files, fatalIfMissing=True)
newest_generated_config_file_ts = GraalNodeJsBuildTask._get_newest_ts(
_generated_config_files, fatalIfMissing=False)
# Lazily generate config files only if `configure` and `configure.py` are older than the files they generate.
# If we don't do this, the `Makefile` always considers `config.gypi` out of date, triggering a second, unnecessary configure.
lazy_generator = ['--lazy-generator'
] if newest_generated_config_file_ts.isNewerThan(
newest_config_file_ts) else []
if _is_windows:
processDevkitRoot(env=build_env)
_setEnvVar(
'PATH',
pathsep.join([build_env['PATH']] + [
mx.library(lib_name).get_path(True)
for lib_name in ('NASM', 'NINJA')
]), build_env)
extra_flags = ['--ninja', '--dest-cpu=x64', '--without-etw']
else:
extra_flags = []
_mxrun(python_cmd() + [
join(_suite.dir,
'configure'), '--partly-static', '--without-dtrace',
'--without-inspector', '--without-node-snapshot',
'--without-node-code-cache', '--java-home',
_java_home(forBuild=True)
] + debug + shared_library + lazy_generator + extra_flags,
cwd=_suite.dir,
print_cmd=True,
env=build_env)
quiet_build = mx.is_continuous_integration(
) and not mx.get_opts().verbose
if _is_windows:
verbose = ['-v'] if mx.get_opts().verbose else []
# The custom env is not used to resolve the location of the executable
_mxrun([join(mx.library('NINJA').get_path(True), 'ninja.exe')] +
verbose +
['-j%d' % self.parallelism, '-C', self._build_dir],
print_cmd=True,
quiet_if_successful=quiet_build,
env=build_env)
else:
verbose = 'V={}'.format('1' if mx.get_opts().verbose else '')
_mxrun([mx.gmake_cmd(),
'-j%d' % self.parallelism, verbose],
cwd=_suite.dir,
print_cmd=True,
quiet_if_successful=quiet_build,
env=build_env)
# put headers for native modules into out/headers
_setEnvVar('HEADERS_ONLY', '1', build_env)
_mxrun(python_cmd() + [
join('tools', 'install.py'), 'install',
join('out', 'headers'), sep
],
quiet_if_successful=not mx.get_opts().verbose,
env=build_env)
post_ts = GraalNodeJsBuildTask._get_newest_ts(
self.subject.getResults(), fatalIfMissing=True)
mx.logv(
'Newest time-stamp before building: {}\nNewest time-stamp after building: {}\nHas built? {}'
.format(pre_ts, post_ts, post_ts.isNewerThan(pre_ts)))
built = post_ts.isNewerThan(pre_ts)
if built and _current_os == 'darwin':
nodePath = join(self._build_dir, 'node')
_mxrun([
'install_name_tool', '-add_rpath',
join(_java_home(), 'jre', 'lib'), '-add_rpath',
join(_java_home(), 'lib'), nodePath
],
print_cmd=True,
env=build_env)
return built
def renaissance_additional_lib(self, lib):
return mx.library(lib).get_path(True)
def additional_lib(self, lib):
return mx.library(lib).get_path(True)
def daCapoPath(self):
lib = mx.library(self.daCapoLibraryName(), False)
if lib:
return lib.get_path(True)
return None
def getLibraryDirectory(self):
return mx.library("WRK2", True).get_path(True)
def get_sigtest_jar(self):
"""
Returns the proper version of the SIGTEST jar used by `mx sigtest`.
"""
return mx.library('SIGTEST_1_2').get_path(resolve=True)
def _gate_scala_dacapo(name, iterations, extraVMarguments=None):
vmargs = ['-Xms2g', '-XX:+UseSerialGC', '-XX:-UseCompressedOops', '-Dgraal.ExitVMOnException=true'] + _remove_empty_entries(extraVMarguments)
scalaDacapoJar = mx.library('DACAPO_SCALA').get_path(True)
_gate_java_benchmark(vmargs + ['-jar', scalaDacapoJar, name, '-n', str(iterations)], r'^===== DaCapo 0\.1\.0(-SNAPSHOT)? ([a-zA-Z0-9_]+) PASSED in ([0-9]+) msec =====')
def bench(self, vm, cwd=None, extraVmOpts=None, vmbuild=None):
"""
Run this program as a benchmark.
Copied from sanitycheck.Test to extend benchmarking for non-JVMs.
"""
if vm in self.ignoredVMs:
return {}
if cwd is None:
cwd = self.defaultCwd
parser = OutputParser()
for successRE in self.successREs:
parser.addMatcher(ValuesMatcher(successRE, {'passed' : '1'}))
for failureRE in self.failureREs:
parser.addMatcher(ValuesMatcher(failureRE, {'failed' : '1'}))
for scoreMatcher in self.scoreMatchers:
parser.addMatcher(scoreMatcher)
if self.benchmarkCompilationRate:
if vm == 'graal':
bps = re.compile(r"ParsedBytecodesPerSecond@final: (?P<rate>[0-9]+)")
ibps = re.compile(r"InlinedBytecodesPerSecond@final: (?P<rate>[0-9]+)")
parser.addMatcher(ValuesMatcher(bps, {'group' : 'ParsedBytecodesPerSecond', 'name' : self.name, 'score' : '<rate>'}))
parser.addMatcher(ValuesMatcher(ibps, {'group' : 'InlinedBytecodesPerSecond', 'name' : self.name, 'score' : '<rate>'}))
else:
ibps = re.compile(r"(?P<compiler>[\w]+) compilation speed: +(?P<rate>[0-9]+) bytes/s {standard")
parser.addMatcher(ValuesMatcher(ibps, {'group' : 'InlinedBytecodesPerSecond', 'name' : '<compiler>:' + self.name, 'score' : '<rate>'}))
startDelim = 'START: ' + self.name
endDelim = 'END: ' + self.name
outputfile = os.environ.get('BENCH_OUTPUT', None)
if outputfile:
# Used only to debug output parsing
with open(outputfile) as fp:
output = fp.read()
start = output.find(startDelim)
end = output.find(endDelim, start)
if start == -1 and end == -1:
return {}
output = output[start + len(startDelim + os.linesep): end]
mx.log(startDelim)
mx.log(output)
mx.log(endDelim)
else:
tee = Tee()
mx.log(startDelim)
# zippy
result = -1
if vm == 'cpython2':
result = mx.run(['python'] + self.cmd[-2:], out=tee.eat)
elif vm == 'cpython':
result = mx.run(['python3'] + self.cmd[-2:], out=tee.eat)
elif vm == 'jython':
result = mx_graal.vm(['-jar', mx.library('JYTHON').path] + self.cmd[-2:], vm = 'original', out=tee.eat)
elif vm == 'pypy':
result = mx.run(['pypy'] + self.cmd[-2:], out=tee.eat)
elif vm == 'pypy3':
result = mx.run(['pypy3'] + self.cmd[-2:], out=tee.eat)
else:
result = mx_graal.vm(self.vmOpts + _noneAsEmptyList(extraVmOpts) + self.cmd, vm, nonZeroIsFatal=False, out=tee.eat, err=subprocess.STDOUT, cwd=cwd, vmbuild=vmbuild)
if result != 0:
mx.abort("Benchmark failed (non-zero retcode)")
# wait for subprocess to finish
time.sleep(.5)
mx.log(endDelim)
output = tee.output.getvalue()
groups = {}
passed = False
for valueMap in parser.parse(output):
assert (valueMap.has_key('name') and valueMap.has_key('score') and valueMap.has_key('group')) or valueMap.has_key('passed') or valueMap.has_key('failed'), valueMap
if valueMap.get('failed') == '1':
mx.abort("Benchmark failed")
if valueMap.get('passed') == '1':
passed = True
groupName = valueMap.get('group')
if groupName:
group = groups.setdefault(groupName, {})
name = valueMap.get('name')
score = valueMap.get('score')
if name and score:
group[name] = score
if not passed:
mx.abort("Benchmark failed (not passed)")
return groups
def _gate_dacapo(name, iterations, extraVMarguments=None):
vmargs = ['-Xms2g', '-XX:+UseSerialGC', '-XX:-UseCompressedOops', '-Djava.net.preferIPv4Stack=true', '-Dgraal.ExitVMOnException=true'] + _noneAsEmptyList(extraVMarguments)
dacapoJar = mx.library('DACAPO').get_path(True)
_gate_java_benchmark(vmargs + ['-jar', dacapoJar, name, '-n', str(iterations)], r'^===== DaCapo 9\.12 ([a-zA-Z0-9_]+) PASSED in ([0-9]+) msec =====')
def sonarqube_upload(args):
"""run SonarQube scanner and upload JaCoCo results"""
sonarqube_cli = mx.library("SONARSCANNER_CLI_3_3_0_1492", True)
parser = ArgumentParser(prog='mx sonarqube-upload')
parser.add_argument('--exclude-generated', action='store_true', help='Exclude generated source files')
parser.add_argument('--skip-coverage', action='store_true', default=False, help='Do not upload coverage reports')
args, sonar_args = mx.extract_VM_args(args, useDoubleDash=True, defaultAllVMArgs=True)
args, other_args = parser.parse_known_args(args)
java_props, other_args = _parse_java_properties(other_args)
def _check_required_prop(prop):
if prop not in java_props:
mx.abort("Required property '{prop}' not present. (Format is '-D{prop}=<value>')".format(prop=prop))
_check_required_prop('sonar.projectKey')
_check_required_prop('sonar.host.url')
basedir = mx.primary_suite().dir
# collect excluded projects
excludes, includes = _jacoco_excludes_includes_projects(limit_to_primary=True)
# collect excluded classes
exclude_classes = _jacoco_exclude_classes(includes)
java_bin = []
java_src = []
java_libs = []
def _visit_deps(dep, edge):
if dep.isJARDistribution() or dep.isLibrary():
java_libs.append(dep.classpath_repr())
mx.walk_deps(includes, visit=_visit_deps)
# collect all sources and binaries -- do exclusion later
for p in includes:
java_src.extend(p.source_dirs())
if not args.exclude_generated:
gen_dir = p.source_gen_dir()
if os.path.exists(gen_dir):
java_src.append(gen_dir)
java_bin.append(p.output_dir())
java_src = [os.path.relpath(s, basedir) for s in java_src]
java_bin = [os.path.relpath(b, basedir) for b in java_bin]
# Overlayed sources and classes must be excluded
jdk_compliance = mx.get_jdk().javaCompliance
overlayed_sources = []
overlayed_classfiles = {}
for p in includes:
if hasattr(p, "multiReleaseJarVersion") and jdk_compliance not in p.javaCompliance: # JDK9+ overlays
for srcDir in p.source_dirs():
for root, _, files in os.walk(srcDir):
for name in files:
if name.endswith('.java') and name != 'package-info.java':
overlayed_sources.append(join(os.path.relpath(root, basedir), name))
elif hasattr(p, "overlayTarget"): # JDK8 overlays
target = mx.project(p.overlayTarget)
overlay_sources = []
for srcDir in p.source_dirs():
for root, _, files in os.walk(srcDir):
for name in files:
if name.endswith('.java') and name != 'package-info.java':
overlay_sources.append(join(os.path.relpath(root, srcDir), name))
print(p, target, overlay_sources)
for srcDir in target.source_dirs():
for root, _, files in os.walk(srcDir):
for name in files:
if name.endswith('.java') and name != 'package-info.java':
s = join(os.path.relpath(root, srcDir), name)
if s in overlay_sources:
overlayed = join(os.path.relpath(root, basedir), name)
overlayed_sources.append(overlayed)
for s in overlay_sources:
classfile = join(os.path.relpath(target.output_dir(), basedir), s[:-len('java')] + 'class')
with open(classfile, 'rb') as fp:
overlayed_classfiles[classfile] = fp.read()
exclude_dirs = []
for p in excludes:
exclude_dirs.extend(p.source_dirs())
exclude_dirs.append(p.source_gen_dir())
javaCompliance = max([p.javaCompliance for p in includes]) if includes else mx.JavaCompliance('1.7')
jacoco_exec = JACOCO_EXEC
if not os.path.exists(jacoco_exec) and not args.skip_coverage:
mx.abort('No JaCoCo report file found: ' + jacoco_exec)
def _add_default_prop(key, value):
if key not in java_props:
java_props[key] = value
# default properties
_add_default_prop('sonar.java.source', str(javaCompliance))
_add_default_prop('sonar.projectBaseDir', basedir)
if not args.skip_coverage:
_add_default_prop('sonar.jacoco.reportPaths', jacoco_exec)
_add_default_prop('sonar.sources', ','.join(java_src))
_add_default_prop('sonar.java.binaries', ','.join(java_bin))
_add_default_prop('sonar.java.libraries', ','.join(java_libs))
exclude_patterns = [os.path.relpath(e, basedir) + '**' for e in exclude_dirs] + \
overlayed_sources + \
list(set([os.path.relpath(match[0], basedir) for _, match in exclude_classes.items()]))
if exclude_patterns:
_add_default_prop('sonar.exclusions', ','.join(exclude_patterns))
_add_default_prop('sonar.coverage.exclusions', ','.join(exclude_patterns))
_add_default_prop('sonar.verbose', 'true' if mx._opts.verbose else 'false')
with tempfile.NamedTemporaryFile(suffix="-sonarqube.properties", mode="w+") as fp:
# prepare properties file
fp.writelines(('{}={}\n'.format(k, v) for k, v in java_props.items()))
fp.flush()
# Since there's no options to exclude individual classes,
# we temporarily delete the overlayed class files instead.
for classfile in overlayed_classfiles:
os.remove(classfile)
try:
# run sonarqube cli
java_args = other_args + ['-Dproject.settings=' + fp.name, '-jar', sonarqube_cli.get_path(True)] + sonar_args
exit_code = mx.run_java(java_args, nonZeroIsFatal=False)
finally:
# Restore temporarily deleted class files
for classfile, data in overlayed_classfiles.items():
with open(classfile, 'wb') as cf:
cf.write(data)
if exit_code != 0:
fp.seek(0)
mx.abort('SonarQube scanner terminated with non-zero exit code: {}\n Properties file:\n{}'.format(
exit_code, ''.join((' ' + l for l in fp.readlines()))))
def findbugs(args, fbArgs=None, suite=None, projects=None):
"""run FindBugs against non-test Java projects"""
findBugsHome = mx.get_env('FINDBUGS_HOME', None)
if suite is None:
suite = mx._primary_suite
if findBugsHome:
findbugsJar = join(findBugsHome, 'lib', 'findbugs.jar')
else:
findbugsLib = join(mx._mx_suite.get_output_root(), 'findbugs-3.0.0')
if not exists(findbugsLib):
tmp = tempfile.mkdtemp(prefix='findbugs-download-tmp', dir=mx._mx_suite.dir)
try:
findbugsDist = mx.library('FINDBUGS_DIST').get_path(resolve=True)
with zipfile.ZipFile(findbugsDist) as zf:
candidates = [e for e in zf.namelist() if e.endswith('/lib/findbugs.jar')]
assert len(candidates) == 1, candidates
libDirInZip = os.path.dirname(candidates[0])
zf.extractall(tmp)
shutil.copytree(join(tmp, libDirInZip), findbugsLib)
finally:
shutil.rmtree(tmp)
findbugsJar = join(findbugsLib, 'findbugs.jar')
assert exists(findbugsJar)
nonTestProjects = [p for p in mx.projects() if _should_test_project(p)]
if not nonTestProjects:
return 0
ignoredClasses = set()
for p in nonTestProjects:
ignore = getattr(p, 'findbugsIgnoresGenerated', False)
if not isinstance(ignore, bool):
mx.abort('Value of attribute "findbugsIgnoresGenerated" must be True or False', context=p)
if ignore is True:
sourceDir = p.source_gen_dir()
for root, _, files in os.walk(sourceDir):
for name in files:
if name.endswith('.java') and '-info' not in name:
pkg = root[len(sourceDir) + 1:].replace(os.sep, '.')
cls = pkg + '.' + name[:-len('.java')]
ignoredClasses.add(cls)
with tempfile.NamedTemporaryFile(suffix='.xml', prefix='findbugs_exclude_filter.', mode='w', delete=False) as fp:
findbugsExcludeFilterFile = fp.name
xmlDoc = mx.XMLDoc()
xmlDoc.open('FindBugsFilter')
for cls in ignoredClasses:
xmlDoc.open('Match')
xmlDoc.element('Class', attributes={'name' : '~' + cls + '.*'})
xmlDoc.close('Match')
xmlDoc.close('FindBugsFilter')
xml = xmlDoc.xml(indent=' ', newl='\n')
print >> fp, xml
outputDirs = map(mx._cygpathU2W, [p.output_dir() for p in nonTestProjects])
javaCompliance = max([p.javaCompliance for p in nonTestProjects])
jdk = mx.get_jdk(javaCompliance)
if jdk.javaCompliance >= "1.9":
mx.log('FindBugs does not yet support JDK9 - skipping')
return 0
findbugsResults = join(suite.dir, 'findbugs.results')
if fbArgs is None:
fbArgs = defaultFindbugsArgs()
cmd = ['-jar', mx._cygpathU2W(findbugsJar)] + fbArgs
cmd = cmd + ['-exclude', findbugsExcludeFilterFile]
cmd = cmd + ['-auxclasspath', mx._separatedCygpathU2W(mx.classpath([p.name for p in nonTestProjects], jdk=jdk)), '-output', mx._cygpathU2W(findbugsResults), '-exitcode'] + args + outputDirs
try:
exitcode = mx.run_java(cmd, nonZeroIsFatal=False, jdk=jdk)
finally:
os.unlink(findbugsExcludeFilterFile)
if exitcode != 0:
with open(findbugsResults) as fp:
mx.log(fp.read())
os.unlink(findbugsResults)
return exitcode
def get_sigtest_jar(self):
return mx.library('SIGTEST_1_3').get_path(resolve=True)
