def jacocoreport(args):
"""create a JaCoCo coverage report
Creates the report from the 'jacoco.exec' file in the current directory.
Default output directory is 'coverage', but an alternative can be provided as an argument."""
jacocoreport = mx.library("JACOCOREPORT", True)
out = 'coverage'
if len(args) == 1:
out = args[0]
elif len(args) > 1:
mx.abort('jacocoreport takes only one argument : an output directory')
includes = list(_jacoco_includes)
for p in mx.projects():
projsetting = getattr(p, 'jacoco', '')
if projsetting == 'include' or projsetting == '':
includes.append(p.name)
includedirs = set()
for p in mx.projects():
projsetting = getattr(p, 'jacoco', '')
if projsetting == 'exclude':
continue
for include in includes:
if include in p.dir:
includedirs.add(p.dir)
for i in includedirs:
bindir = i + '/bin'
mx.ensure_dir_exists(bindir)
mx.run_java(['-jar', jacocoreport.get_path(True), '--in', 'jacoco.exec', '--out', out] + sorted(includedirs))
def jacocoreport(args):
"""create a JaCoCo coverage report
Creates the report from the 'jacoco.exec' file in the current directory.
Default output directory is 'coverage', but an alternative can be provided as an argument."""
jacocoreport = mx.library("JACOCOREPORT", True)
out = 'coverage'
if len(args) == 1:
out = args[0]
elif len(args) > 1:
mx.abort('jacocoreport takes only one argument : an output directory')
includes = list(_jacoco_includes)
for p in mx.projects():
projsetting = getattr(p, 'jacoco', '')
if projsetting == 'include' or projsetting == '':
includes.append(p.name)
includedirs = set()
for p in mx.projects():
projsetting = getattr(p, 'jacoco', '')
if projsetting == 'exclude':
continue
for include in includes:
if include in p.dir:
includedirs.add(p.dir)
for i in includedirs:
bindir = i + '/bin'
mx.ensure_dir_exists(bindir)
mx.run_java(['-jar', jacocoreport.get_path(True), '--in', 'jacoco.exec', '--out', out] + sorted(includedirs))
def _jacoco_excludes_includes():
includes = list(_jacoco_includes)
baseExcludes = []
for p in mx.projects():
if p.isJavaProject():
projsetting = getattr(p, 'jacoco', '')
if not _jacoco_is_package_whitelisted(p.name):
pass
elif projsetting == 'exclude':
baseExcludes.append(p.name)
elif projsetting == 'include':
includes.append(p.name + '.*')
if _jacoco_whitelisted_packages:
includes.extend((x + '.*' for x in _jacoco_whitelisted_packages))
def _filter(l):
# filter out specific classes which are already covered by a baseExclude package
return [clazz for clazz in l if not any([clazz.startswith(package) for package in baseExcludes])]
excludes = []
for p in mx.projects():
if p.isJavaProject() and p.name not in baseExcludes and _jacoco_is_package_whitelisted(p.name):
excludes += _filter(
p.find_classes_with_annotations(None, _jacoco_excluded_annotations, includeInnerClasses=True,
includeGenSrc=True).keys())
excludes += _filter(p.find_classes_with_matching_source_line(None, lambda line: 'JaCoCo Exclude' in line,
includeInnerClasses=True,
includeGenSrc=True).keys())
excludes += [package + '.*' for package in baseExcludes]
return excludes, includes
def get_jacoco_agent_args():
'''
Gets the args to be added to a VM command line for injecting the JaCoCo agent
if use of JaCoCo has been requested otherwise returns None.
'''
if _jacoco == 'on' or _jacoco == 'append':
jacocoagent = mx.library("JACOCOAGENT", True)
includes = list(_jacoco_includes)
baseExcludes = []
for p in mx.projects():
projsetting = getattr(p, 'jacoco', '')
if projsetting == 'exclude':
baseExcludes.append(p.name)
if projsetting == 'include':
includes.append(p.name + '.*')
def _filter(l):
# filter out specific classes which are already covered by a baseExclude package
return [
clazz for clazz in l if not any(
[clazz.startswith(package) for package in baseExcludes])
]
excludes = []
for p in mx.projects():
if p.isJavaProject():
excludes += _filter(
p.find_classes_with_annotations(
None,
_jacoco_excluded_annotations,
includeInnerClasses=True).keys())
excludes += _filter(
p.find_classes_with_matching_source_line(
None,
lambda line: 'JaCoCo Exclude' in line,
includeInnerClasses=True).keys())
excludes += [package + '.*' for package in baseExcludes]
agentOptions = {
'append': 'true' if _jacoco == 'append' else 'false',
'bootclasspath': 'true',
'includes': ':'.join(includes),
'excludes': ':'.join(excludes),
'destfile': 'jacoco.exec'
}
return [
'-javaagent:' + jacocoagent.get_path(True) + '=' +
','.join([k + '=' + v for k, v in agentOptions.items()])
]
return None
def checkheaders(args):
"""check Java source headers against any required pattern"""
failures = {}
for p in mx.projects():
if not p.isJavaProject():
continue
csConfig = join(mx.project(p.checkstyleProj).dir, '.checkstyle_checks.xml')
if not exists(csConfig):
mx.log('Cannot check headers for ' + p.name + ' - ' + csConfig + ' does not exist')
continue
dom = xml.dom.minidom.parse(csConfig)
for module in dom.getElementsByTagName('module'):
if module.getAttribute('name') == 'RegexpHeader':
for prop in module.getElementsByTagName('property'):
if prop.getAttribute('name') == 'header':
value = prop.getAttribute('value')
matcher = re.compile(value, re.MULTILINE)
for sourceDir in p.source_dirs():
for root, _, files in os.walk(sourceDir):
for name in files:
if name.endswith('.java') and name != 'package-info.java':
f = join(root, name)
with open(f) as fp:
content = fp.read()
if not matcher.match(content):
failures[f] = csConfig
for n, v in failures.iteritems():
mx.log('{0}: header does not match RegexpHeader defined in {1}'.format(n, v))
return len(failures)
def checkheaders(args):
"""check Java source headers against any required pattern"""
failures = {}
for p in mx.projects():
if not p.isJavaProject():
continue
csConfig = join(mx.project(p.checkstyleProj).dir, ".checkstyle_checks.xml")
if not exists(csConfig):
mx.log("Cannot check headers for " + p.name + " - " + csConfig + " does not exist")
continue
dom = xml.dom.minidom.parse(csConfig)
for module in dom.getElementsByTagName("module"):
if module.getAttribute("name") == "RegexpHeader":
for prop in module.getElementsByTagName("property"):
if prop.getAttribute("name") == "header":
value = prop.getAttribute("value")
matcher = re.compile(value, re.MULTILINE)
for sourceDir in p.source_dirs():
for root, _, files in os.walk(sourceDir):
for name in files:
if name.endswith(".java") and name != "package-info.java":
f = join(root, name)
with open(f) as fp:
content = fp.read()
if not matcher.match(content):
failures[f] = csConfig
for n, v in failures.iteritems():
mx.log("{0}: header does not match RegexpHeader defined in {1}".format(n, v))
return len(failures)
def jacocoreport(args):
"""create a JaCoCo coverage report
Creates the report from the 'jacoco.exec' file in the current directory.
Default output directory is 'coverage', but an alternative can be provided as an argument."""
dist_name = "MX_JACOCO_REPORT"
dist = mx.distribution(dist_name)
jdk = mx.get_jdk(dist.javaCompliance)
parser = ArgumentParser(prog='mx jacocoreport')
parser.add_argument('--format',
help='Export format (HTML or XML)',
default='html',
choices=['html', 'xml'])
parser.add_argument('output_directory',
help='Output directory',
default='coverage',
nargs='?')
args = parser.parse_args(args)
# list of strings of the form "project-dir:binary-dir"
includedirs = []
for p in mx.projects():
projsetting = getattr(p, 'jacoco', '')
if projsetting == 'include' or projsetting == '':
if isinstance(p, mx.ClasspathDependency):
includedirs.append(p.dir + ":" + p.classpath_repr(jdk))
mx.run_java([
'-cp',
mx.classpath([dist_name], jdk=jdk), '-jar', dist.path, '--in',
'jacoco.exec', '--out', args.output_directory, '--format', args.format
] + sorted(includedirs),
jdk=jdk)
def jacocoreport(args):
"""create a JaCoCo coverage report
Creates the report from the 'jacoco.exec' file in the current directory.
Default output directory is 'coverage', but an alternative can be provided as an argument."""
dist_name = "MX_JACOCO_REPORT"
dist = mx.distribution(dist_name)
jdk = mx.get_jdk(dist.javaCompliance)
out = 'coverage'
if len(args) == 1:
out = args[0]
elif len(args) > 1:
mx.abort('jacocoreport takes only one argument : an output directory')
# list of strings of the form "project-dir:binary-dir"
includedirs = []
for p in mx.projects():
projsetting = getattr(p, 'jacoco', '')
if projsetting == 'include' or projsetting == '':
if isinstance(p, mx.ClasspathDependency):
includedirs.append(p.dir + ":" + p.classpath_repr(jdk))
mx.run_java([
'-cp',
mx.classpath([dist_name], jdk=jdk), '-jar', dist.path, '--in',
'jacoco.exec', '--out', out
] + sorted(includedirs),
jdk=jdk)
def ideclean(args):
"""remove all IDE project configurations"""
def rm(path):
if exists(path):
os.remove(path)
for s in mx.suites() + [mx._mx_suite]:
rm(join(s.get_mx_output_dir(), 'eclipse-config.zip'))
rm(join(s.get_mx_output_dir(), 'netbeans-config.zip'))
shutil.rmtree(join(s.dir, '.idea'), ignore_errors=True)
for p in mx.projects() + mx._mx_suite.projects:
if not p.isJavaProject():
continue
shutil.rmtree(join(p.dir, '.settings'), ignore_errors=True)
shutil.rmtree(join(p.dir, '.externalToolBuilders'), ignore_errors=True)
shutil.rmtree(join(p.dir, 'nbproject'), ignore_errors=True)
rm(join(p.dir, '.classpath'))
rm(join(p.dir, '.checkstyle'))
rm(join(p.dir, '.project'))
rm(join(p.dir, '.factorypath'))
rm(join(p.dir, p.name + '.iml'))
rm(join(p.dir, 'build.xml'))
rm(join(p.dir, 'eclipse-build.xml'))
try:
rm(join(p.dir, p.name + '.jar'))
except:
mx.log_error("Error removing {0}".format(p.name + '.jar'))
for d in mx._dists.values():
if not d.isJARDistribution():
continue
if d.get_ide_project_dir():
shutil.rmtree(d.get_ide_project_dir(), ignore_errors=True)
def jackpot(args, suite=None, nonZeroIsFatal=False):
"""run Jackpot 3.0 against non-test Java projects"""
jackpotHome = mx.get_env('JACKPOT_HOME', None)
if jackpotHome:
jackpotJar = join(jackpotHome, 'jackpot.jar')
else:
jackpotJar = mx.library('JACKPOT').get_path(resolve=True)
assert exists(jackpotJar)
if suite is None:
suite = mx.primary_suite()
nonTestProjects = [p for p in mx.projects() if _should_test_project(p)]
if not nonTestProjects:
return 0
groups = []
for p in nonTestProjects:
javacClasspath = []
deps = []
p.walk_deps(visit=lambda dep, edge: deps.append(dep)
if dep.isLibrary() or dep.isJavaProject() else None)
annotationProcessorOnlyDeps = []
if len(p.annotation_processors()) > 0:
for apDep in p.annotation_processors():
if not apDep in deps:
deps.append(apDep)
annotationProcessorOnlyDeps.append(apDep)
for dep in deps:
if dep == p:
continue
if dep in annotationProcessorOnlyDeps:
continue
javacClasspath.append(dep.classpath_repr(resolve=True))
sourceLevel = min(p.javaCompliance.value, 9)
groups = groups + [
'--group', "--classpath " + mx._separatedCygpathU2W(
_escape_string(os.pathsep.join(javacClasspath))) +
" --source " + str(sourceLevel) + " " +
" ".join([_escape_string(d) for d in p.source_dirs()])
]
cmd = [
'-classpath',
mx._cygpathU2W(jackpotJar),
'org.netbeans.modules.jackpot30.cmdline.Main'
]
cmd = cmd + ['--fail-on-warnings', '--progress'] + args + groups
jdk = mx.get_jdk(mx.JavaCompliance("8"),
cancel='cannot run Jackpot',
purpose="run Jackpot")
if jdk is None:
mx.warn('Skipping Jackpot since JDK 8 is not available')
return 0
else:
return mx.run_java(cmd, nonZeroIsFatal=nonZeroIsFatal, jdk=jdk)
def _sigtest_generate(args, suite=None, projects=None):
"""run sigtest generator for Java projects with API"""
sigtestlib = mx.library('SIGTEST').get_path(resolve=True)
nonTestProjects = [p for p in mx.projects() if _should_test_project(p)]
if not nonTestProjects:
return 0
javaCompliance = max([p.javaCompliance for p in nonTestProjects])
for p in nonTestProjects:
sigtestResults = p.dir + os.sep + 'snapshot.sigtest'
jdk = mx.get_jdk(javaCompliance)
cmd = [
'-cp',
mx._cygpathU2W(sigtestlib),
'com.sun.tdk.signaturetest.Setup',
'-Static',
'-FileName',
sigtestResults,
'-ClassPath',
mx.classpath(p, jdk=jdk) + os.pathsep + jdk.bootclasspath(),
]
for pkg in mx._find_packages(p):
cmd = cmd + ['-PackageWithoutSubpackages', pkg]
exitcode = mx.run_java(cmd,
nonZeroIsFatal=False,
jdk=mx.get_jdk(javaCompliance))
if exitcode != 95:
mx.abort('Exit code was ' + str(exitcode) + ' while generating ' +
sigtestResults)
if not exists(sigtestResults):
mx.abort('Cannot generate ' + sigtestResults)
mx.log("Sigtest snapshot generated to " + sigtestResults)
return 0
def clangformat(args=None):
""" Runs clang-format on C/C++ files in native projects of the primary suite """
parser = ArgumentParser(prog='mx clangformat')
parser.add_argument(
'--with-projects',
action='store_true',
help=
'check native projects. Defaults to true unless a path is specified.')
parser.add_argument('paths',
metavar='path',
nargs='*',
help='check given paths')
args = parser.parse_args(args)
paths = [(p, "<cmd-line-argument>") for p in args.paths]
if not paths or args.with_projects:
paths += [(p.dir, p.name) for p in mx.projects(limit_to_primary=True)
if p.isNativeProject() and getattr(p, "clangFormat", True)]
error = False
for f, reason in paths:
if not checkCFiles(f, reason):
error = True
if error:
mx.log_error("found formatting errors!")
sys.exit(-1)
def checkheaders(args):
"""check Java source headers against any required pattern"""
failures = {}
for p in mx.projects():
if not p.isJavaProject():
continue
csConfig = join(mx.project(p.checkstyleProj).dir, '.checkstyle_checks.xml')
if not exists(csConfig):
mx.log('Cannot check headers for ' + p.name + ' - ' + csConfig + ' does not exist')
continue
dom = xml.dom.minidom.parse(csConfig)
for module in dom.getElementsByTagName('module'):
if module.getAttribute('name') == 'RegexpHeader':
for prop in module.getElementsByTagName('property'):
if prop.getAttribute('name') == 'header':
value = prop.getAttribute('value')
matcher = re.compile(value, re.MULTILINE)
for sourceDir in p.source_dirs():
for root, _, files in os.walk(sourceDir):
for name in files:
if name.endswith('.java') and name != 'package-info.java':
f = join(root, name)
with open(f) as fp:
content = fp.read()
if not matcher.match(content):
failures[f] = csConfig
for n, v in failures.iteritems():
mx.log('{0}: header does not match RegexpHeader defined in {1}'.format(n, v))
return len(failures)
def jacocoreport(args):
"""create a JaCoCo coverage report
Creates the report from the 'jacoco.exec' file in the current directory.
Default output directory is 'coverage', but an alternative can be provided as an argument."""
dist_name = "MX_JACOCO_REPORT"
mx.command_function("build")(['--dependencies', dist_name])
dist = mx.distribution(dist_name)
jdk = mx.get_jdk(dist.javaCompliance)
parser = ArgumentParser(prog='mx jacocoreport')
parser.add_argument('--format',
help='Export format (HTML or XML)',
default='html',
choices=['html', 'xml'])
parser.add_argument('--omit-excluded',
action='store_true',
help='omit excluded files from report')
parser.add_argument('output_directory',
help='Output directory',
default='coverage',
nargs='?')
args = parser.parse_args(args)
# list of strings of the form "project-dir:binary-dir"
includedirs = []
for p in mx.projects():
projsetting = getattr(p, 'jacoco', '')
if projsetting in ('include', '') and _jacoco_is_package_whitelisted(
p.name):
if isinstance(p, mx.ClasspathDependency):
if args.omit_excluded and p.is_test_project(
): # skip test projects when omit-excluded
continue
source_dirs = []
if p.isJavaProject():
source_dirs += p.source_dirs() + [p.source_gen_dir()]
includedirs.append(
":".join([p.dir, p.classpath_repr(jdk)] + source_dirs))
def _run_reporter(extra_args=None):
mx.run_java([
'-cp',
mx.classpath([dist_name],
jdk=jdk), '-jar', dist.path, '--in', JACOCO_EXEC,
'--out', args.output_directory, '--format', args.format
] + (extra_args or []) + sorted(includedirs),
jdk=jdk,
addDefaultArgs=False)
if not args.omit_excluded:
_run_reporter()
else:
with tempfile.NamedTemporaryFile(suffix="jacoco-report-exclude",
mode="w") as fp:
excludes, _ = _jacoco_excludes_includes()
fp.writelines((e + "\n" for e in excludes))
fp.flush()
_run_reporter(['--exclude-file', fp.name])
def _find_version_base_project(versioned_project):
extended_packages = versioned_project.extended_java_packages()
if not extended_packages:
mx.abort(
'Project with a multiReleaseJarVersion attribute must have sources in a package defined by project without multiReleaseJarVersion attribute',
context=versioned_project)
base_project = None
base_package = None
for extended_package in extended_packages:
for p in mx.projects():
if versioned_project != p and p.isJavaProject() and not hasattr(
p, 'multiReleaseJarVersion'):
if extended_package in p.defined_java_packages():
if base_project is None:
base_project = p
base_package = extended_package
else:
if base_project != p:
mx.abort(
'Multi-release jar versioned project {} must extend packages from exactly one project but extends {} from {} and {} from {}'
.format(versioned_project, extended_package, p,
base_project, base_package))
if not base_project:
mx.abort(
'Multi-release jar versioned project {} must extend package(s) from another project'
.format(versioned_project))
return base_project
def jackpot(args, suite=None, nonZeroIsFatal=False):
"""run Jackpot 11.1 against non-test Java projects"""
jackpotHome = mx.get_env('JACKPOT_HOME', None)
if jackpotHome:
jackpotJar = join(jackpotHome, 'jackpot.jar')
else:
jackpotJar = mx.library('JACKPOT').get_path(resolve=True)
assert exists(jackpotJar)
if suite is None:
suite = mx.primary_suite()
nonTestProjects = [p for p in mx.projects() if _should_test_project(p)]
if not nonTestProjects:
return 0
groups = []
for p in nonTestProjects:
javacClasspath = []
deps = []
p.walk_deps(visit=lambda dep, edge: deps.append(dep) if dep.isLibrary() or dep.isJavaProject() else None)
annotationProcessorOnlyDeps = []
if len(p.annotation_processors()) > 0:
for apDep in p.annotation_processors():
if not apDep in deps:
deps.append(apDep)
annotationProcessorOnlyDeps.append(apDep)
for dep in deps:
if dep == p:
continue
if dep in annotationProcessorOnlyDeps:
continue
javacClasspath.append(dep.classpath_repr(resolve=True))
sourceLevel = min(p.javaCompliance.value, 9)
groups = groups + ['--group', "--classpath " + mx._separatedCygpathU2W(_escape_string(os.pathsep.join(javacClasspath))) + " --source " + str(sourceLevel) + " " + " ".join([_escape_string(d) for d in p.source_dirs()])]
cmd = ['--add-exports=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED', '--add-opens=java.base/java.net=ALL-UNNAMED', '--add-opens=java.desktop/sun.awt=ALL-UNNAMED']
cmd = cmd + ['-classpath', mx._cygpathU2W(jackpotJar), 'org.netbeans.modules.jackpot30.cmdline.Main']
jackCmd = ['--fail-on-warnings', '--progress'] + args + groups
jdk = mx.get_jdk(mx.JavaCompliance("11+"), cancel='cannot run Jackpot', purpose="run Jackpot")
if jdk is None:
mx.warn('Skipping Jackpot since JDK 11+ is not available')
return 0
else:
with tempfile.NamedTemporaryFile(mode='w', suffix='.jackpot') as f:
for c in jackCmd:
print(c, file=f)
f.flush()
ret = mx.run_java(cmd + ['@' + f.name], nonZeroIsFatal=nonZeroIsFatal, jdk=jdk)
if ret != 0:
mx.warn('To simulate the failure execute `mx -p {0} jackpot`.'.format(suite.dir))
mx.warn('To fix the error automatically try `mx -p {0} jackpot --apply`'.format(suite.dir))
return ret
def get_jacoco_agent_args():
"""
Gets the args to be added to a VM command line for injecting the JaCoCo agent
if use of JaCoCo has been requested otherwise returns None.
"""
if _jacoco == "on" or _jacoco == "append":
jacocoagent = mx.library("JACOCOAGENT", True)
includes = list(_jacoco_includes)
baseExcludes = []
for p in mx.projects():
projsetting = getattr(p, "jacoco", "")
if projsetting == "exclude":
baseExcludes.append(p.name)
if projsetting == "include":
includes.append(p.name + ".*")
def _filter(l):
# filter out specific classes which are already covered by a baseExclude package
return [clazz for clazz in l if not any([clazz.startswith(package) for package in baseExcludes])]
excludes = []
for p in mx.projects():
if p.isJavaProject():
excludes += _filter(
p.find_classes_with_annotations(None, _jacoco_excluded_annotations, includeInnerClasses=True).keys()
)
excludes += _filter(
p.find_classes_with_matching_source_line(
None, lambda line: "JaCoCo Exclude" in line, includeInnerClasses=True
).keys()
)
excludes += [package + ".*" for package in baseExcludes]
agentOptions = {
"append": "true" if _jacoco == "append" else "false",
"bootclasspath": "true",
"includes": ":".join(includes),
"excludes": ":".join(excludes),
"destfile": "jacoco.exec",
}
return [
"-javaagent:" + jacocoagent.get_path(True) + "=" + ",".join([k + "=" + v for k, v in agentOptions.items()])
]
return None
def findbugs(args, fbArgs=None, suite=None, projects=None):
"""run FindBugs against non-test Java projects"""
findBugsHome = mx.get_env('FINDBUGS_HOME', None)
if suite is None:
suite = mx._primary_suite
if findBugsHome:
findbugsJar = join(findBugsHome, 'lib', 'findbugs.jar')
else:
findbugsLib = join(mx._mx_suite.get_output_root(), 'findbugs-3.0.0')
if not exists(findbugsLib):
tmp = tempfile.mkdtemp(prefix='findbugs-download-tmp',
dir=mx._mx_suite.dir)
try:
findbugsDist = mx.library('FINDBUGS_DIST').get_path(
resolve=True)
with zipfile.ZipFile(findbugsDist) as zf:
candidates = [
e for e in zf.namelist()
if e.endswith('/lib/findbugs.jar')
]
assert len(candidates) == 1, candidates
libDirInZip = os.path.dirname(candidates[0])
zf.extractall(tmp)
shutil.copytree(join(tmp, libDirInZip), findbugsLib)
finally:
shutil.rmtree(tmp)
findbugsJar = join(findbugsLib, 'findbugs.jar')
assert exists(findbugsJar)
nonTestProjects = [p for p in mx.projects() if _should_test_project(p)]
if not nonTestProjects:
return 0
outputDirs = map(mx._cygpathU2W, [p.output_dir() for p in nonTestProjects])
javaCompliance = max([p.javaCompliance for p in nonTestProjects])
jdk = mx.get_jdk(javaCompliance)
if jdk.javaCompliance >= "1.9":
mx.log('FindBugs does not yet support JDK9 - skipping')
return 0
findbugsResults = join(suite.dir, 'findbugs.results')
if fbArgs is None:
fbArgs = defaultFindbugsArgs()
cmd = ['-jar', mx._cygpathU2W(findbugsJar)] + fbArgs
cmd = cmd + [
'-auxclasspath',
mx._separatedCygpathU2W(
mx.classpath([p.name
for p in nonTestProjects], jdk=jdk)), '-output',
mx._cygpathU2W(findbugsResults), '-exitcode'
] + args + outputDirs
exitcode = mx.run_java(cmd, nonZeroIsFatal=False, jdk=jdk)
if exitcode != 0:
with open(findbugsResults) as fp:
mx.log(fp.read())
os.unlink(findbugsResults)
return exitcode
def _sigtest_check(checktype, args, suite=None, projects=None):
"""run sigtest against Java projects with API"""
sigtestlib = mx.library('SIGTEST').get_path(resolve=True)
nonTestProjects = [p for p in mx.projects() if _should_test_project(p)]
if not nonTestProjects:
return 1
javaCompliance = max([p.javaCompliance for p in nonTestProjects])
class OutputCapture:
def __init__(self):
self.data = ""
def __call__(self, data):
self.data += data
failed = None
for p in nonTestProjects:
sigtestResults = p.dir + os.sep + 'snapshot.sigtest'
if not os.path.exists(sigtestResults):
continue
jdk = mx.get_jdk(javaCompliance)
cmd = ['-cp', mx._cygpathU2W(sigtestlib), 'com.sun.tdk.signaturetest.SignatureTest',
'-Static', '-Mode', 'bin', '-FileName', sigtestResults,
'-ClassPath', mx.classpath(p, jdk=jdk) + os.pathsep + jdk.bootclasspath(),
]
if checktype != 'all':
cmd.append('-b')
for pkg in mx._find_packages(p):
cmd = cmd + ['-PackageWithoutSubpackages', pkg]
out = OutputCapture()
print 'Checking ' + checktype + ' signature changes against ' + sigtestResults
exitcode = mx.run_java(cmd, nonZeroIsFatal=False, jdk=mx.get_jdk(javaCompliance), out=out, err=out)
mx.ensure_dir_exists(p.get_output_root())
with open(p.get_output_root() + os.path.sep + 'sigtest-junit.xml', 'w') as f:
f.write('<?xml version="1.0" encoding="UTF-8" ?>\n')
f.write('<testsuite tests="1" name="' + p.name + '.sigtest.' + checktype + '">\n')
f.write('<testcase classname="' + p.name + '" name="sigtest.' + checktype + '">\n')
if exitcode != 95:
print out.data
failed = sigtestResults
f.write('<failure type="SignatureCheck"><![CDATA[\n')
f.write(out.data)
f.write(']]></failure>')
else:
f.write('<system-err><![CDATA[\n')
f.write(out.data)
f.write(']]></system-err>')
f.write('</testcase>\n')
f.write('</testsuite>\n')
if failed:
mx.abort('Signature error in ' + failed)
else:
print 'OK.'
return 0
def get_jacoco_agent_args():
'''
Gets the args to be added to a VM command line for injecting the JaCoCo agent
if use of JaCoCo has been requested otherwise returns None.
'''
if _jacoco == 'on' or _jacoco == 'append':
jacocoagent = mx.library("JACOCOAGENT", True)
includes = list(_jacoco_includes)
baseExcludes = []
for p in mx.projects():
projsetting = getattr(p, 'jacoco', '')
if projsetting == 'exclude':
baseExcludes.append(p.name)
if projsetting == 'include':
includes.append(p.name + '.*')
def _filter(l):
# filter out specific classes which are already covered by a baseExclude package
return [clazz for clazz in l if not any([clazz.startswith(package) for package in baseExcludes])]
excludes = []
for p in mx.projects():
if p.isJavaProject():
excludes += _filter(p.find_classes_with_annotations(None, _jacoco_excluded_annotations, includeInnerClasses=True).keys())
excludes += _filter(p.find_classes_with_matching_source_line(None, lambda line: 'JaCoCo Exclude' in line, includeInnerClasses=True).keys())
excludes += [package + '.*' for package in baseExcludes]
agentOptions = {
'append' : 'true' if _jacoco == 'append' else 'false',
'bootclasspath' : 'true',
'includes' : ':'.join(includes),
'excludes' : ':'.join(excludes),
'destfile' : 'jacoco.exec'
}
return ['-javaagent:' + jacocoagent.get_path(True) + '=' + ','.join([k + '=' + v for k, v in agentOptions.items()])]
return None
def _jacoco_excludes_includes_projects(limit_to_primary=False):
includes = []
excludes = []
projects = mx.projects(limit_to_primary=limit_to_primary)
for p in projects:
if p.isJavaProject():
projsetting = getattr(p, 'jacoco', '')
if not _jacoco_is_package_whitelisted(p.name):
excludes.append(p)
elif projsetting == 'exclude':
excludes.append(p)
else:
includes.append(p)
return excludes, includes
def javadoc(args):
"""build the Javadoc for all packages"""
if not args:
projectNames = []
for p in mx.projects(True, True):
projectNames.append(p.name)
mx.javadoc(['--unified', '--projects', ','.join(projectNames)], includeDeps=False)
else:
mx.javadoc(['--unified'] + args)
javadocDir = os.sep.join([_suite.dir, 'javadoc'])
index = os.sep.join([javadocDir, 'index.html'])
if exists(index):
indexContent = open(index, 'r').read()
new_file = open(index, "w")
new_file.write(indexContent)
checkLinks(javadocDir)
def javadoc(args):
"""build the Javadoc for all packages"""
if not args:
projectNames = []
for p in mx.projects(True, True):
projectNames.append(p.name)
mx.javadoc(['--unified', '--projects', ','.join(projectNames)], includeDeps=False)
else:
mx.javadoc(['--unified'] + args)
javadocDir = os.sep.join([_suite.dir, 'javadoc'])
index = os.sep.join([javadocDir, 'index.html'])
if exists(index):
indexContent = open(index, 'r').read()
new_file = open(index, "w")
new_file.write(indexContent)
checkLinks(javadocDir)
def findbugs(args, fbArgs=None, suite=None, projects=None):
"""run FindBugs against non-test Java projects"""
findBugsHome = mx.get_env('FINDBUGS_HOME', None)
if suite is None:
suite = mx._primary_suite
if findBugsHome:
findbugsJar = join(findBugsHome, 'lib', 'findbugs.jar')
else:
findbugsLib = join(mx._mx_suite.get_output_root(), 'findbugs-3.0.0')
if not exists(findbugsLib):
tmp = tempfile.mkdtemp(prefix='findbugs-download-tmp', dir=mx._mx_suite.dir)
try:
findbugsDist = mx.library('FINDBUGS_DIST').get_path(resolve=True)
with zipfile.ZipFile(findbugsDist) as zf:
candidates = [e for e in zf.namelist() if e.endswith('/lib/findbugs.jar')]
assert len(candidates) == 1, candidates
libDirInZip = os.path.dirname(candidates[0])
zf.extractall(tmp)
shutil.copytree(join(tmp, libDirInZip), findbugsLib)
finally:
shutil.rmtree(tmp)
findbugsJar = join(findbugsLib, 'findbugs.jar')
assert exists(findbugsJar)
nonTestProjects = [p for p in mx.projects() if _should_test_project(p)]
if not nonTestProjects:
return 0
outputDirs = map(mx._cygpathU2W, [p.output_dir() for p in nonTestProjects])
javaCompliance = max([p.javaCompliance for p in nonTestProjects])
jdk = mx.get_jdk(javaCompliance)
if jdk.javaCompliance >= "1.9":
mx.log('FindBugs does not yet support JDK9 - skipping')
return 0
findbugsResults = join(suite.dir, 'findbugs.results')
if fbArgs is None:
fbArgs = defaultFindbugsArgs()
cmd = ['-jar', mx._cygpathU2W(findbugsJar)] + fbArgs
cmd = cmd + ['-auxclasspath', mx._separatedCygpathU2W(mx.classpath([p.name for p in nonTestProjects], jdk=jdk)), '-output', mx._cygpathU2W(findbugsResults), '-exitcode'] + args + outputDirs
exitcode = mx.run_java(cmd, nonZeroIsFatal=False, jdk=jdk)
if exitcode != 0:
with open(findbugsResults) as fp:
mx.log(fp.read())
os.unlink(findbugsResults)
return exitcode
def jackpot(args, suite=None, nonZeroIsFatal=False):
"""run Jackpot 3.0 against non-test Java projects"""
jackpotHome = mx.get_env('JACKPOT_HOME', None)
if jackpotHome:
jackpotJar = join(jackpotHome, 'jackpot.jar')
else:
jackpotJar = mx.library('JACKPOT').get_path(resolve=True)
assert exists(jackpotJar)
if suite is None:
suite = mx._primary_suite
nonTestProjects = [p for p in mx.projects() if _should_test_project(p)]
if not nonTestProjects:
return 0
groups = []
for p in nonTestProjects:
javacClasspath = []
deps = []
p.walk_deps(visit=lambda dep, edge: deps.append(dep) if dep.isLibrary() or dep.isJavaProject() else None)
annotationProcessorOnlyDeps = []
if len(p.annotation_processors()) > 0:
for apDep in p.annotation_processors():
if not apDep in deps:
deps.append(apDep)
annotationProcessorOnlyDeps.append(apDep)
for dep in deps:
if dep == p:
continue
if dep in annotationProcessorOnlyDeps:
continue
javacClasspath.append(dep.classpath_repr(resolve=True))
javaCompliance = p.javaCompliance
groups = groups + ['--group', "--classpath " + mx._separatedCygpathU2W(_escape_string(os.pathsep.join(javacClasspath))) + " --source " + str(p.javaCompliance) + " " + " ".join([_escape_string(d) for d in p.source_dirs()])]
cmd = ['-classpath', mx._cygpathU2W(jackpotJar), 'org.netbeans.modules.jackpot30.cmdline.Main']
cmd = cmd + ['--fail-on-warnings', '--progress'] + args + groups
return mx.run_java(cmd, nonZeroIsFatal=nonZeroIsFatal, jdk=mx.get_jdk(javaCompliance))
def find_test_candidates(annotations, suite, jdk, buildCacheDir='unittest'):
"""
Finds all classes containing methods annotated with one of the supplied annotations.
To speed up subsequent invocations, the results are cached in the `buildCacheDir`.
:param list annotations: a list of annotations to recognize test methods, e.g. ['@Test', '@Parameters']
:param suite: the mx suite in which to look for test classes. If no suite is given, the primary suite is used.
:param JDKConfig jdk: the JDK for which the list of classes must be found
:param str buildCacheDir: a path relative to the mx suite output root that is used to store the cache files.
:return: a dictionary associating each found test class with the distribution it occurs in.
"""
assert not isabs(buildCacheDir), "buildCacheDir must be a relative path"
compat_suite = suite if suite else mx.primary_suite()
if suite != mx._mx_suite and compat_suite.getMxCompatibility(
).useDistsForUnittest():
jar_distributions = [
d for d in mx.sorted_dists()
if d.isJARDistribution() and exists(d.classpath_repr(
resolve=False)) and (not suite or d.suite == suite)
]
# find a corresponding distribution for each test
candidates = _find_classes_by_annotated_methods(
annotations, jar_distributions, buildCacheDir, jdk)
else:
binary_deps = [
d for d in mx.dependencies(opt_limit_to_suite=True)
if d.isJARDistribution() and isinstance(d.suite, mx.BinarySuite)
and (not suite or suite == d.suite)
]
candidates = _find_classes_by_annotated_methods(
annotations, binary_deps, buildCacheDir, jdk)
for p in mx.projects(opt_limit_to_suite=True):
if not p.isJavaProject():
continue
if suite and not p.suite == suite:
continue
if jdk.javaCompliance < p.javaCompliance:
continue
for c in _find_classes_with_annotations(p, None, annotations):
candidates[c] = p
return candidates
def spotbugs(args,
fbArgs=None,
suite=None,
projects=None,
jarFileName='spotbugs.jar'):
projectsToTest = [p for p in mx.projects() if _should_test_project(p)]
projectsByVersion = {}
for p in projectsToTest:
compat = p.suite.getMxCompatibility()
spotbugsVersion = compat.spotbugs_version()
if spotbugsVersion not in projectsByVersion:
projectsByVersion[spotbugsVersion] = []
projectsByVersion[spotbugsVersion].append(p)
resultcode = 0
for spotbugsVersion, versionProjects in projectsByVersion.items():
mx.logv('Running spotbugs version {} on projects {}'.format(
spotbugsVersion, versionProjects))
resultcode = max(
resultcode,
_spotbugs(args, fbArgs, suite, versionProjects, spotbugsVersion))
return resultcode
def _sigtest_generate(args, suite=None, projects=None):
"""run sigtest generator for Java projects with API"""
sigtestlib = mx.library('SIGTEST').get_path(resolve=True)
nonTestProjects = [p for p in mx.projects() if _should_test_project(p)]
if not nonTestProjects:
return 0
javaCompliance = max([p.javaCompliance for p in nonTestProjects])
for p in nonTestProjects:
sigtestResults = p.dir + os.sep + 'snapshot.sigtest'
cmd = ['-cp', mx._cygpathU2W(sigtestlib), 'com.sun.tdk.signaturetest.Setup',
'-Static', '-FileName', sigtestResults,
'-ClassPath', mx.classpath(p) + os.pathsep + mx.get_jdk(javaCompliance).bootclasspath(),
]
for pkg in mx.find_packages(p):
cmd = cmd + ['-PackageWithoutSubpackages', pkg]
exitcode = mx.run_java(cmd, nonZeroIsFatal=False, jdk=mx.get_jdk(javaCompliance))
if exitcode != 95:
mx.abort('Exit code was ' + str(exitcode) + ' while generating ' + sigtestResults)
if not exists(sigtestResults):
mx.abort('Cannot generate ' + sigtestResults)
mx.log("Sigtest snapshot generated to " + sigtestResults)
return 0
def _sigtest_check(checktype, args, suite=None, projects=None):
"""run sigtest against Java projects with API"""
sigtestlib = mx.library('SIGTEST').get_path(resolve=True)
nonTestProjects = [p for p in mx.projects() if _should_test_project(p)]
if not nonTestProjects:
return 1
javaCompliance = max([p.javaCompliance for p in nonTestProjects])
class OutputCapture:
def __init__(self):
self.data = ""
def __call__(self, data):
self.data += data
failed = None
for p in nonTestProjects:
sigtestResults = p.dir + os.sep + 'snapshot.sigtest'
if not os.path.exists(sigtestResults):
continue
jdk = mx.get_jdk(javaCompliance)
cmd = [
'-cp',
mx._cygpathU2W(sigtestlib),
'com.sun.tdk.signaturetest.SignatureTest',
'-Static',
'-Mode',
'bin',
'-FileName',
sigtestResults,
'-ClassPath',
mx.classpath(p, jdk=jdk) + os.pathsep + jdk.bootclasspath(),
]
if checktype != 'all':
cmd.append('-b')
for pkg in mx._find_packages(p):
cmd = cmd + ['-PackageWithoutSubpackages', pkg]
out = OutputCapture()
print 'Checking ' + checktype + ' signature changes against ' + sigtestResults
exitcode = mx.run_java(cmd,
nonZeroIsFatal=False,
jdk=mx.get_jdk(javaCompliance),
out=out,
err=out)
mx.ensure_dir_exists(p.get_output_root())
with open(p.get_output_root() + os.path.sep + 'sigtest-junit.xml',
'w') as f:
f.write('<?xml version="1.0" encoding="UTF-8" ?>\n')
f.write('<testsuite tests="1" name="' + p.name + '.sigtest.' +
checktype + '">\n')
f.write('<testcase classname="' + p.name + '" name="sigtest.' +
checktype + '">\n')
if exitcode != 95:
print out.data
failed = sigtestResults
f.write('<failure type="SignatureCheck"><![CDATA[\n')
f.write(out.data)
f.write(']]></failure>')
else:
f.write('<system-err><![CDATA[\n')
f.write(out.data)
f.write(']]></system-err>')
f.write('</testcase>\n')
f.write('</testsuite>\n')
if failed:
mx.abort('Signature error in ' + failed)
else:
print 'OK.'
return 0
def _unittest_config_participant(config):
vmArgs, mainClass, mainClassArgs = config
cpIndex, cp = mx.find_classpath_arg(vmArgs)
if cp:
cp = _uniqify(cp.split(os.pathsep))
if isJDK8:
# Remove entries from class path that are in Graal
excluded = set()
for entry in _jvmci_classpath:
dist = entry.dist()
excluded.update((d.output_dir() for d in dist.archived_deps() if d.isJavaProject()))
excluded.add(dist.path)
cp = os.pathsep.join([e for e in cp if e not in excluded])
vmArgs[cpIndex] = cp
else:
# Remove entries from class path that are in the Graal
assert _graal_module_descriptor is not None
module = as_java_module(_graal_module_descriptor.dist(), jdk)
junitCp = [e.classpath_repr() for e in mx.classpath_entries(['JUNIT'])]
excluded = frozenset([classpathEntry.classpath_repr() for classpathEntry in get_module_deps(module.dist)] + junitCp)
cp = [classpathEntry for classpathEntry in cp if classpathEntry not in excluded]
vmArgs[cpIndex] = os.pathsep.join(cp)
# Junit libraries are made into automatic modules so that they are visible to tests
# patched into Graal. These automatic modules must be declared to be read by
# Graal which means they must also be made root modules (i.e., ``-addmods``)
# since ``-XaddReads`` can only be applied to root modules.
junitModules = [_automatic_module_name(e) for e in junitCp]
vmArgs.extend(['-modulepath', os.pathsep.join(junitCp)])
vmArgs.extend(['-addmods', ','.join(junitModules)])
vmArgs.extend(['-XaddReads:' + module.name + '=' + ','.join(junitModules)])
# Explicitly export JVMCI to Graal
addedExports = {}
for concealingModule, packages in module.concealedRequires.iteritems():
if concealingModule == 'jdk.vm.ci':
for package in packages:
addedExports.setdefault(concealingModule + '/' + package, set()).add(module.name)
patches = []
graalConcealedPackages = list(module.conceals)
pathToProject = {p.output_dir() : p for p in mx.projects() if p.isJavaProject()}
for classpathEntry in cp:
# Export concealed JDK packages used by the class path entry
_add_exports_for_concealed_packages(classpathEntry, pathToProject, addedExports, 'ALL-UNNAMED')
# Patch the class path entry into Graal if it defines packages already defined by Graal.
# Packages definitions cannot be split between modules.
packages = frozenset(_defined_packages(classpathEntry))
if not packages.isdisjoint(module.packages):
patches.append(classpathEntry)
extraPackages = packages - module.packages
if extraPackages:
# From http://openjdk.java.net/jeps/261:
# If a package found in a module definition on a patch path is not already exported
# by that module then it will, still, not be exported. It can be exported explicitly
# via either the reflection API or the -XaddExports option.
graalConcealedPackages.extend(extraPackages)
if patches:
vmArgs.append('-Xpatch:' + module.name + '=' + os.pathsep.join(patches))
# Export all Graal packages to make them available to test classes
for package in graalConcealedPackages:
addedExports.setdefault(module.name + '/' + package, set()).update(junitModules + ['ALL-UNNAMED'])
vmArgs.extend(['-XaddExports:' + export + '=' + ','.join(sorted(targets)) for export, targets in addedExports.iteritems()])
if isJDK8:
# Run the VM in a mode where application/test classes can
# access JVMCI loaded classes.
vmArgs.append('-XX:-UseJVMCIClassLoader')
return (vmArgs, mainClass, mainClassArgs)
def _run_tests(args, harness, vmLauncher, annotations, testfile, blacklist, whitelist, regex, suite):
vmArgs, tests = mx.extract_VM_args(args)
for t in tests:
if t.startswith("-"):
mx.abort("VM option " + t + " must precede " + tests[0])
candidates = {}
for p in mx.projects(opt_limit_to_suite=True):
if not p.isJavaProject():
continue
if suite and not p.suite == suite:
continue
if mx.get_jdk().javaCompliance < p.javaCompliance:
continue
for c in _find_classes_with_annotations(p, None, annotations):
candidates[c] = p
classes = []
if len(tests) == 0:
classes = candidates.keys()
jdk = mx.get_jdk()
projectsCp = mx.classpath(
[
pcp.name
for pcp in mx.projects(opt_limit_to_suite=True)
if pcp.isJavaProject() and pcp.javaCompliance <= jdk.javaCompliance
]
)
else:
projs = set()
found = False
if len(tests) == 1 and "#" in tests[0]:
words = tests[0].split("#")
if len(words) != 2:
mx.abort("Method specification is class#method: " + tests[0])
t, method = words
for c, p in candidates.iteritems():
# prefer exact matches first
if t == c:
found = True
classes.append(c)
projs.add(p.name)
if not found:
for c, p in candidates.iteritems():
if t in c:
found = True
classes.append(c)
projs.add(p.name)
if not found:
mx.log('warning: no tests matched by substring "' + t)
elif len(classes) != 1:
mx.abort("More than one test matches substring {0} {1}".format(t, classes))
classes = [c + "#" + method for c in classes]
else:
for t in tests:
if "#" in t:
mx.abort("Method specifications can only be used in a single test: " + t)
for c, p in candidates.iteritems():
if t in c:
found = True
classes.append(c)
projs.add(p.name)
if not found:
mx.log('warning: no tests matched by substring "' + t)
projectsCp = mx.classpath(projs)
if blacklist:
classes = [c for c in classes if not any((glob.match(c) for glob in blacklist))]
if whitelist:
classes = [c for c in classes if any((glob.match(c) for glob in whitelist))]
if regex:
classes = [c for c in classes if re.search(regex, c)]
if len(classes) != 0:
f_testfile = open(testfile, "w")
for c in classes:
f_testfile.write(c + "\n")
f_testfile.close()
harness(projectsCp, vmLauncher, vmArgs)
def _unittest_config_participant(config):
vmArgs, mainClass, mainClassArgs = config
cpIndex, cp = mx.find_classpath_arg(vmArgs)
if cp:
cp = _uniqify(cp.split(os.pathsep))
if isJDK8:
# Remove entries from class path that are in Graal or on the boot class path
redundantClasspathEntries = set()
for dist in [entry.dist() for entry in _jvmci_classpath]:
redundantClasspathEntries.update((d.output_dir() for d in dist.archived_deps() if d.isJavaProject()))
redundantClasspathEntries.add(dist.path)
cp = os.pathsep.join([e for e in cp if e not in redundantClasspathEntries])
vmArgs[cpIndex] = cp
else:
deployedModules = []
redundantClasspathEntries = set()
for dist in [entry.dist() for entry in _jvmci_classpath] + _bootclasspath_appends:
deployedModule = as_java_module(dist, jdk)
deployedModules.append(deployedModule)
redundantClasspathEntries.update(mx.classpath(dist, preferProjects=False, jdk=jdk).split(os.pathsep))
redundantClasspathEntries.update(mx.classpath(dist, preferProjects=True, jdk=jdk).split(os.pathsep))
if hasattr(dist, 'overlaps'):
for o in dist.overlaps:
path = mx.distribution(o).classpath_repr()
if path:
redundantClasspathEntries.add(path)
# Remove entries from the class path that are in the deployed modules
cp = [classpathEntry for classpathEntry in cp if classpathEntry not in redundantClasspathEntries]
vmArgs[cpIndex] = os.pathsep.join(cp)
# Junit libraries are made into automatic modules so that they are visible to tests
# patched into modules. These automatic modules must be declared to be read by
# Graal which means they must also be made root modules (i.e., ``--add-modules``)
# since ``--add-reads`` can only be applied to root modules.
junitCp = [e.classpath_repr() for e in mx.classpath_entries(['JUNIT'])]
junitModules = [_automatic_module_name(e) for e in junitCp]
vmArgs.append('--module-path=' + os.pathsep.join(junitCp))
vmArgs.append('--add-modules=' + ','.join(junitModules + [m.name for m in deployedModules]))
for deployedModule in deployedModules:
vmArgs.append('--add-reads=' + deployedModule.name + '=' + ','.join(junitModules))
# Explicitly export concealed JVMCI packages required by Graal. Even though
# normally exported via jdk.vm.ci.services.Services.exportJVMCITo(), the
# Junit harness wants to access JVMCI classes (e.g., when loading classes
# to find test methods) without going through that entry point.
addedExports = {}
for deployedModule in deployedModules:
for concealingModule, packages in deployedModule.concealedRequires.iteritems():
if concealingModule == 'jdk.vm.ci':
for package in packages:
addedExports.setdefault(concealingModule + '/' + package, set()).add(deployedModule.name)
pathToProject = {p.output_dir() : p for p in mx.projects() if p.isJavaProject()}
for classpathEntry in cp:
# Export concealed packages used by the class path entry
_add_exports_for_concealed_packages(classpathEntry, pathToProject, addedExports, 'ALL-UNNAMED', deployedModules)
for deployedModule in deployedModules:
assert deployedModule.dist.path != classpathEntry, deployedModule.dist.path + ' should no longer be on the class path'
# Ensure the class path entry does not define packages already defined by the module.
# Packages definitions cannot be split between modules.
classpathEntryPackages = frozenset(_defined_packages(classpathEntry))
intersection = classpathEntryPackages.intersection(deployedModule.packages)
if intersection:
mx.abort(classpathEntry + ' cannot extend package(s) defined in the module ' + deployedModule.name + ': ' + ', '.join(intersection))
vmArgs.extend(['--add-exports=' + export + '=' + ','.join(sorted(targets)) for export, targets in addedExports.iteritems()])
if isJDK8:
# Run the VM in a mode where application/test classes can
# access JVMCI loaded classes.
vmArgs.append('-XX:-UseJVMCIClassLoader')
return (vmArgs, mainClass, mainClassArgs)
def coverage_upload(args):
parser = ArgumentParser(prog='mx coverage-upload')
parser.add_argument('--upload-url', required=False, default=mx.get_env('COVERAGE_UPLOAD_URL'), help='Format is like rsync: user@host:/directory')
parser.add_argument('--build-name', required=False, default=mx.get_env('BUILD_NAME'))
parser.add_argument('--build-url', required=False, default=mx.get_env('BUILD_URL'))
parser.add_argument('--build-number', required=False, default=mx.get_env('BUILD_NUMBER'))
args, other_args = parser.parse_known_args(args)
if not args.upload_url:
parser.print_help()
return
remote_host, remote_basedir = args.upload_url.split(':')
if not remote_host:
mx.abort('Cannot determine remote host from {}'.format(args.upload_url))
primary = mx.primary_suite()
info = primary.vc.parent_info(primary.dir)
rev = primary.vc.parent(primary.dir)
if len(remote_basedir) > 0 and not remote_basedir.endswith('/'):
remote_basedir += '/'
remote_dir = '{}_{}_{}'.format(primary.name, datetime.datetime.fromtimestamp(info['author-ts']).strftime('%Y-%m-%d_%H_%M'), rev[:7])
if args.build_name:
remote_dir += '_' + args.build_name
if args.build_number:
remote_dir += '_' + args.build_number
upload_dir = remote_basedir + remote_dir
includes, excludes = _jacocoreport(['--omit-excluded'] + other_args)
# Upload jar+sources
coverage_sources = 'java_sources.tar.gz'
coverage_binaries = 'java_binaries.tar.gz'
with mx.Archiver(os.path.realpath(coverage_sources), kind='tgz') as sources, mx.Archiver(os.path.realpath(coverage_binaries), kind='tgz') as binaries:
def _visit_deps(dep, edge):
if dep.isJavaProject() and not dep.is_test_project():
binaries.zf.add(dep.output_dir(), dep.name)
for d in dep.source_dirs():
sources.zf.add(d, dep.name)
if os.path.exists(dep.source_gen_dir()):
sources.zf.add(dep.source_gen_dir(), dep.name)
mx.walk_deps(mx.projects(), visit=_visit_deps)
files = [get_jacoco_dest_file(), 'coverage', coverage_sources, coverage_binaries]
print("Syncing {} to {}:{}".format(" ".join(files), remote_host, upload_dir))
mx.run([
'bash',
'-c',
r'tar -czf - {files} | ssh {remote} bash -c \'"mkdir -p {remotedir} && cd {remotedir} && cat | tar -x{verbose}z && chmod -R 755 ."\''
.format(
files=" ".join(files),
remote=remote_host,
remotedir=upload_dir,
verbose='v' if mx._opts.verbose else '')
])
def upload_string(content, path):
mx.run(['ssh', remote_host, 'bash', '-c', 'cat > "' + path + '"'], stdin=content)
upload_string(json.dumps({
'timestamp': time.time(),
'suite': primary.name,
'revision': rev,
'directory': remote_dir,
'build_name': args.build_name,
'build_url': args.build_url,
'jdk_version': str(mx.get_jdk().version),
'build_number': args.build_number,
'primary_info': info,
'excludes': [str(e) for e in excludes],
'includes': [str(i) for i in includes]}), upload_dir + '/description.json')
mx.run(['ssh', remote_host, 'bash', '-c', r'"(echo \[; for i in {remote_basedir}/*/description.json; do if \[ -s \$i \];then cat \$i; echo ,; fi done; echo null\]) > {remote_basedir}/index.json"'.format(remote_basedir=remote_basedir)])
upload_string("""<html>
<script language="javascript">
function urlChange(url) {
if (url.pathname !== "blank") {
window.history.replaceState(null, null, url.pathname.replace("/coverage_upload/", "/coverage_upload/#"))
}
}
</script>
<frameset rows="40,*">
<frame id="navigation" src="navigation.html"/>
<frame id="content" src="" onload="urlChange(this.contentWindow.location);" />
</frameset>
</html>""", remote_basedir + '/index.html')
js_library_url = rewriteurl("https://ajax.googleapis.com/ajax/libs/angularjs/1.7.7/angular.js")
upload_string(r"""<html>
<head>
<script src="%js_library_url"></script>
<script language="javascript">
var App = angular.module('myApp', [])
.controller('IndexCtrl', function IndexCtrl($scope, $http) {
var hash = parent.window.location.hash;
if(hash) {
hash = hash.substring(1, hash.length); // remove leading hash
}
$http.get('index.json').then(function(response, status) {
var data = response.data.filter(x => x != null);
/*
#GR-17399
Filter build that are unique per suite with revision as key and merge builds.
*/
data = data
.filter(x => !x.hasOwnProperty('merge'))
.filter( // filter builds that are unique per suite with revision as key
x => !data
.filter(z => x != z && x.suite == z.suite) // exclude self build and build for other suites.
.map(z => z.revision) // map from array of build to array of revision
.includes(x.revision) // check if revision of x is index data.
).concat(data.filter(x => x.hasOwnProperty('merge'))); // concat unique build with merged build.
data.sort((l,r) => r.timestamp - l.timestamp);
if(data.length > 0) {
var startdir;
if(hash) {
startdir = data.find(build => hash.includes(build.directory));
startdir.hash = hash;
}
if(!startdir) {
startdir = data[0];
}
$scope.directory = startdir;
}
$scope.data = data;
});
$scope.$watch('directory', (dir, olddir) => {
if(dir) {
var content = parent.document.getElementById("content");
var contentDocument = content.contentDocument || content.contentWindow.document;
var newpath;
if(olddir && olddir.suite === dir.suite) {
newpath = contentDocument.location.href.replace(olddir.directory, dir.directory);
} else {
newpath = dir.hasOwnProperty('hash') ? hash : dir.directory + "/coverage/";
}
contentDocument.location.href = newpath;
parent.window.history.replaceState(undefined, undefined, "#" + newpath.replace(/^.+coverage_upload\//, ""));
}
});
$scope.step = (i) => $scope.directory = $scope.data[$scope.data.indexOf($scope.directory)+i];
});
function copy(url) {
var content = parent.document.getElementById("content");
var contentDocument = content.contentDocument || content.contentWindow.document;
var copyText = document.getElementById("copy");
copyText.value = contentDocument.location.href.replace("coverage_upload/", "coverage_upload/#");
copyText.select();
document.execCommand("copy");
}
</script>
</head>
<body ng-app="myApp" ng-controller="IndexCtrl">
<button ng-click="step(1)" ng-disabled="data.indexOf(directory) >= data.length-1"><<</button>
<button ng-click="step(-1)" ng-disabled="data.indexOf(directory) <= 0">>></button>
<select ng-model="directory" ng-options="(i.primary_info['author-ts']*1000|date:'yy-MM-dd hh:mm') + ' ' + i.build_name + ' ' + i.revision.substr(0,8) group by i.suite for i in data"></select>
<a href="{{directory.build_url}}" ng-if="directory.build_url" target="_blank">Build</a> Commit: {{directory.revision.substr(0,5)}}: {{directory.primary_info.description}}
<input type="text" style="opacity: 0;width: 20;" id="copy" />
<button style="float: right;" onclick="copy(window.location);">Share url</button>
</body>
</html>""".replace("%js_library_url", js_library_url), remote_basedir + '/navigation.html')
def junit(args, harness, parser=None, jdk_default=None):
"""run Junit tests"""
suppliedParser = parser is not None
parser = parser if suppliedParser else ArgumentParser(prog='mx junit')
parser.add_argument('--tests', action='store', help='pattern to match test classes')
parser.add_argument('--J', dest='vm_args', action='append', help='target VM arguments (e.g. --J @-dsa)', metavar='@<args>')
parser.add_argument('--jdk', action='store', help='jdk to use')
if suppliedParser:
parser.add_argument('remainder', nargs=REMAINDER, metavar='...')
args = parser.parse_args(args)
vmArgs = ['-ea', '-esa']
if args.vm_args:
vmArgs = vmArgs + mx_fastr.split_j_args(args.vm_args)
testfile = os.environ.get('MX_TESTFILE', None)
if testfile is None:
(_, testfile) = tempfile.mkstemp(".testclasses", "mx")
os.close(_)
candidates = []
if args.jdk:
jdk = mx.get_jdk(tag=args.jdk)
if not jdk:
mx.abort("jdk '" + args.jdk + "' not found")
else:
if not jdk_default:
jdk = mx.get_jdk()
else:
jdk = jdk_default
for p in mx.projects(opt_limit_to_suite=True):
if not p.isJavaProject() or jdk.javaCompliance < p.javaCompliance:
continue
candidates += _find_classes_with_annotations(p, None, ['@Test']).keys()
tests = [] if args.tests is None else [name for name in args.tests.split(',')]
classes = []
if len(tests) == 0:
classes = candidates
else:
for t in tests:
found = False
for c in candidates:
if t in c:
found = True
classes.append(c)
if not found:
mx.warn('no tests matched by substring "' + t + '"')
vmArgs += mx.get_runtime_jvm_args([pcp.name for pcp in mx.projects(opt_limit_to_suite=True) if pcp.isJavaProject() and pcp.javaCompliance <= jdk.javaCompliance], jdk=jdk)
if len(classes) != 0:
if len(classes) == 1:
testClassArgs = ['--testclass', classes[0]]
else:
with open(testfile, 'w') as f:
for c in classes:
f.write(c + '\n')
testClassArgs = ['--testsfile', testfile]
junitArgs = ['com.oracle.truffle.r.test.FastRJUnitWrapper'] + testClassArgs
rc = harness(args, vmArgs, jdk, junitArgs)
return rc
else:
return 0
def findbugs(args, fbArgs=None, suite=None, projects=None):
"""run FindBugs against non-test Java projects"""
findBugsHome = mx.get_env('FINDBUGS_HOME', None)
if suite is None:
suite = mx.primary_suite()
if findBugsHome:
findbugsJar = join(findBugsHome, 'lib', 'findbugs.jar')
else:
findbugsLib = join(mx._mx_suite.get_output_root(), 'findbugs-3.0.0')
if not exists(findbugsLib):
tmp = tempfile.mkdtemp(prefix='findbugs-download-tmp', dir=mx._mx_suite.dir)
try:
findbugsDist = mx.library('FINDBUGS_DIST').get_path(resolve=True)
with zipfile.ZipFile(findbugsDist) as zf:
candidates = [e for e in zf.namelist() if e.endswith('/lib/findbugs.jar')]
assert len(candidates) == 1, candidates
libDirInZip = os.path.dirname(candidates[0])
zf.extractall(tmp)
shutil.copytree(join(tmp, libDirInZip), findbugsLib)
finally:
shutil.rmtree(tmp)
findbugsJar = join(findbugsLib, 'findbugs.jar')
assert exists(findbugsJar)
projectsToTest = [p for p in mx.projects() if _should_test_project(p)]
if not projectsToTest:
return 0
ignoredClasses = set()
for p in projectsToTest:
ignore = getattr(p, 'findbugsIgnoresGenerated', False)
if not isinstance(ignore, bool):
mx.abort('Value of attribute "findbugsIgnoresGenerated" must be True or False', context=p)
if ignore is True:
sourceDir = p.source_gen_dir()
for root, _, files in os.walk(sourceDir):
for name in files:
if name.endswith('.java') and '-info' not in name:
pkg = root[len(sourceDir) + 1:].replace(os.sep, '.')
cls = pkg + '.' + name[:-len('.java')]
ignoredClasses.add(cls)
with tempfile.NamedTemporaryFile(suffix='.xml', prefix='findbugs_exclude_filter.', mode='w', delete=False) as fp:
findbugsExcludeFilterFile = fp.name
xmlDoc = mx.XMLDoc()
xmlDoc.open('FindBugsFilter')
# Javac from JDK11 might have been used to compile the classes so
# disable the check for redundant null tests since FindBugs does
# not (yet) detect and suppress warnings about patterns generated by
# this version of javac. This will be removed once
# https://github.com/spotbugs/spotbugs/issues/600 is resolved.
xmlDoc.open('Match')
xmlDoc.element('Bug', attributes={'pattern' : 'RCN_REDUNDANT_NULLCHECK_WOULD_HAVE_BEEN_A_NPE'})
xmlDoc.close('Match')
for cls in ignoredClasses:
xmlDoc.open('Match')
xmlDoc.element('Class', attributes={'name' : '~' + cls + '.*'})
xmlDoc.close('Match')
xmlDoc.close('FindBugsFilter')
xml = xmlDoc.xml(indent=' ', newl='\n')
print >> fp, xml
outputDirs = map(mx._cygpathU2W, [p.output_dir() for p in projectsToTest])
javaCompliance = max([p.javaCompliance for p in projectsToTest])
jdk = mx.get_jdk(javaCompliance)
if jdk.javaCompliance >= '9':
mx.log('FindBugs does not yet support JDK9 - skipping')
return 0
findbugsResults = join(suite.dir, 'findbugs.results')
if fbArgs is None:
fbArgs = defaultFindbugsArgs()
cmd = ['-jar', mx._cygpathU2W(findbugsJar)] + fbArgs
cmd = cmd + ['-exclude', findbugsExcludeFilterFile]
cmd = cmd + ['-auxclasspath', mx._separatedCygpathU2W(mx.classpath([p.name for p in projectsToTest], jdk=jdk)), '-output', mx._cygpathU2W(findbugsResults), '-exitcode'] + args + outputDirs
try:
exitcode = mx.run_java(cmd, nonZeroIsFatal=False, jdk=jdk)
finally:
os.unlink(findbugsExcludeFilterFile)
if exitcode != 0:
with open(findbugsResults) as fp:
mx.log(fp.read())
os.unlink(findbugsResults)
return exitcode
def junit(args):
'''run ZipPy Junit tests'''
parser = ArgumentParser(prog='mx junit')
parser.add_argument(
'--tests',
action='store',
help=
'patterns to match test classes (specify multiple patterns using \',\')'
)
parser.add_argument('--J',
dest='vm_args',
action='append',
help='Java VM arguments (e.g. --J @-dsa)',
metavar='@<args>')
parser.add_argument('--jdk',
action='store',
help='JDK to use for the "java" command')
args = parser.parse_args(args)
vmArgs = ['-ea', '-esa']
# enable when necessary
# vmArgs += ['-Xss12m']
if args.vm_args:
vmArgs = vmArgs + mx_fastr.split_j_args(args.vm_args)
testfile = os.environ.get('MX_TESTFILE', None)
if testfile is None:
(_, testfile) = tempfile.mkstemp(".testclasses", "mx")
os.close(_)
if args.jdk:
jdk = mx.get_jdk(tag=args.jdk)
if not jdk:
mx.abort("JDK '" + args.jdk + "' not found!")
else:
tag = 'jvmci' if _mx_graal else None
jdk = mx.get_jdk(tag=tag)
candidates = []
for p in mx.projects(opt_limit_to_suite=True, limit_to_primary=True):
if not p.isJavaProject() or jdk.javaCompliance < p.javaCompliance:
continue
candidates += mx_unittest._find_classes_with_annotations(
p, None, ['@Test']).keys()
tests = [] if args.tests is None else [
name for name in args.tests.split(',')
]
classes = []
if len(tests) == 0:
classes = candidates
else:
for test in tests:
exists = False
for candidate in candidates:
if test in candidate:
exists = True
classes.append(candidate)
if not exists:
mx.warn('no tests matched by substring "' + test + '"')
vmArgs += mx.get_runtime_jvm_args(['ZIPPY', 'ZIPPY_UNIT_TESTS'], jdk=jdk)
if len(classes) != 0:
if len(classes) == 1:
testClassArgs = ['--testclass', classes[0]]
else:
with open(testfile, 'w') as f:
for c in classes:
f.write(c + '\n')
testClassArgs = ['--testsfile', testfile]
junitArgs = ['edu.uci.python.test.ZipPyJUnitRunner'] + testClassArgs
rc = mx.run_java(vmArgs + junitArgs, nonZeroIsFatal=False, jdk=jdk)
return rc
else:
return 0
def _run_tests(args, harness, vmLauncher, annotations, testfile, blacklist,
whitelist, regex, suite):
vmArgs, tests = mx.extract_VM_args(args)
for t in tests:
if t.startswith('-'):
mx.abort('VM option ' + t + ' must precede ' + tests[0])
# this is what should be used
compat_suite = suite if suite else mx.primary_suite()
if suite != mx._mx_suite and compat_suite.getMxCompatibility(
).useDistsForUnittest():
jar_distributions = [
d for d in mx.sorted_dists()
if d.isJARDistribution() and exists(d.classpath_repr(
resolve=False)) and (not suite or d.suite == suite)
]
# find a corresponding distribution for each test
candidates = _find_classes_by_annotated_methods(
annotations, jar_distributions, vmLauncher.jdk())
else:
binary_deps = [
d for d in mx.dependencies(opt_limit_to_suite=True)
if d.isJARDistribution() and isinstance(d.suite, mx.BinarySuite)
and (not suite or suite == d.suite)
]
candidates = _find_classes_by_annotated_methods(
annotations, binary_deps, vmLauncher.jdk())
for p in mx.projects(opt_limit_to_suite=True):
if not p.isJavaProject():
continue
if suite and not p.suite == suite:
continue
if vmLauncher.jdk().javaCompliance < p.javaCompliance:
continue
for c in _find_classes_with_annotations(p, None, annotations):
candidates[c] = p
classes = []
if len(tests) == 0:
classes = candidates.keys()
depsContainingTests = set(candidates.values())
else:
depsContainingTests = set()
found = False
if len(tests) == 1 and '#' in tests[0]:
words = tests[0].split('#')
if len(words) != 2:
mx.abort("Method specification is class#method: " + tests[0])
t, method = words
for c, p in candidates.iteritems():
# prefer exact matches first
if t == c:
found = True
classes.append(c)
depsContainingTests.add(p)
if not found:
for c, p in candidates.iteritems():
if t in c:
found = True
classes.append(c)
depsContainingTests.add(p)
if not found:
mx.warn('no tests matched by substring: ' + t +
' (did you forget to run "mx build"?)')
elif len(classes) != 1:
mx.abort('More than one test matches substring {0} {1}'.format(
t, classes))
classes = [c + "#" + method for c in classes]
else:
for t in tests:
if '#' in t:
mx.abort(
'Method specifications can only be used in a single test: '
+ t)
for c, p in candidates.iteritems():
if t in c:
found = True
classes.append(c)
depsContainingTests.add(p)
if not found:
mx.warn('no tests matched by substring: ' + t +
' (did you forget to run "mx build"?)')
if blacklist:
classes = [
c for c in classes
if not any((glob.match(c) for glob in blacklist))
]
if whitelist:
classes = [
c for c in classes if any((glob.match(c) for glob in whitelist))
]
if regex:
classes = [c for c in classes if re.search(regex, c)]
if len(classes) != 0:
f_testfile = open(testfile, 'w')
for c in classes:
f_testfile.write(c + '\n')
f_testfile.close()
harness(depsContainingTests, vmLauncher, vmArgs)
def _run_tests(args, harness, vmLauncher, annotations, testfile, blacklist, whitelist, regex, suite):
vmArgs, tests = mx.extract_VM_args(args)
for t in tests:
if t.startswith('-'):
mx.abort('VM option ' + t + ' must precede ' + tests[0])
# Dictionary from fully qualified class names to the project or distribution containing the class
candidates = _find_classes_by_annotated_methods(annotations, suite)
jdk = mx.get_jdk()
for p in mx.projects(opt_limit_to_suite=True):
if not p.isJavaProject():
continue
if suite and not p.suite == suite:
continue
if jdk.javaCompliance < p.javaCompliance:
continue
for c in _find_classes_with_annotations(p, None, annotations):
candidates[c] = p
classes = []
if len(tests) == 0:
classes = candidates.keys()
depsContainingTests = set(candidates.values())
else:
depsContainingTests = set()
found = False
if len(tests) == 1 and '#' in tests[0]:
words = tests[0].split('#')
if len(words) != 2:
mx.abort("Method specification is class#method: " + tests[0])
t, method = words
for c, p in candidates.iteritems():
# prefer exact matches first
if t == c:
found = True
classes.append(c)
depsContainingTests.add(p)
if not found:
for c, p in candidates.iteritems():
if t in c:
found = True
classes.append(c)
depsContainingTests.add(p)
if not found:
mx.log('warning: no tests matched by substring "' + t)
elif len(classes) != 1:
mx.abort('More than one test matches substring {0} {1}'.format(t, classes))
classes = [c + "#" + method for c in classes]
else:
for t in tests:
if '#' in t:
mx.abort('Method specifications can only be used in a single test: ' + t)
for c, p in candidates.iteritems():
if t in c:
found = True
classes.append(c)
depsContainingTests.add(p)
if not found:
mx.log('warning: no tests matched by substring "' + t)
unittestCp = mx.classpath(depsContainingTests)
if blacklist:
classes = [c for c in classes if not any((glob.match(c) for glob in blacklist))]
if whitelist:
classes = [c for c in classes if any((glob.match(c) for glob in whitelist))]
if regex:
classes = [c for c in classes if re.search(regex, c)]
if len(classes) != 0:
f_testfile = open(testfile, 'w')
for c in classes:
f_testfile.write(c + '\n')
f_testfile.close()
harness(unittestCp, vmLauncher, vmArgs)
def _unittest_config_participant(config):
vmArgs, mainClass, mainClassArgs = config
cpIndex, cp = mx.find_classpath_arg(vmArgs)
if cp:
cp = _uniqify(cp.split(os.pathsep))
if isJDK8:
# Remove entries from class path that are in Graal or on the boot class path
redundantClasspathEntries = set()
for dist in [entry.dist() for entry in _jvmci_classpath]:
redundantClasspathEntries.update((d.output_dir() for d in dist.archived_deps() if d.isJavaProject()))
redundantClasspathEntries.add(dist.path)
cp = os.pathsep.join([e for e in cp if e not in redundantClasspathEntries])
vmArgs[cpIndex] = cp
else:
deployedModules = []
redundantClasspathEntries = set()
for dist in [entry.dist() for entry in _jvmci_classpath] + _bootclasspath_appends:
deployedModule = as_java_module(dist, jdk)
deployedModules.append(deployedModule)
redundantClasspathEntries.update(mx.classpath(dist, preferProjects=False, jdk=jdk).split(os.pathsep))
redundantClasspathEntries.update(mx.classpath(dist, preferProjects=True, jdk=jdk).split(os.pathsep))
# Remove entries from the class path that are in the deployed modules
cp = [classpathEntry for classpathEntry in cp if classpathEntry not in redundantClasspathEntries]
vmArgs[cpIndex] = os.pathsep.join(cp)
# Junit libraries are made into automatic modules so that they are visible to tests
# patched into modules. These automatic modules must be declared to be read by
# Graal which means they must also be made root modules (i.e., ``-addmods``)
# since ``-XaddReads`` can only be applied to root modules.
junitCp = [e.classpath_repr() for e in mx.classpath_entries(['JUNIT'])]
junitModules = [_automatic_module_name(e) for e in junitCp]
vmArgs.extend(['-modulepath', os.pathsep.join(junitCp)])
vmArgs.extend(['-addmods', ','.join(junitModules + [m.name for m in deployedModules])])
for deployedModule in deployedModules:
vmArgs.append('-XaddReads:' + deployedModule.name + '=' + ','.join(junitModules))
# Explicitly export concealed JVMCI packages required by Graal. Even though
# normally exported via jdk.vm.ci.services.Services.exportJVMCITo(), the
# Junit harness wants to access JVMCI classes (e.g., when loading classes
# to find test methods) without going through that entry point.
addedExports = {}
for deployedModule in deployedModules:
for concealingModule, packages in deployedModule.concealedRequires.iteritems():
if concealingModule == 'jdk.vm.ci':
for package in packages:
addedExports.setdefault(concealingModule + '/' + package, set()).add(deployedModule.name)
patches = {}
pathToProject = {p.output_dir() : p for p in mx.projects() if p.isJavaProject()}
for classpathEntry in cp:
# Export concealed packages used by the class path entry
_add_exports_for_concealed_packages(classpathEntry, pathToProject, addedExports, 'ALL-UNNAMED', deployedModules)
for deployedModule in deployedModules:
assert deployedModule.dist.path != classpathEntry, deployedModule.dist.path + ' should no longer be on the class path'
# Patch the class path entry into a module if it defines packages already defined by the module.
# Packages definitions cannot be split between modules.
classpathEntryPackages = frozenset(_defined_packages(classpathEntry))
if not classpathEntryPackages.isdisjoint(deployedModule.packages):
patches.setdefault(deployedModule.name, []).append(classpathEntry)
extraPackages = classpathEntryPackages - frozenset(deployedModule.exports.iterkeys())
if extraPackages:
# From http://openjdk.java.net/jeps/261:
# If a package found in a module definition on a patch path is not already exported
# by that module then it will, still, not be exported. It can be exported explicitly
# via either the reflection API or the -XaddExports option.
for package in extraPackages:
addedExports.setdefault(deployedModule.name + '/' + package, set()).update(junitModules + ['ALL-UNNAMED'])
for moduleName, cpEntries in patches.iteritems():
vmArgs.append('-Xpatch:' + moduleName + '=' + os.pathsep.join(cpEntries))
vmArgs.extend(['-XaddExports:' + export + '=' + ','.join(sorted(targets)) for export, targets in addedExports.iteritems()])
if isJDK8:
# Run the VM in a mode where application/test classes can
# access JVMCI loaded classes.
vmArgs.append('-XX:-UseJVMCIClassLoader')
return (vmArgs, mainClass, mainClassArgs)
def _run_tests(args, harness, vmLauncher, annotations, testfile, blacklist, whitelist, regex, suite):
vmArgs, tests = mx.extract_VM_args(args)
for t in tests:
if t.startswith('-'):
mx.abort('VM option ' + t + ' must precede ' + tests[0])
# this is what should be used
compat_suite = suite if suite else mx.primary_suite()
if suite != mx._mx_suite and compat_suite.getMxCompatibility().useDistsForUnittest():
jar_distributions = [d for d in mx.sorted_dists() if d.isJARDistribution() and (not suite or d.suite == suite)]
# find a corresponding distribution for each test
candidates = _find_classes_by_annotated_methods(annotations, jar_distributions, vmLauncher.jdk())
else:
binary_deps = [d for d in mx.dependencies(opt_limit_to_suite=True) if d.isJARDistribution() and
isinstance(d.suite, mx.BinarySuite) and (not suite or suite == d.suite)]
candidates = _find_classes_by_annotated_methods(annotations, binary_deps, vmLauncher.jdk())
for p in mx.projects(opt_limit_to_suite=True):
if not p.isJavaProject():
continue
if suite and not p.suite == suite:
continue
if vmLauncher.jdk().javaCompliance < p.javaCompliance:
continue
for c in _find_classes_with_annotations(p, None, annotations):
candidates[c] = p
classes = []
if len(tests) == 0:
classes = candidates.keys()
depsContainingTests = set(candidates.values())
else:
depsContainingTests = set()
found = False
if len(tests) == 1 and '#' in tests[0]:
words = tests[0].split('#')
if len(words) != 2:
mx.abort("Method specification is class#method: " + tests[0])
t, method = words
for c, p in candidates.iteritems():
# prefer exact matches first
if t == c:
found = True
classes.append(c)
depsContainingTests.add(p)
if not found:
for c, p in candidates.iteritems():
if t in c:
found = True
classes.append(c)
depsContainingTests.add(p)
if not found:
mx.log('warning: no tests matched by substring: ' + t)
elif len(classes) != 1:
mx.abort('More than one test matches substring {0} {1}'.format(t, classes))
classes = [c + "#" + method for c in classes]
else:
for t in tests:
if '#' in t:
mx.abort('Method specifications can only be used in a single test: ' + t)
for c, p in candidates.iteritems():
if t in c:
found = True
classes.append(c)
depsContainingTests.add(p)
if not found:
mx.log('warning: no tests matched by substring: ' + t)
if blacklist:
classes = [c for c in classes if not any((glob.match(c) for glob in blacklist))]
if whitelist:
classes = [c for c in classes if any((glob.match(c) for glob in whitelist))]
if regex:
classes = [c for c in classes if re.search(regex, c)]
if len(classes) != 0:
f_testfile = open(testfile, 'w')
for c in classes:
f_testfile.write(c + '\n')
f_testfile.close()
harness(depsContainingTests, vmLauncher, vmArgs)
def _sigtest_check(checktype, args, suite=None, projects=None):
"""run sigtest against Java projects with API"""
nonTestProjects = [p for p in mx.projects() if _should_test_project(p)]
if not nonTestProjects:
return 1
javaCompliance = max([p.javaCompliance for p in nonTestProjects])
class OutputCapture:
def __init__(self):
self.data = ""
def __call__(self, data):
self.data += data
failed = None
for p in nonTestProjects:
sigtestlib = p.suite.getMxCompatibility().get_sigtest_jar()
sigtestResults = p.dir + os.sep + 'snapshot.sigtest'
if not os.path.exists(sigtestResults):
continue
jdk = mx.get_jdk(javaCompliance)
cmd = [
'-cp',
mx._cygpathU2W(sigtestlib),
'com.sun.tdk.signaturetest.SignatureTest',
'-BootCP',
'-Static',
'-Mode',
'bin',
'-FileName',
sigtestResults,
'-ClassPath',
mx.classpath(p, jdk=jdk),
]
if args.human:
cmd.append('-H')
if checktype != 'all':
cmd.append('-b')
for pkg in mx._find_packages(p):
cmd = cmd + ['-PackageWithoutSubpackages', pkg]
out = OutputCapture()
print('Checking ' + checktype + ' signature changes against ' +
sigtestResults)
exitcode = mx.run_java(cmd,
nonZeroIsFatal=False,
jdk=mx.get_jdk(javaCompliance),
out=out,
err=out)
mx.ensure_dir_exists(p.get_output_root())
with open(p.get_output_root() + os.path.sep + 'sigtest-junit.xml',
'w') as f:
f.write('<?xml version="1.0" encoding="UTF-8" ?>\n')
f.write('<testsuite tests="1" name="' + p.name + '.sigtest.' +
checktype + '">\n')
f.write('<testcase classname="' + p.name + '" name="sigtest.' +
checktype + '">\n')
if exitcode != 95:
print(out.data)
failed = sigtestResults
f.write('<failure type="SignatureCheck"><![CDATA[\n')
f.write(out.data)
f.write(']]></failure>')
else:
f.write('<system-err><![CDATA[\n')
f.write(out.data)
f.write(']]></system-err>')
f.write('</testcase>\n')
f.write('</testsuite>\n')
if failed:
print(
'\nThe signature check detected changes in the API by comparing it with previous signature files.'
)
print(
'To fix this restore the original API or regenerate the signature files with:'
)
print('mx sigtest --generate')
mx.abort('Signature error in ' + failed)
else:
print('OK.')
return 0
def _sigtest_generate(args, suite=None, projects=None):
"""run sigtest generator for Java projects with API"""
nonTestProjects = [p for p in mx.projects() if _should_test_project(p)]
if not nonTestProjects:
return 0
javaCompliance = max([p.javaCompliance for p in nonTestProjects])
for p in nonTestProjects:
sigtestlib = p.suite.getMxCompatibility().get_sigtest_jar()
sigtestResults = p.dir + os.sep + 'snapshot.sigtest'
jdk = mx.get_jdk(javaCompliance)
cp = mx.classpath(p, jdk=jdk)
cmd = [
'-cp',
mx._cygpathU2W(sigtestlib),
'com.sun.tdk.signaturetest.Setup',
'-BootCP',
'-Static',
'-FileName',
sigtestResults,
'-ClassPath',
cp,
]
if args.human:
cmd.append('-H')
infos = set()
packages = mx._find_packages(p, packageInfos=infos)
for pkg in packages:
cmd = cmd + ['-PackageWithoutSubpackages', pkg]
javapExe = jdk.javap
if not exists(javapExe):
mx.abort('The javap executable does not exist: ' + javapExe)
class OutputCapture:
def __init__(self):
self.data = ""
def __call__(self, data):
self.data += data
for pkg in infos:
oc = OutputCapture()
ignore = OutputCapture()
code = mx.run([
javapExe, '-private', '-verbose', '-classpath', cp,
pkg + '.package-info'
],
out=oc,
err=ignore,
nonZeroIsFatal=False)
if code == 0:
if oc.data.find('\nRuntimeVisibleAnnotations:\n'
) == -1 and oc.data.find(
'\nRuntimeInvisibleAnnotations:\n') == -1:
mx.abort(
'GR-22788: ecj generated an empty {}.package-info: rebuild with javac!'
.format(pkg))
exitcode = mx.run_java(cmd,
nonZeroIsFatal=False,
jdk=mx.get_jdk(javaCompliance))
if exitcode != 95:
mx.abort('Exit code was ' + str(exitcode) + ' while generating ' +
sigtestResults)
if not exists(sigtestResults):
mx.abort('Cannot generate ' + sigtestResults)
mx.log("Sigtest snapshot generated to " + sigtestResults)
return 0
def _run_tests(args, harness, vmLauncher, annotations, testfile, blacklist,
whitelist, regex, suite):
vmArgs, tests = mx.extract_VM_args(args)
for t in tests:
if t.startswith('-'):
mx.abort('VM option ' + t + ' must precede ' + tests[0])
# Dictionary from fully qualified class names to the project or distribution containing the class
candidates = _find_classes_by_annotated_methods(annotations, suite)
jdk = mx.get_jdk()
for p in mx.projects(opt_limit_to_suite=True):
if not p.isJavaProject():
continue
if suite and not p.suite == suite:
continue
if jdk.javaCompliance < p.javaCompliance:
continue
for c in _find_classes_with_annotations(p, None, annotations):
candidates[c] = p
classes = []
if len(tests) == 0:
classes = candidates.keys()
depsContainingTests = set(candidates.values())
else:
depsContainingTests = set()
found = False
if len(tests) == 1 and '#' in tests[0]:
words = tests[0].split('#')
if len(words) != 2:
mx.abort("Method specification is class#method: " + tests[0])
t, method = words
for c, p in candidates.iteritems():
# prefer exact matches first
if t == c:
found = True
classes.append(c)
depsContainingTests.add(p)
if not found:
for c, p in candidates.iteritems():
if t in c:
found = True
classes.append(c)
depsContainingTests.add(p)
if not found:
mx.log('warning: no tests matched by substring "' + t)
elif len(classes) != 1:
mx.abort('More than one test matches substring {0} {1}'.format(
t, classes))
classes = [c + "#" + method for c in classes]
else:
for t in tests:
if '#' in t:
mx.abort(
'Method specifications can only be used in a single test: '
+ t)
for c, p in candidates.iteritems():
if t in c:
found = True
classes.append(c)
depsContainingTests.add(p)
if not found:
mx.log('warning: no tests matched by substring "' + t)
unittestCp = mx.classpath(depsContainingTests, jdk=vmLauncher.jdk())
if blacklist:
classes = [
c for c in classes
if not any((glob.match(c) for glob in blacklist))
]
if whitelist:
classes = [
c for c in classes if any((glob.match(c) for glob in whitelist))
]
if regex:
classes = [c for c in classes if re.search(regex, c)]
if len(classes) != 0:
f_testfile = open(testfile, 'w')
for c in classes:
f_testfile.write(c + '\n')
f_testfile.close()
harness(unittestCp, vmLauncher, vmArgs)
def junit(args, harness, parser=None, jdk_default=None):
"""run Junit tests"""
suppliedParser = parser is not None
parser = parser if suppliedParser else ArgumentParser(prog='mx junit')
parser.add_argument('--tests',
action='store',
help='pattern to match test classes')
parser.add_argument('--J',
dest='vm_args',
action='append',
help='target VM arguments (e.g. --J @-dsa)',
metavar='@<args>')
parser.add_argument('--jdk', action='store', help='jdk to use')
if suppliedParser:
parser.add_argument('remainder', nargs=REMAINDER, metavar='...')
args = parser.parse_args(args)
vmArgs = ['-ea', '-esa']
if args.vm_args:
vmArgs = vmArgs + mx_fastr.split_j_args(args.vm_args)
testfile = os.environ.get('MX_TESTFILE', None)
if testfile is None:
(_, testfile) = tempfile.mkstemp(".testclasses", "mx")
os.close(_)
candidates = []
if args.jdk:
jdk = mx.get_jdk(tag=args.jdk)
if not jdk:
mx.abort("jdk '" + args.jdk + "' not found")
else:
if not jdk_default:
jdk = mx.get_jdk()
else:
jdk = jdk_default
for p in mx.projects(opt_limit_to_suite=True):
if not p.isJavaProject() or jdk.javaCompliance < p.javaCompliance:
continue
candidates += _find_classes_with_annotations(p, None, ['@Test']).keys()
tests = [] if args.tests is None else [
name for name in args.tests.split(',')
]
classes = []
if len(tests) == 0:
classes = candidates
else:
for t in tests:
found = False
for c in candidates:
if t in c:
found = True
classes.append(c)
if not found:
mx.warn('no tests matched by substring "' + t + '"')
vmArgs += mx.get_runtime_jvm_args([
pcp.name for pcp in mx.projects(opt_limit_to_suite=True)
if pcp.isJavaProject() and pcp.javaCompliance <= jdk.javaCompliance
],
jdk=jdk)
if len(classes) != 0:
if len(classes) == 1:
testClassArgs = ['--testclass', classes[0]]
else:
with open(testfile, 'w') as f:
for c in classes:
f.write(c + '\n')
testClassArgs = ['--testsfile', testfile]
junitArgs = ['com.oracle.truffle.r.test.FastRJUnitWrapper'
] + testClassArgs
rc = harness(args, vmArgs, jdk, junitArgs)
return rc
else:
return 0
