def get_module_deps(dist):
"""
Gets the JAR distributions and their constituent Java projects whose artifacts (i.e., class files and
resources) are the input to the Java module jar created by `make_java_module` for a given distribution.
:return: the set of `JARDistribution` objects and their constituent `JavaProject` transitive
dependencies denoted by the ``moduledeps`` attribute
"""
if dist.suite.getMxCompatibility().moduleDepsEqualDistDeps():
return dist.archived_deps()
if not hasattr(dist, '.module_deps'):
roots = getattr(dist, 'moduledeps', [])
if not roots:
return roots
for root in roots:
if not root.isJARDistribution():
mx.abort('moduledeps can (currently) only include JAR distributions: ' + str(root), context=dist)
moduledeps = []
def _visit(dep, edges):
if dep is not dist:
if dep.isJavaProject() or dep.isJARDistribution():
if dep not in moduledeps:
moduledeps.append(dep)
else:
mx.abort('modules can (currently) only include JAR distributions and Java projects: ' + str(dep), context=dist)
def _preVisit(dst, edge):
return not dst.isJreLibrary() and not dst.isJdkLibrary()
mx.walk_deps(roots, preVisit=_preVisit, visit=_visit)
setattr(dist, '.module_deps', moduledeps)
return getattr(dist, '.module_deps')
def maven_plugin_install(args):
# First install native-image-maven-plugin dependencies into local maven repository
deps = []
def visit(dep, edge):
if isinstance(dep, mx.Distribution):
deps.append(dep)
mx.walk_deps([mx.dependency('substratevm:SVM_DRIVER')],
visit=visit,
ignoredEdges=[mx.DEP_ANNOTATION_PROCESSOR, mx.DEP_BUILD])
svmVersion = '{0}-SNAPSHOT'.format(suite.vc.parent(suite.vc_dir))
mx.maven_deploy([
'--version-string', svmVersion, '--suppress-javadoc',
'--all-distributions', '--validate=none', '--all-suites',
'--skip-existing', '--only', ','.join(dep.qualifiedName()
for dep in deps)
])
deploy_native_image_maven_plugin(svmVersion)
success_message = [
'',
'Use the following plugin snippet to enable native-image building for your maven project:',
'',
'<plugin>',
' <groupId>com.oracle.substratevm</groupId>',
' <artifactId>native-image-maven-plugin</artifactId>',
' <version>' + svmVersion + '</version>',
' <executions>',
' <execution>',
' <goals>',
' <goal>native-image</goal>',
' </goals>',
' <phase>package</phase>',
' </execution>',
' </executions>',
'</plugin>',
'',
]
mx.log('\n'.join(success_message))
def maven_plugin_install(args):
# First install native-image-maven-plugin dependencies into local maven repository
deps = []
def visit(dep, edge):
if isinstance(dep, mx.Distribution):
deps.append(dep)
mx.walk_deps([mx.dependency('substratevm:SVM_DRIVER')], visit=visit, ignoredEdges=[mx.DEP_ANNOTATION_PROCESSOR, mx.DEP_BUILD])
svmVersion = '{0}-SNAPSHOT'.format(suite.vc.parent(suite.vc_dir))
mx.maven_deploy([
'--version-string', svmVersion,
'--suppress-javadoc',
'--all-distributions',
'--validate=none',
'--all-suites',
'--skip-existing',
'--only', ','.join(dep.qualifiedName() for dep in deps)
])
deploy_native_image_maven_plugin(svmVersion)
success_message = [
'',
'Use the following plugin snippet to enable native-image building for your maven project:',
'',
'<plugin>',
' <groupId>com.oracle.substratevm</groupId>',
' <artifactId>native-image-maven-plugin</artifactId>',
' <version>' + svmVersion + '</version>',
' <executions>',
' <execution>',
' <goals>',
' <goal>native-image</goal>',
' </goals>',
' <phase>package</phase>',
' </execution>',
' </executions>',
'</plugin>',
'',
]
mx.log('\n'.join(success_message))
def sonarqube_upload(args):
"""run SonarQube scanner and upload JaCoCo results"""
sonarqube_cli = mx.library("SONARSCANNER_CLI_4_2_0_1873", True)
parser = ArgumentParser(prog='mx sonarqube-upload')
parser.add_argument('--exclude-generated', action='store_true', help='Exclude generated source files')
parser.add_argument('--skip-coverage', action='store_true', default=False, help='Do not upload coverage reports')
args, sonar_args = mx.extract_VM_args(args, useDoubleDash=True, defaultAllVMArgs=True)
args, other_args = parser.parse_known_args(args)
java_props, other_args = _parse_java_properties(other_args)
def _check_required_prop(prop):
if prop not in java_props:
mx.abort("Required property '{prop}' not present. (Format is '-D{prop}=<value>')".format(prop=prop))
_check_required_prop('sonar.projectKey')
_check_required_prop('sonar.host.url')
basedir = mx.primary_suite().dir
# collect excluded projects
excludes, includes = _jacoco_excludes_includes_projects(limit_to_primary=True)
# collect excluded classes
exclude_classes = _jacoco_exclude_classes(includes)
java_bin = []
java_src = []
java_libs = []
def _visit_deps(dep, edge):
if dep.isJARDistribution() or dep.isLibrary():
java_libs.append(dep.classpath_repr())
mx.walk_deps(includes, visit=_visit_deps)
# collect all sources and binaries -- do exclusion later
for p in includes:
java_src.extend(p.source_dirs())
if not args.exclude_generated:
gen_dir = p.source_gen_dir()
if os.path.exists(gen_dir):
java_src.append(gen_dir)
java_bin.append(p.output_dir())
java_src = [os.path.relpath(s, basedir) for s in java_src]
java_bin = [os.path.relpath(b, basedir) for b in java_bin]
# Overlayed sources and classes must be excluded
jdk_compliance = mx.get_jdk().javaCompliance
overlayed_sources = []
overlayed_classfiles = {}
for p in includes:
if hasattr(p, "multiReleaseJarVersion") and jdk_compliance not in p.javaCompliance: # JDK9+ overlays
for srcDir in p.source_dirs():
for root, _, files in os.walk(srcDir):
for name in files:
if name.endswith('.java') and name != 'package-info.java':
overlayed_sources.append(join(os.path.relpath(root, basedir), name))
elif hasattr(p, "overlayTarget"): # JDK8 overlays
target = mx.project(p.overlayTarget)
overlay_sources = []
for srcDir in p.source_dirs():
for root, _, files in os.walk(srcDir):
for name in files:
if name.endswith('.java') and name != 'package-info.java':
overlay_sources.append(join(os.path.relpath(root, srcDir), name))
print(p, target, overlay_sources)
for srcDir in target.source_dirs():
for root, _, files in os.walk(srcDir):
for name in files:
if name.endswith('.java') and name != 'package-info.java':
s = join(os.path.relpath(root, srcDir), name)
if s in overlay_sources:
overlayed = join(os.path.relpath(root, basedir), name)
overlayed_sources.append(overlayed)
for s in overlay_sources:
classfile = join(os.path.relpath(target.output_dir(), basedir), s[:-len('java')] + 'class')
with open(classfile, 'rb') as fp:
overlayed_classfiles[classfile] = fp.read()
exclude_dirs = []
for p in excludes:
exclude_dirs.extend(p.source_dirs())
exclude_dirs.append(p.source_gen_dir())
javaCompliance = max([p.javaCompliance for p in includes]) if includes else mx.JavaCompliance('1.7')
jacoco_exec = get_jacoco_dest_file()
if not os.path.exists(jacoco_exec) and not args.skip_coverage:
mx.abort('No JaCoCo report file found: ' + jacoco_exec)
def _add_default_prop(key, value):
if key not in java_props:
java_props[key] = value
# default properties
_add_default_prop('sonar.java.source', str(javaCompliance))
_add_default_prop('sonar.projectBaseDir', basedir)
if not args.skip_coverage:
_add_default_prop('sonar.jacoco.reportPaths', jacoco_exec)
_add_default_prop('sonar.sources', ','.join(java_src))
_add_default_prop('sonar.java.binaries', ','.join(java_bin))
_add_default_prop('sonar.java.libraries', ','.join(java_libs))
exclude_patterns = [os.path.relpath(e, basedir) + '**' for e in exclude_dirs] + \
overlayed_sources + \
list(set([os.path.relpath(match[0], basedir) for _, match in exclude_classes.items()]))
if exclude_patterns:
_add_default_prop('sonar.exclusions', ','.join(exclude_patterns))
_add_default_prop('sonar.coverage.exclusions', ','.join(exclude_patterns))
_add_default_prop('sonar.verbose', 'true' if mx._opts.verbose else 'false')
with tempfile.NamedTemporaryFile(suffix="-sonarqube.properties", mode="w+") as fp:
# prepare properties file
fp.writelines(('{}={}\n'.format(k, v) for k, v in java_props.items()))
fp.flush()
# Since there's no options to exclude individual classes,
# we temporarily delete the overlayed class files instead.
for classfile in overlayed_classfiles:
os.remove(classfile)
try:
# run sonarqube cli
java_args = other_args + ['-Dproject.settings=' + fp.name, '-jar', sonarqube_cli.get_path(True)] + sonar_args
exit_code = mx.run_java(java_args, nonZeroIsFatal=False)
finally:
# Restore temporarily deleted class files
for classfile, data in overlayed_classfiles.items():
with open(classfile, 'wb') as cf:
cf.write(data)
if exit_code != 0:
fp.seek(0)
mx.abort('SonarQube scanner terminated with non-zero exit code: {}\n Properties file:\n{}'.format(
exit_code, ''.join((' ' + l for l in fp.readlines()))))
def coverage_upload(args):
parser = ArgumentParser(prog='mx coverage-upload')
parser.add_argument('--upload-url', required=False, default=mx.get_env('COVERAGE_UPLOAD_URL'), help='Format is like rsync: user@host:/directory')
parser.add_argument('--build-name', required=False, default=mx.get_env('BUILD_NAME'))
parser.add_argument('--build-url', required=False, default=mx.get_env('BUILD_URL'))
parser.add_argument('--build-number', required=False, default=mx.get_env('BUILD_NUMBER'))
args, other_args = parser.parse_known_args(args)
if not args.upload_url:
parser.print_help()
return
remote_host, remote_basedir = args.upload_url.split(':')
if not remote_host:
mx.abort('Cannot determine remote host from {}'.format(args.upload_url))
primary = mx.primary_suite()
info = primary.vc.parent_info(primary.dir)
rev = primary.vc.parent(primary.dir)
if len(remote_basedir) > 0 and not remote_basedir.endswith('/'):
remote_basedir += '/'
remote_dir = '{}_{}_{}'.format(primary.name, datetime.datetime.fromtimestamp(info['author-ts']).strftime('%Y-%m-%d_%H_%M'), rev[:7])
if args.build_name:
remote_dir += '_' + args.build_name
if args.build_number:
remote_dir += '_' + args.build_number
upload_dir = remote_basedir + remote_dir
includes, excludes = _jacocoreport(['--omit-excluded'] + other_args)
# Upload jar+sources
coverage_sources = 'java_sources.tar.gz'
coverage_binaries = 'java_binaries.tar.gz'
with mx.Archiver(os.path.realpath(coverage_sources), kind='tgz') as sources, mx.Archiver(os.path.realpath(coverage_binaries), kind='tgz') as binaries:
def _visit_deps(dep, edge):
if dep.isJavaProject() and not dep.is_test_project():
binaries.zf.add(dep.output_dir(), dep.name)
for d in dep.source_dirs():
sources.zf.add(d, dep.name)
if os.path.exists(dep.source_gen_dir()):
sources.zf.add(dep.source_gen_dir(), dep.name)
mx.walk_deps(mx.projects(), visit=_visit_deps)
files = [get_jacoco_dest_file(), 'coverage', coverage_sources, coverage_binaries]
print("Syncing {} to {}:{}".format(" ".join(files), remote_host, upload_dir))
mx.run([
'bash',
'-c',
r'tar -czf - {files} | ssh {remote} bash -c \'"mkdir -p {remotedir} && cd {remotedir} && cat | tar -x{verbose}z && chmod -R 755 ."\''
.format(
files=" ".join(files),
remote=remote_host,
remotedir=upload_dir,
verbose='v' if mx._opts.verbose else '')
])
def upload_string(content, path):
mx.run(['ssh', remote_host, 'bash', '-c', 'cat > "' + path + '"'], stdin=content)
upload_string(json.dumps({
'timestamp': time.time(),
'suite': primary.name,
'revision': rev,
'directory': remote_dir,
'build_name': args.build_name,
'build_url': args.build_url,
'jdk_version': str(mx.get_jdk().version),
'build_number': args.build_number,
'primary_info': info,
'excludes': [str(e) for e in excludes],
'includes': [str(i) for i in includes]}), upload_dir + '/description.json')
mx.run(['ssh', remote_host, 'bash', '-c', r'"(echo \[; for i in {remote_basedir}/*/description.json; do if \[ -s \$i \];then cat \$i; echo ,; fi done; echo null\]) > {remote_basedir}/index.json"'.format(remote_basedir=remote_basedir)])
upload_string("""<html>
<script language="javascript">
function urlChange(url) {
if (url.pathname !== "blank") {
window.history.replaceState(null, null, url.pathname.replace("/coverage_upload/", "/coverage_upload/#"))
}
}
</script>
<frameset rows="40,*">
<frame id="navigation" src="navigation.html"/>
<frame id="content" src="" onload="urlChange(this.contentWindow.location);" />
</frameset>
</html>""", remote_basedir + '/index.html')
js_library_url = rewriteurl("https://ajax.googleapis.com/ajax/libs/angularjs/1.7.7/angular.js")
upload_string(r"""<html>
<head>
<script src="%js_library_url"></script>
<script language="javascript">
var App = angular.module('myApp', [])
.controller('IndexCtrl', function IndexCtrl($scope, $http) {
var hash = parent.window.location.hash;
if(hash) {
hash = hash.substring(1, hash.length); // remove leading hash
}
$http.get('index.json').then(function(response, status) {
var data = response.data.filter(x => x != null);
/*
#GR-17399
Filter build that are unique per suite with revision as key and merge builds.
*/
data = data
.filter(x => !x.hasOwnProperty('merge'))
.filter( // filter builds that are unique per suite with revision as key
x => !data
.filter(z => x != z && x.suite == z.suite) // exclude self build and build for other suites.
.map(z => z.revision) // map from array of build to array of revision
.includes(x.revision) // check if revision of x is index data.
).concat(data.filter(x => x.hasOwnProperty('merge'))); // concat unique build with merged build.
data.sort((l,r) => r.timestamp - l.timestamp);
if(data.length > 0) {
var startdir;
if(hash) {
startdir = data.find(build => hash.includes(build.directory));
startdir.hash = hash;
}
if(!startdir) {
startdir = data[0];
}
$scope.directory = startdir;
}
$scope.data = data;
});
$scope.$watch('directory', (dir, olddir) => {
if(dir) {
var content = parent.document.getElementById("content");
var contentDocument = content.contentDocument || content.contentWindow.document;
var newpath;
if(olddir && olddir.suite === dir.suite) {
newpath = contentDocument.location.href.replace(olddir.directory, dir.directory);
} else {
newpath = dir.hasOwnProperty('hash') ? hash : dir.directory + "/coverage/";
}
contentDocument.location.href = newpath;
parent.window.history.replaceState(undefined, undefined, "#" + newpath.replace(/^.+coverage_upload\//, ""));
}
});
$scope.step = (i) => $scope.directory = $scope.data[$scope.data.indexOf($scope.directory)+i];
});
function copy(url) {
var content = parent.document.getElementById("content");
var contentDocument = content.contentDocument || content.contentWindow.document;
var copyText = document.getElementById("copy");
copyText.value = contentDocument.location.href.replace("coverage_upload/", "coverage_upload/#");
copyText.select();
document.execCommand("copy");
}
</script>
</head>
<body ng-app="myApp" ng-controller="IndexCtrl">
<button ng-click="step(1)" ng-disabled="data.indexOf(directory) >= data.length-1"><<</button>
<button ng-click="step(-1)" ng-disabled="data.indexOf(directory) <= 0">>></button>
<select ng-model="directory" ng-options="(i.primary_info['author-ts']*1000|date:'yy-MM-dd hh:mm') + ' ' + i.build_name + ' ' + i.revision.substr(0,8) group by i.suite for i in data"></select>
<a href="{{directory.build_url}}" ng-if="directory.build_url" target="_blank">Build</a> Commit: {{directory.revision.substr(0,5)}}: {{directory.primary_info.description}}
<input type="text" style="opacity: 0;width: 20;" id="copy" />
<button style="float: right;" onclick="copy(window.location);">Share url</button>
</body>
</html>""".replace("%js_library_url", js_library_url), remote_basedir + '/navigation.html')
def do_build_makefile(mf, selectedDists):
jdk = mx.get_jdk()
bootClassPath = jdk.bootclasspath(filtered=False)
bootClassPath = bootClassPath.replace(os.path.realpath(jdk.home), "$(ABS_BOOTDIR)")
jdkBootClassPathVariableName = "JDK_BOOTCLASSPATH"
mf.add_definition("""# This Makefile is generated automatically, do not edit
TARGET=.
# Bootstrap JDK to be used (for javac and jar)
ABS_BOOTDIR=
JAVAC=$(ABS_BOOTDIR)/bin/javac -g -target """ + str(jdk.javaCompliance) + """
JAR=$(ABS_BOOTDIR)/bin/jar
HS_COMMON_SRC=.
# Directories, where the generated property-files reside within the JAR files
PROVIDERS_INF=/META-INF/jvmci.providers
SERVICES_INF=/META-INF/jvmci.services
OPTIONS_INF=/META-INF/jvmci.options
JARS = $(foreach dist,$(DISTRIBUTIONS),$($(dist)_JAR))
ifeq ($(ABS_BOOTDIR),)
$(error Variable ABS_BOOTDIR must be set to a JDK installation.)
endif
ifeq ($(MAKE_VERBOSE),)
QUIETLY=@
endif
# Required to construct a whitespace for use with subst
space :=
space +=
# Takes the provider files created by ServiceProviderProcessor (the processor
# for the @ServiceProvider annotation) and merges them into a single file.
# Arguments:
# 1: directory with contents of the JAR file
define process_providers
$(eval providers := $(1)/$(PROVIDERS_INF))
$(eval services := $(1)/$(SERVICES_INF))
$(QUIETLY) test -d $(services) || mkdir -p $(services)
$(QUIETLY) test ! -d $(providers) || (cd $(providers) && for i in $$(ls); do c=$$(cat $$i); echo $$i >> $(abspath $(services))/$$c; rm $$i; done)
@# Since all projects are built together with one javac call we cannot determine
@# which project contains HotSpotVMConfig.inline.hpp so we hardcode it.
$(eval vmconfig := $(1)/hotspot/HotSpotVMConfig.inline.hpp)
$(eval vmconfigDest := $(HS_COMMON_SRC)/../mxbuild/jvmci/jdk.vm.ci.hotspot/src_gen/hotspot)
$(QUIETLY) test ! -f $(vmconfig) || (mkdir -p $(vmconfigDest) && cp $(vmconfig) $(vmconfigDest))
endef
# Finds the *_OptionsDescriptors classes created by OptionProcessor (the processor for the @Option annotation)
# and appends their names to services/jdk.vm.ci.options.OptionDescriptors.
# Arguments:
# 1: directory with contents of the JAR file
define process_options
$(eval services := $(1)/META-INF/services)
$(QUIETLY) test -d $(services) || mkdir -p $(services)
$(eval optionDescriptors := $(1)/META-INF/services/jdk.vm.ci.options.OptionDescriptors)
$(QUIETLY) cd $(1) && for i in $$(find . -name '*_OptionDescriptors.class' 2>/dev/null); do echo $${i} | sed 's:\\./\\(.*\\)\\.class:\\1:g' | tr '/' '.' >> $(abspath $(optionDescriptors)); done
endef
# Extracts META-INF/jvmci.services from a JAR file into a given directory
# Arguments:
# 1: JAR file to extract
# 2: target directory (which already exists)
define extract
$(eval TMP := $(shell mktemp -d $(TARGET)/tmp_XXXXX))
$(QUIETLY) cd $(TMP) && $(JAR) xf $(abspath $(1)) && \\
(test ! -d .$(SERVICES_INF) || cp -r .$(SERVICES_INF) $(abspath $(2)));
$(QUIETLY) rm -r $(TMP);
$(QUIETLY) cp $(1) $(2)
endef
# Calls $(JAVAC) with the boot class path $(JDK_BOOTCLASSPATH) and sources taken from the automatic variable $^
# Arguments:
# 1: processorpath
# 2: classpath
# 3: resources to copy
# 4: target JAR file
define build_and_jar
$(info Building $(4))
$(eval TMP := $(shell mkdir -p $(TARGET) && mktemp -d $(TARGET)/tmp_XXXXX))
$(QUIETLY) $(JAVAC) -d $(TMP) -processorpath :$(1) -bootclasspath $(JDK_BOOTCLASSPATH) -cp :$(2) $(filter %.java,$^)
$(QUIETLY) test "$(3)" = "" || cp -r $(3) $(TMP)
$(QUIETLY) $(call process_options,$(TMP))
$(QUIETLY) $(call process_providers,$(TMP))
$(QUIETLY) mkdir -p $(shell dirname $(4))
$(QUIETLY) $(JAR) -0cf $(4) -C $(TMP) .
$(QUIETLY) rm -r $(TMP)
endef
# Verifies that make/defs.make contains an appropriate line for each JVMCI service
# and that only existing JVMCI services are exported.
# Arguments:
# 1: list of service files
# 2: variable name for directory of service files
define verify_defs_make
$(eval defs := make/defs.make)
$(eval uncondPattern := EXPORT_LIST += $$$$($(2))/)
$(eval condPattern := CONDITIONAL_EXPORT_LIST += $$$$($(2))/)
$(eval unconditionalExports := $(shell grep '^EXPORT_LIST += $$($2)' make/defs.make | sed 's:.*($(2))/::g'))
$(eval conditionalExports := $(shell grep '^CONDITIONAL_EXPORT_LIST += $$($2)' make/defs.make | sed 's:.*($(2))/::g'))
$(eval allExports := $(unconditionalExports) $(conditionalExports))
$(foreach file,$(1),$(if $(findstring $(file),$(allExports)), ,$(error "Line matching '$(uncondPattern)$(file)' or '$(condPattern)$(file)' not found in $(defs)")))
$(foreach export,$(unconditionalExports),$(if $(findstring $(export),$(1)), ,$(error "The line '$(uncondPattern)$(export)' should not be in $(defs)")))
endef
all: default
\t$(info Put $(EXPORTED_FILES) into SHARED_DIR $(SHARED_DIR))
\t$(shell mkdir -p $(SHARED_DIR))
\t$(foreach export,$(EXPORTED_FILES),$(call extract,$(export),$(SHARED_DIR)))
export: all
\t$(call verify_defs_make,$(notdir $(wildcard $(SHARED_DIR)/jvmci.services/*)),EXPORT_JRE_LIB_JVMCI_SERVICES_DIR)
.PHONY: export
clean:
\t$(QUIETLY) rm $(JARS) 2> /dev/null || true
\t$(QUIETLY) rmdir -p $(dir $(JARS)) 2> /dev/null || true
.PHONY: export clean
""")
assert selectedDists
selectedDists = [mx.dependency(s) for s in selectedDists]
dists = []
def _visit(dep, edge):
if dep.isDistribution():
dists.append(dep)
mx.walk_deps(roots=selectedDists, visit=_visit, ignoredEdges=[mx.DEP_EXCLUDED])
mf.add_definition(jdkBootClassPathVariableName + " = " + bootClassPath)
for dist in dists: make_dist_rule(dist, mf)
mf.add_definition("DISTRIBUTIONS = " + ' '.join([dist.name for dist in dists]))
mf.add_rule("default: $({}_JAR)\n.PHONY: default\n".format("_JAR) $(".join([d.name for d in selectedDists])))
return True
