def _create_jdk_bundle(jdkBuildDir, debugLevel, jdkImageDir):
"""
Creates a tar.gz JDK archive, an accompanying tar.gz.sha1 file with its
SHA1 signature plus symlinks to the archive for non-canonical architecture names.
"""
arches = _get_jdk_bundle_arches()
jdkTgzPath = join(_suite.get_output_root(), 'jdk-bundles', 'jdk9-{}-{}-{}.tar.gz'.format(debugLevel, _get_openjdk_os(), arches[0]))
with mx.Archiver(jdkTgzPath, kind='tgz') as arc:
mx.log('Creating ' + jdkTgzPath)
for root, _, filenames in os.walk(jdkImageDir):
for name in filenames:
f = join(root, name)
arcname = 'jdk1.9.0/' + os.path.relpath(f, jdkImageDir)
arc.zf.add(name=f, arcname=arcname, recursive=False)
with open(jdkTgzPath + '.sha1', 'w') as fp:
mx.log('Creating ' + jdkTgzPath + '.sha1')
fp.write(mx.sha1OfFile(jdkTgzPath))
def _create_link(source, link_name):
if exists(link_name):
os.remove(link_name)
mx.log('Creating ' + link_name + ' -> ' + source)
os.symlink(source, link_name)
for arch in arches[1:]:
link_name = join(_suite.get_output_root(), 'jdk-bundles', 'jdk9-{}-{}-{}.tar.gz'.format(debugLevel, _get_openjdk_os(), arch))
jdkTgzName = os.path.basename(jdkTgzPath)
_create_link(jdkTgzName, link_name)
_create_link(jdkTgzName + '.sha1', link_name + '.sha1')
def _create_jdk_bundle(jdkBuildDir):
"""
Creates a tar.gz JDK archive, an accompanying tar.gz.sha1 file with its
SHA1 signature plus symlinks to the archive for non-canonical architecture names.
"""
jdkImageDir = join(jdkBuildDir, 'images', 'jdk')
arches = _get_jdk_bundle_arches()
jdkTgzPath = join(_suite.get_output_root(), 'jdk-bundles',
'jdk9-{}-{}.tar.gz'.format(_get_openjdk_os(), arches[0]))
with mx.Archiver(jdkTgzPath, kind='tgz') as arc:
mx.log('Creating ' + jdkTgzPath)
for root, _, filenames in os.walk(jdkImageDir):
for name in filenames:
f = join(root, name)
arcname = 'jdk1.9.0/' + os.path.relpath(f, jdkImageDir)
arc.zf.add(name=f, arcname=arcname, recursive=False)
# The OpenJDK build creates an empty cacerts file so grab one from
# the default JDK which is assumed to be an OracleJDK
cacerts = join(
mx.get_jdk(tag='default').home, 'jre', 'lib', 'security',
'cacerts')
arc.zf.add(name=cacerts, arcname='jdk1.9.0/lib/security/cacerts')
with open(jdkTgzPath + '.sha1', 'w') as fp:
mx.log('Creating ' + jdkTgzPath + '.sha1')
fp.write(mx.sha1OfFile(jdkTgzPath))
def _create_link(source, link_name):
if exists(link_name):
os.remove(link_name)
mx.log('Creating ' + link_name + ' -> ' + source)
os.symlink(source, link_name)
for arch in arches[1:]:
link_name = join(_suite.get_output_root(), 'jdk-bundles',
'jdk9-{}-{}.tar.gz'.format(_get_openjdk_os(), arch))
jdkTgzName = os.path.basename(jdkTgzPath)
_create_link(jdkTgzName, link_name)
_create_link(jdkTgzName + '.sha1', link_name + '.sha1')
def _update_JVMCI_library():
"""
Updates the "path" and "sha1" attributes of the "JVMCI" library to
refer to a jvmci.jar created from the JVMCI classes in JDK9.
"""
suiteDict = _suite.suiteDict
jvmciLib = suiteDict['libraries']['JVMCI']
d = join(_suite.get_output_root(), abspath(_jdk.home)[1:])
path = join(d, 'jvmci.jar')
explodedModule = join(_jdk.home, 'modules', 'jdk.vm.ci')
if exists(explodedModule):
jarInputs = {}
newestJarInput = None
for root, _, files in os.walk(explodedModule):
relpath = root[len(explodedModule) + 1:]
for f in files:
arcname = join(relpath, f).replace(os.sep, '/')
jarInput = join(root, f)
jarInputs[arcname] = jarInput
t = mx.TimeStampFile(jarInput)
if newestJarInput is None or t.isNewerThan(newestJarInput):
newestJarInput = t
if not exists(path) or newestJarInput.isNewerThan(path):
with mx.Archiver(path, kind='zip') as arc:
for arcname, jarInput in jarInputs.iteritems():
with open(jarInput, 'rb') as fp:
contents = fp.read()
arc.zf.writestr(arcname, contents)
else:
# Use the jdk.internal.jimage utility since it's the only way
# to partially read .jimage files as the JDK9 jimage tool
# does not support partial extraction.
bootmodules = join(_jdk.home, 'lib', 'modules', 'bootmodules.jimage')
if not exists(bootmodules):
mx.abort('Could not find JVMCI classes at ' + bootmodules + ' or ' + explodedModule)
if not exists(path) or mx.TimeStampFile(bootmodules).isNewerThan(path):
mx.ensure_dir_exists(d)
javaSource = join(d, 'ExtractJVMCI.java')
with open(javaSource, 'w') as fp:
print >> fp, """import java.io.FileOutputStream;
import java.util.jar.JarEntry;
import java.util.jar.JarOutputStream;
import jdk.internal.jimage.BasicImageReader;
public class ExtractJVMCI {
public static void main(String[] args) throws Exception {
BasicImageReader image = BasicImageReader.open(args[0]);
String[] names = image.getEntryNames();
if (names.length == 0) {
return;
}
try (JarOutputStream jos = new JarOutputStream(new FileOutputStream(args[1]))) {
for (String name : names) {
if (name.startsWith("/jdk.vm.ci/")) {
String ename = name.substring("/jdk.vm.ci/".length());
JarEntry je = new JarEntry(ename);
jos.putNextEntry(je);
jos.write(image.getResource(name));
jos.closeEntry();
}
}
}
}
}
"""
mx.run([_jdk.javac, '-d', d, javaSource])
mx.run([_jdk.java, '-cp', d, 'ExtractJVMCI', bootmodules, path])
if not exists(path):
mx.abort('Could not find the JVMCI classes in ' + bootmodules)
jvmciLib['path'] = path
jvmciLib['sha1'] = mx.sha1OfFile(path)
def build(self):
if os.environ.has_key('FASTR_NO_RELEASE'):
mx.log('FastR: not updating release project')
return
# copy the release directories
output_dir = self.subject.dir
fastr_dir = mx_fastr._fastr_suite.dir
for d in ['bin', 'include', 'lib', 'library', 'etc', 'share', 'doc']:
target_dir = join(output_dir, d)
if os.path.exists(target_dir):
shutil.rmtree(target_dir)
shutil.copytree(join(fastr_dir, d), target_dir)
# copyrights
copyrights_dir = join(fastr_dir, 'mx.fastr', 'copyrights')
with open(join(output_dir, 'COPYRIGHT'), 'w') as outfile:
for copyright_file in os.listdir(copyrights_dir):
basename = os.path.basename(copyright_file)
if basename.endswith('copyright.star'):
with open(join(copyrights_dir, copyright_file)) as infile:
data = infile.read()
outfile.write(data)
# license/README
shutil.copy(join(fastr_dir, 'LICENSE'), output_dir)
shutil.copy(join(fastr_dir, 'README.md'), output_dir)
# canonicalize R_HOME_DIR in bin/R
bin_dir = join(output_dir, 'bin')
rcmd = join(bin_dir, 'R')
# R is the generic shell script (taken essentially verbatim from GNU R)
with open(rcmd) as f:
lines = f.readlines()
with open(rcmd, 'w') as f:
for line in lines:
if line.startswith('R_HOME_DIR='):
f.write('R_HOME_DIR="$(dirname $0)/.."\n')
# produces a canonical path
line = 'R_HOME_DIR="$(unset CDPATH && cd ${R_HOME_DIR} && pwd)"\n'
f.write(line)
# jar files for the launchers
jars_dir = join(bin_dir, 'fastr_jars')
if not os.path.exists(jars_dir):
os.mkdir(jars_dir)
fastr_classfiles = dict()
# visitor to collect/copy all the classes/jar files needed by the launchers
def dep_visit(dep, edge):
if isinstance(dep, mx.JARDistribution):
shutil.copy(join(dep.suite.dir, dep.path), jars_dir)
elif isinstance(dep, mx.Library):
jar_name = dep.name.lower() + '.jar'
shutil.copyfile(join(dep.suite.dir, dep.path), join(jars_dir, jar_name))
elif isinstance(dep, mx.JavaProject):
if 'com.oracle.truffle.r' in dep.name:
classfiles_dir = dep.output_dir()
for root, _, classfiles in os.walk(classfiles_dir):
for classfile in classfiles:
fastr_classfiles[os.path.relpath(join(root, classfile), classfiles_dir)] = join(root, classfile)
self.subject.walk_deps(visit=dep_visit)
# create the fastr.jar file
with mx.Archiver(join(jars_dir, 'fastr.jar')) as arc:
arc.zf.writestr("META-INF/MANIFEST.MF", "Manifest-Version: 1.0\n")
for arcname, path in fastr_classfiles.iteritems():
with open(path, 'r') as f:
contents = f.read()
arc.zf.writestr(arcname, contents)
# create the classpath string
classpath = []
for _, _, jars in os.walk(jars_dir):
for jar in jars:
classpath.append(join("$R_HOME/bin/fastr_jars", jar))
classpath_string = ":".join(classpath)
# replace the mx exec scripts with native Java launchers, setting the classpath from above
bin_exec_dir = join(bin_dir, 'exec')
r_launcher = join(self.subject.dir, 'src', 'R_launcher')
template_dict = {'CLASSPATH': classpath_string}
self._template(r_launcher, join(bin_exec_dir, 'R'), template_dict)
shutil.rmtree(join(bin_dir, 'execRextras'))
rscript_launcher = join(self.subject.dir, 'src', 'Rscript_launcher')
self._template(rscript_launcher, join(bin_dir, 'Rscript'), template_dict)
def coverage_upload(args):
parser = ArgumentParser(prog='mx coverage-upload')
parser.add_argument('--upload-url', required=False, default=mx.get_env('COVERAGE_UPLOAD_URL'), help='Format is like rsync: user@host:/directory')
parser.add_argument('--build-name', required=False, default=mx.get_env('BUILD_NAME'))
parser.add_argument('--build-url', required=False, default=mx.get_env('BUILD_URL'))
parser.add_argument('--build-number', required=False, default=mx.get_env('BUILD_NUMBER'))
args, other_args = parser.parse_known_args(args)
if not args.upload_url:
parser.print_help()
return
remote_host, remote_basedir = args.upload_url.split(':')
if not remote_host:
mx.abort('Cannot determine remote host from {}'.format(args.upload_url))
primary = mx.primary_suite()
info = primary.vc.parent_info(primary.dir)
rev = primary.vc.parent(primary.dir)
if len(remote_basedir) > 0 and not remote_basedir.endswith('/'):
remote_basedir += '/'
remote_dir = '{}_{}_{}'.format(primary.name, datetime.datetime.fromtimestamp(info['author-ts']).strftime('%Y-%m-%d_%H_%M'), rev[:7])
if args.build_name:
remote_dir += '_' + args.build_name
if args.build_number:
remote_dir += '_' + args.build_number
upload_dir = remote_basedir + remote_dir
includes, excludes = _jacocoreport(['--omit-excluded'] + other_args)
# Upload jar+sources
coverage_sources = 'java_sources.tar.gz'
coverage_binaries = 'java_binaries.tar.gz'
with mx.Archiver(os.path.realpath(coverage_sources), kind='tgz') as sources, mx.Archiver(os.path.realpath(coverage_binaries), kind='tgz') as binaries:
def _visit_deps(dep, edge):
if dep.isJavaProject() and not dep.is_test_project():
binaries.zf.add(dep.output_dir(), dep.name)
for d in dep.source_dirs():
sources.zf.add(d, dep.name)
if os.path.exists(dep.source_gen_dir()):
sources.zf.add(dep.source_gen_dir(), dep.name)
mx.walk_deps(mx.projects(), visit=_visit_deps)
files = [get_jacoco_dest_file(), 'coverage', coverage_sources, coverage_binaries]
print("Syncing {} to {}:{}".format(" ".join(files), remote_host, upload_dir))
mx.run([
'bash',
'-c',
r'tar -czf - {files} | ssh {remote} bash -c \'"mkdir -p {remotedir} && cd {remotedir} && cat | tar -x{verbose}z && chmod -R 755 ."\''
.format(
files=" ".join(files),
remote=remote_host,
remotedir=upload_dir,
verbose='v' if mx._opts.verbose else '')
])
def upload_string(content, path):
mx.run(['ssh', remote_host, 'bash', '-c', 'cat > "' + path + '"'], stdin=content)
upload_string(json.dumps({
'timestamp': time.time(),
'suite': primary.name,
'revision': rev,
'directory': remote_dir,
'build_name': args.build_name,
'build_url': args.build_url,
'jdk_version': str(mx.get_jdk().version),
'build_number': args.build_number,
'primary_info': info,
'excludes': [str(e) for e in excludes],
'includes': [str(i) for i in includes]}), upload_dir + '/description.json')
mx.run(['ssh', remote_host, 'bash', '-c', r'"(echo \[; for i in {remote_basedir}/*/description.json; do if \[ -s \$i \];then cat \$i; echo ,; fi done; echo null\]) > {remote_basedir}/index.json"'.format(remote_basedir=remote_basedir)])
upload_string("""<html>
<script language="javascript">
function urlChange(url) {
if (url.pathname !== "blank") {
window.history.replaceState(null, null, url.pathname.replace("/coverage_upload/", "/coverage_upload/#"))
}
}
</script>
<frameset rows="40,*">
<frame id="navigation" src="navigation.html"/>
<frame id="content" src="" onload="urlChange(this.contentWindow.location);" />
</frameset>
</html>""", remote_basedir + '/index.html')
js_library_url = rewriteurl("https://ajax.googleapis.com/ajax/libs/angularjs/1.7.7/angular.js")
upload_string(r"""<html>
<head>
<script src="%js_library_url"></script>
<script language="javascript">
var App = angular.module('myApp', [])
.controller('IndexCtrl', function IndexCtrl($scope, $http) {
var hash = parent.window.location.hash;
if(hash) {
hash = hash.substring(1, hash.length); // remove leading hash
}
$http.get('index.json').then(function(response, status) {
var data = response.data.filter(x => x != null);
/*
#GR-17399
Filter build that are unique per suite with revision as key and merge builds.
*/
data = data
.filter(x => !x.hasOwnProperty('merge'))
.filter( // filter builds that are unique per suite with revision as key
x => !data
.filter(z => x != z && x.suite == z.suite) // exclude self build and build for other suites.
.map(z => z.revision) // map from array of build to array of revision
.includes(x.revision) // check if revision of x is index data.
).concat(data.filter(x => x.hasOwnProperty('merge'))); // concat unique build with merged build.
data.sort((l,r) => r.timestamp - l.timestamp);
if(data.length > 0) {
var startdir;
if(hash) {
startdir = data.find(build => hash.includes(build.directory));
startdir.hash = hash;
}
if(!startdir) {
startdir = data[0];
}
$scope.directory = startdir;
}
$scope.data = data;
});
$scope.$watch('directory', (dir, olddir) => {
if(dir) {
var content = parent.document.getElementById("content");
var contentDocument = content.contentDocument || content.contentWindow.document;
var newpath;
if(olddir && olddir.suite === dir.suite) {
newpath = contentDocument.location.href.replace(olddir.directory, dir.directory);
} else {
newpath = dir.hasOwnProperty('hash') ? hash : dir.directory + "/coverage/";
}
contentDocument.location.href = newpath;
parent.window.history.replaceState(undefined, undefined, "#" + newpath.replace(/^.+coverage_upload\//, ""));
}
});
$scope.step = (i) => $scope.directory = $scope.data[$scope.data.indexOf($scope.directory)+i];
});
function copy(url) {
var content = parent.document.getElementById("content");
var contentDocument = content.contentDocument || content.contentWindow.document;
var copyText = document.getElementById("copy");
copyText.value = contentDocument.location.href.replace("coverage_upload/", "coverage_upload/#");
copyText.select();
document.execCommand("copy");
}
</script>
</head>
<body ng-app="myApp" ng-controller="IndexCtrl">
<button ng-click="step(1)" ng-disabled="data.indexOf(directory) >= data.length-1"><<</button>
<button ng-click="step(-1)" ng-disabled="data.indexOf(directory) <= 0">>></button>
<select ng-model="directory" ng-options="(i.primary_info['author-ts']*1000|date:'yy-MM-dd hh:mm') + ' ' + i.build_name + ' ' + i.revision.substr(0,8) group by i.suite for i in data"></select>
<a href="{{directory.build_url}}" ng-if="directory.build_url" target="_blank">Build</a> Commit: {{directory.revision.substr(0,5)}}: {{directory.primary_info.description}}
<input type="text" style="opacity: 0;width: 20;" id="copy" />
<button style="float: right;" onclick="copy(window.location);">Share url</button>
</body>
</html>""".replace("%js_library_url", js_library_url), remote_basedir + '/navigation.html')
def build(self):
if not os.environ.has_key('FASTR_RELEASE'):
mx.log('FastR: set FASTR_RELEASE to update release project')
return
# copy the release directories
output_dir = self.subject.dir
fastr_dir = mx_fastr._fastr_suite.dir
for d in ['bin', 'include', 'library', 'etc', 'share', 'doc']:
target_dir = join(output_dir, d)
if os.path.exists(target_dir):
shutil.rmtree(target_dir)
shutil.copytree(join(fastr_dir, d), target_dir)
lib_fastr_dir = join(fastr_dir, 'lib')
lib_output_dir = join(output_dir, 'lib')
if os.path.exists(lib_output_dir):
shutil.rmtree(lib_output_dir)
os.mkdir(lib_output_dir)
for f in os.listdir(lib_fastr_dir):
source_file = join(lib_fastr_dir, f)
target_file = join(lib_output_dir, f)
if f != '.DS_Store':
if os.path.islink(source_file):
os.symlink(os.readlink(source_file), target_file)
else:
shutil.copy(source_file, target_file)
# copyrights
copyrights_dir = join(fastr_dir, 'mx.fastr', 'copyrights')
with open(join(output_dir, 'COPYRIGHT'), 'w') as outfile:
for copyright_file in os.listdir(copyrights_dir):
basename = os.path.basename(copyright_file)
if basename.endswith('copyright.star'):
with open(join(copyrights_dir, copyright_file)) as infile:
data = infile.read()
outfile.write(data)
# license/README
shutil.copy(join(fastr_dir, 'LICENSE'), output_dir)
shutil.copy(join(fastr_dir, 'README.md'), output_dir)
# canonicalize R_HOME_DIR in bin/R
bin_dir = join(output_dir, 'bin')
rcmd = join(bin_dir, 'R')
# R is the generic shell script (taken essentially verbatim from GNU R)
with open(rcmd) as f:
lines = f.readlines()
with open(rcmd, 'w') as f:
for line in lines:
if line.startswith('R_HOME_DIR='):
f.write(
"""
source="${BASH_SOURCE[0]}"
while [ -h "$source" ] ; do
prev_source="$source"
source="$(readlink "$source")";
if [[ "$source" != /* ]]; then
# if the link was relative, it was relative to where it came from
dir="$( cd -P "$( dirname "$prev_source" )" && pwd )"
source="$dir/$source"
fi
done
r_bin="$( cd -P "$( dirname "$source" )" && pwd )"
R_HOME_DIR="$( dirname "$r_bin" )"
""")
elif line.strip() == "#!/bin/sh":
f.write("#!/usr/bin/env bash\n")
else:
f.write(line)
# jar files for the launchers
jars_dir = join(bin_dir, 'fastr_jars')
if not os.path.exists(jars_dir):
os.mkdir(jars_dir)
fastr_classfiles = dict()
# visitor to collect/copy all the classes/jar files needed by the launchers
def dep_visit(dep, edge):
if isinstance(dep, mx.JARDistribution):
shutil.copy(join(dep.suite.dir, dep.path), jars_dir)
elif isinstance(dep, mx.Library):
if not dep.name.lower() == 'jdk_tools':
jar_name = dep.name.lower() + '.jar'
shutil.copyfile(join(dep.suite.dir, dep.path), join(jars_dir, jar_name))
elif isinstance(dep, mx.JavaProject):
if 'com.oracle.truffle.r' in dep.name:
classfiles_dir = dep.output_dir()
for root, _, classfiles in os.walk(classfiles_dir):
for classfile in classfiles:
fastr_classfiles[os.path.relpath(join(root, classfile), classfiles_dir)] = join(root, classfile)
self.subject.walk_deps(visit=dep_visit)
# create the fastr.jar file
with mx.Archiver(join(jars_dir, 'fastr.jar')) as arc:
arc.zf.writestr("META-INF/MANIFEST.MF", "Manifest-Version: 1.0\n")
for arcname, path in fastr_classfiles.iteritems():
with open(path, 'r') as f:
contents = f.read()
arc.zf.writestr(arcname, contents)
# create the classpath string
classpath = []
for _, _, jars in os.walk(jars_dir):
for jar in jars:
classpath.append(join("$R_HOME/bin/fastr_jars", jar))
classpath_string = ":".join(classpath)
# replace the mx exec scripts with native Java launchers, setting the classpath from above
bin_exec_dir = join(bin_dir, 'exec')
r_launcher = join(self.subject.dir, 'src', 'R_launcher')
template_dict = {'CLASSPATH': classpath_string}
self._template(r_launcher, join(bin_exec_dir, 'R'), template_dict)
shutil.rmtree(join(bin_dir, 'execRextras'))
rscript_launcher = join(self.subject.dir, 'src', 'Rscript_launcher')
self._template(rscript_launcher, join(bin_dir, 'Rscript'), template_dict)
