def _create_jdk_bundle(jdkBuildDir, debugLevel, jdkImageDir): """ Creates a tar.gz JDK archive, an accompanying tar.gz.sha1 file with its SHA1 signature plus symlinks to the archive for non-canonical architecture names. """ arches = _get_jdk_bundle_arches() jdkTgzPath = join(_suite.get_output_root(), 'jdk-bundles', 'jdk9-{}-{}-{}.tar.gz'.format(debugLevel, _get_openjdk_os(), arches[0])) with mx.Archiver(jdkTgzPath, kind='tgz') as arc: mx.log('Creating ' + jdkTgzPath) for root, _, filenames in os.walk(jdkImageDir): for name in filenames: f = join(root, name) arcname = 'jdk1.9.0/' + os.path.relpath(f, jdkImageDir) arc.zf.add(name=f, arcname=arcname, recursive=False) with open(jdkTgzPath + '.sha1', 'w') as fp: mx.log('Creating ' + jdkTgzPath + '.sha1') fp.write(mx.sha1OfFile(jdkTgzPath)) def _create_link(source, link_name): if exists(link_name): os.remove(link_name) mx.log('Creating ' + link_name + ' -> ' + source) os.symlink(source, link_name) for arch in arches[1:]: link_name = join(_suite.get_output_root(), 'jdk-bundles', 'jdk9-{}-{}-{}.tar.gz'.format(debugLevel, _get_openjdk_os(), arch)) jdkTgzName = os.path.basename(jdkTgzPath) _create_link(jdkTgzName, link_name) _create_link(jdkTgzName + '.sha1', link_name + '.sha1')
def _create_jdk_bundle(jdkBuildDir): """ Creates a tar.gz JDK archive, an accompanying tar.gz.sha1 file with its SHA1 signature plus symlinks to the archive for non-canonical architecture names. """ jdkImageDir = join(jdkBuildDir, 'images', 'jdk') arches = _get_jdk_bundle_arches() jdkTgzPath = join(_suite.get_output_root(), 'jdk-bundles', 'jdk9-{}-{}.tar.gz'.format(_get_openjdk_os(), arches[0])) with mx.Archiver(jdkTgzPath, kind='tgz') as arc: mx.log('Creating ' + jdkTgzPath) for root, _, filenames in os.walk(jdkImageDir): for name in filenames: f = join(root, name) arcname = 'jdk1.9.0/' + os.path.relpath(f, jdkImageDir) arc.zf.add(name=f, arcname=arcname, recursive=False) # The OpenJDK build creates an empty cacerts file so grab one from # the default JDK which is assumed to be an OracleJDK cacerts = join( mx.get_jdk(tag='default').home, 'jre', 'lib', 'security', 'cacerts') arc.zf.add(name=cacerts, arcname='jdk1.9.0/lib/security/cacerts') with open(jdkTgzPath + '.sha1', 'w') as fp: mx.log('Creating ' + jdkTgzPath + '.sha1') fp.write(mx.sha1OfFile(jdkTgzPath)) def _create_link(source, link_name): if exists(link_name): os.remove(link_name) mx.log('Creating ' + link_name + ' -> ' + source) os.symlink(source, link_name) for arch in arches[1:]: link_name = join(_suite.get_output_root(), 'jdk-bundles', 'jdk9-{}-{}.tar.gz'.format(_get_openjdk_os(), arch)) jdkTgzName = os.path.basename(jdkTgzPath) _create_link(jdkTgzName, link_name) _create_link(jdkTgzName + '.sha1', link_name + '.sha1')
def _update_JVMCI_library(): """ Updates the "path" and "sha1" attributes of the "JVMCI" library to refer to a jvmci.jar created from the JVMCI classes in JDK9. """ suiteDict = _suite.suiteDict jvmciLib = suiteDict['libraries']['JVMCI'] d = join(_suite.get_output_root(), abspath(_jdk.home)[1:]) path = join(d, 'jvmci.jar') explodedModule = join(_jdk.home, 'modules', 'jdk.vm.ci') if exists(explodedModule): jarInputs = {} newestJarInput = None for root, _, files in os.walk(explodedModule): relpath = root[len(explodedModule) + 1:] for f in files: arcname = join(relpath, f).replace(os.sep, '/') jarInput = join(root, f) jarInputs[arcname] = jarInput t = mx.TimeStampFile(jarInput) if newestJarInput is None or t.isNewerThan(newestJarInput): newestJarInput = t if not exists(path) or newestJarInput.isNewerThan(path): with mx.Archiver(path, kind='zip') as arc: for arcname, jarInput in jarInputs.iteritems(): with open(jarInput, 'rb') as fp: contents = fp.read() arc.zf.writestr(arcname, contents) else: # Use the jdk.internal.jimage utility since it's the only way # to partially read .jimage files as the JDK9 jimage tool # does not support partial extraction. bootmodules = join(_jdk.home, 'lib', 'modules', 'bootmodules.jimage') if not exists(bootmodules): mx.abort('Could not find JVMCI classes at ' + bootmodules + ' or ' + explodedModule) if not exists(path) or mx.TimeStampFile(bootmodules).isNewerThan(path): mx.ensure_dir_exists(d) javaSource = join(d, 'ExtractJVMCI.java') with open(javaSource, 'w') as fp: print >> fp, """import java.io.FileOutputStream; import java.util.jar.JarEntry; import java.util.jar.JarOutputStream; import jdk.internal.jimage.BasicImageReader; public class ExtractJVMCI { public static void main(String[] args) throws Exception { BasicImageReader image = BasicImageReader.open(args[0]); String[] names = image.getEntryNames(); if (names.length == 0) { return; } try (JarOutputStream jos = new JarOutputStream(new FileOutputStream(args[1]))) { for (String name : names) { if (name.startsWith("/jdk.vm.ci/")) { String ename = name.substring("/jdk.vm.ci/".length()); JarEntry je = new JarEntry(ename); jos.putNextEntry(je); jos.write(image.getResource(name)); jos.closeEntry(); } } } } } """ mx.run([_jdk.javac, '-d', d, javaSource]) mx.run([_jdk.java, '-cp', d, 'ExtractJVMCI', bootmodules, path]) if not exists(path): mx.abort('Could not find the JVMCI classes in ' + bootmodules) jvmciLib['path'] = path jvmciLib['sha1'] = mx.sha1OfFile(path)
def build(self): if os.environ.has_key('FASTR_NO_RELEASE'): mx.log('FastR: not updating release project') return # copy the release directories output_dir = self.subject.dir fastr_dir = mx_fastr._fastr_suite.dir for d in ['bin', 'include', 'lib', 'library', 'etc', 'share', 'doc']: target_dir = join(output_dir, d) if os.path.exists(target_dir): shutil.rmtree(target_dir) shutil.copytree(join(fastr_dir, d), target_dir) # copyrights copyrights_dir = join(fastr_dir, 'mx.fastr', 'copyrights') with open(join(output_dir, 'COPYRIGHT'), 'w') as outfile: for copyright_file in os.listdir(copyrights_dir): basename = os.path.basename(copyright_file) if basename.endswith('copyright.star'): with open(join(copyrights_dir, copyright_file)) as infile: data = infile.read() outfile.write(data) # license/README shutil.copy(join(fastr_dir, 'LICENSE'), output_dir) shutil.copy(join(fastr_dir, 'README.md'), output_dir) # canonicalize R_HOME_DIR in bin/R bin_dir = join(output_dir, 'bin') rcmd = join(bin_dir, 'R') # R is the generic shell script (taken essentially verbatim from GNU R) with open(rcmd) as f: lines = f.readlines() with open(rcmd, 'w') as f: for line in lines: if line.startswith('R_HOME_DIR='): f.write('R_HOME_DIR="$(dirname $0)/.."\n') # produces a canonical path line = 'R_HOME_DIR="$(unset CDPATH && cd ${R_HOME_DIR} && pwd)"\n' f.write(line) # jar files for the launchers jars_dir = join(bin_dir, 'fastr_jars') if not os.path.exists(jars_dir): os.mkdir(jars_dir) fastr_classfiles = dict() # visitor to collect/copy all the classes/jar files needed by the launchers def dep_visit(dep, edge): if isinstance(dep, mx.JARDistribution): shutil.copy(join(dep.suite.dir, dep.path), jars_dir) elif isinstance(dep, mx.Library): jar_name = dep.name.lower() + '.jar' shutil.copyfile(join(dep.suite.dir, dep.path), join(jars_dir, jar_name)) elif isinstance(dep, mx.JavaProject): if 'com.oracle.truffle.r' in dep.name: classfiles_dir = dep.output_dir() for root, _, classfiles in os.walk(classfiles_dir): for classfile in classfiles: fastr_classfiles[os.path.relpath(join(root, classfile), classfiles_dir)] = join(root, classfile) self.subject.walk_deps(visit=dep_visit) # create the fastr.jar file with mx.Archiver(join(jars_dir, 'fastr.jar')) as arc: arc.zf.writestr("META-INF/MANIFEST.MF", "Manifest-Version: 1.0\n") for arcname, path in fastr_classfiles.iteritems(): with open(path, 'r') as f: contents = f.read() arc.zf.writestr(arcname, contents) # create the classpath string classpath = [] for _, _, jars in os.walk(jars_dir): for jar in jars: classpath.append(join("$R_HOME/bin/fastr_jars", jar)) classpath_string = ":".join(classpath) # replace the mx exec scripts with native Java launchers, setting the classpath from above bin_exec_dir = join(bin_dir, 'exec') r_launcher = join(self.subject.dir, 'src', 'R_launcher') template_dict = {'CLASSPATH': classpath_string} self._template(r_launcher, join(bin_exec_dir, 'R'), template_dict) shutil.rmtree(join(bin_dir, 'execRextras')) rscript_launcher = join(self.subject.dir, 'src', 'Rscript_launcher') self._template(rscript_launcher, join(bin_dir, 'Rscript'), template_dict)
def coverage_upload(args): parser = ArgumentParser(prog='mx coverage-upload') parser.add_argument('--upload-url', required=False, default=mx.get_env('COVERAGE_UPLOAD_URL'), help='Format is like rsync: user@host:/directory') parser.add_argument('--build-name', required=False, default=mx.get_env('BUILD_NAME')) parser.add_argument('--build-url', required=False, default=mx.get_env('BUILD_URL')) parser.add_argument('--build-number', required=False, default=mx.get_env('BUILD_NUMBER')) args, other_args = parser.parse_known_args(args) if not args.upload_url: parser.print_help() return remote_host, remote_basedir = args.upload_url.split(':') if not remote_host: mx.abort('Cannot determine remote host from {}'.format(args.upload_url)) primary = mx.primary_suite() info = primary.vc.parent_info(primary.dir) rev = primary.vc.parent(primary.dir) if len(remote_basedir) > 0 and not remote_basedir.endswith('/'): remote_basedir += '/' remote_dir = '{}_{}_{}'.format(primary.name, datetime.datetime.fromtimestamp(info['author-ts']).strftime('%Y-%m-%d_%H_%M'), rev[:7]) if args.build_name: remote_dir += '_' + args.build_name if args.build_number: remote_dir += '_' + args.build_number upload_dir = remote_basedir + remote_dir includes, excludes = _jacocoreport(['--omit-excluded'] + other_args) # Upload jar+sources coverage_sources = 'java_sources.tar.gz' coverage_binaries = 'java_binaries.tar.gz' with mx.Archiver(os.path.realpath(coverage_sources), kind='tgz') as sources, mx.Archiver(os.path.realpath(coverage_binaries), kind='tgz') as binaries: def _visit_deps(dep, edge): if dep.isJavaProject() and not dep.is_test_project(): binaries.zf.add(dep.output_dir(), dep.name) for d in dep.source_dirs(): sources.zf.add(d, dep.name) if os.path.exists(dep.source_gen_dir()): sources.zf.add(dep.source_gen_dir(), dep.name) mx.walk_deps(mx.projects(), visit=_visit_deps) files = [get_jacoco_dest_file(), 'coverage', coverage_sources, coverage_binaries] print("Syncing {} to {}:{}".format(" ".join(files), remote_host, upload_dir)) mx.run([ 'bash', '-c', r'tar -czf - {files} | ssh {remote} bash -c \'"mkdir -p {remotedir} && cd {remotedir} && cat | tar -x{verbose}z && chmod -R 755 ."\'' .format( files=" ".join(files), remote=remote_host, remotedir=upload_dir, verbose='v' if mx._opts.verbose else '') ]) def upload_string(content, path): mx.run(['ssh', remote_host, 'bash', '-c', 'cat > "' + path + '"'], stdin=content) upload_string(json.dumps({ 'timestamp': time.time(), 'suite': primary.name, 'revision': rev, 'directory': remote_dir, 'build_name': args.build_name, 'build_url': args.build_url, 'jdk_version': str(mx.get_jdk().version), 'build_number': args.build_number, 'primary_info': info, 'excludes': [str(e) for e in excludes], 'includes': [str(i) for i in includes]}), upload_dir + '/description.json') mx.run(['ssh', remote_host, 'bash', '-c', r'"(echo \[; for i in {remote_basedir}/*/description.json; do if \[ -s \$i \];then cat \$i; echo ,; fi done; echo null\]) > {remote_basedir}/index.json"'.format(remote_basedir=remote_basedir)]) upload_string("""<html> <script language="javascript"> function urlChange(url) { if (url.pathname !== "blank") { window.history.replaceState(null, null, url.pathname.replace("/coverage_upload/", "/coverage_upload/#")) } } </script> <frameset rows="40,*"> <frame id="navigation" src="navigation.html"/> <frame id="content" src="" onload="urlChange(this.contentWindow.location);" /> </frameset> </html>""", remote_basedir + '/index.html') js_library_url = rewriteurl("https://ajax.googleapis.com/ajax/libs/angularjs/1.7.7/angular.js") upload_string(r"""<html> <head> <script src="%js_library_url"></script> <script language="javascript"> var App = angular.module('myApp', []) .controller('IndexCtrl', function IndexCtrl($scope, $http) { var hash = parent.window.location.hash; if(hash) { hash = hash.substring(1, hash.length); // remove leading hash } $http.get('index.json').then(function(response, status) { var data = response.data.filter(x => x != null); /* #GR-17399 Filter build that are unique per suite with revision as key and merge builds. */ data = data .filter(x => !x.hasOwnProperty('merge')) .filter( // filter builds that are unique per suite with revision as key x => !data .filter(z => x != z && x.suite == z.suite) // exclude self build and build for other suites. .map(z => z.revision) // map from array of build to array of revision .includes(x.revision) // check if revision of x is index data. ).concat(data.filter(x => x.hasOwnProperty('merge'))); // concat unique build with merged build. data.sort((l,r) => r.timestamp - l.timestamp); if(data.length > 0) { var startdir; if(hash) { startdir = data.find(build => hash.includes(build.directory)); startdir.hash = hash; } if(!startdir) { startdir = data[0]; } $scope.directory = startdir; } $scope.data = data; }); $scope.$watch('directory', (dir, olddir) => { if(dir) { var content = parent.document.getElementById("content"); var contentDocument = content.contentDocument || content.contentWindow.document; var newpath; if(olddir && olddir.suite === dir.suite) { newpath = contentDocument.location.href.replace(olddir.directory, dir.directory); } else { newpath = dir.hasOwnProperty('hash') ? hash : dir.directory + "/coverage/"; } contentDocument.location.href = newpath; parent.window.history.replaceState(undefined, undefined, "#" + newpath.replace(/^.+coverage_upload\//, "")); } }); $scope.step = (i) => $scope.directory = $scope.data[$scope.data.indexOf($scope.directory)+i]; }); function copy(url) { var content = parent.document.getElementById("content"); var contentDocument = content.contentDocument || content.contentWindow.document; var copyText = document.getElementById("copy"); copyText.value = contentDocument.location.href.replace("coverage_upload/", "coverage_upload/#"); copyText.select(); document.execCommand("copy"); } </script> </head> <body ng-app="myApp" ng-controller="IndexCtrl"> <button ng-click="step(1)" ng-disabled="data.indexOf(directory) >= data.length-1"><<</button> <button ng-click="step(-1)" ng-disabled="data.indexOf(directory) <= 0">>></button> <select ng-model="directory" ng-options="(i.primary_info['author-ts']*1000|date:'yy-MM-dd hh:mm') + ' ' + i.build_name + ' ' + i.revision.substr(0,8) group by i.suite for i in data"></select> <a href="{{directory.build_url}}" ng-if="directory.build_url" target="_blank">Build</a> Commit: {{directory.revision.substr(0,5)}}: {{directory.primary_info.description}} <input type="text" style="opacity: 0;width: 20;" id="copy" /> <button style="float: right;" onclick="copy(window.location);">Share url</button> </body> </html>""".replace("%js_library_url", js_library_url), remote_basedir + '/navigation.html')
def build(self): if not os.environ.has_key('FASTR_RELEASE'): mx.log('FastR: set FASTR_RELEASE to update release project') return # copy the release directories output_dir = self.subject.dir fastr_dir = mx_fastr._fastr_suite.dir for d in ['bin', 'include', 'library', 'etc', 'share', 'doc']: target_dir = join(output_dir, d) if os.path.exists(target_dir): shutil.rmtree(target_dir) shutil.copytree(join(fastr_dir, d), target_dir) lib_fastr_dir = join(fastr_dir, 'lib') lib_output_dir = join(output_dir, 'lib') if os.path.exists(lib_output_dir): shutil.rmtree(lib_output_dir) os.mkdir(lib_output_dir) for f in os.listdir(lib_fastr_dir): source_file = join(lib_fastr_dir, f) target_file = join(lib_output_dir, f) if f != '.DS_Store': if os.path.islink(source_file): os.symlink(os.readlink(source_file), target_file) else: shutil.copy(source_file, target_file) # copyrights copyrights_dir = join(fastr_dir, 'mx.fastr', 'copyrights') with open(join(output_dir, 'COPYRIGHT'), 'w') as outfile: for copyright_file in os.listdir(copyrights_dir): basename = os.path.basename(copyright_file) if basename.endswith('copyright.star'): with open(join(copyrights_dir, copyright_file)) as infile: data = infile.read() outfile.write(data) # license/README shutil.copy(join(fastr_dir, 'LICENSE'), output_dir) shutil.copy(join(fastr_dir, 'README.md'), output_dir) # canonicalize R_HOME_DIR in bin/R bin_dir = join(output_dir, 'bin') rcmd = join(bin_dir, 'R') # R is the generic shell script (taken essentially verbatim from GNU R) with open(rcmd) as f: lines = f.readlines() with open(rcmd, 'w') as f: for line in lines: if line.startswith('R_HOME_DIR='): f.write( """ source="${BASH_SOURCE[0]}" while [ -h "$source" ] ; do prev_source="$source" source="$(readlink "$source")"; if [[ "$source" != /* ]]; then # if the link was relative, it was relative to where it came from dir="$( cd -P "$( dirname "$prev_source" )" && pwd )" source="$dir/$source" fi done r_bin="$( cd -P "$( dirname "$source" )" && pwd )" R_HOME_DIR="$( dirname "$r_bin" )" """) elif line.strip() == "#!/bin/sh": f.write("#!/usr/bin/env bash\n") else: f.write(line) # jar files for the launchers jars_dir = join(bin_dir, 'fastr_jars') if not os.path.exists(jars_dir): os.mkdir(jars_dir) fastr_classfiles = dict() # visitor to collect/copy all the classes/jar files needed by the launchers def dep_visit(dep, edge): if isinstance(dep, mx.JARDistribution): shutil.copy(join(dep.suite.dir, dep.path), jars_dir) elif isinstance(dep, mx.Library): if not dep.name.lower() == 'jdk_tools': jar_name = dep.name.lower() + '.jar' shutil.copyfile(join(dep.suite.dir, dep.path), join(jars_dir, jar_name)) elif isinstance(dep, mx.JavaProject): if 'com.oracle.truffle.r' in dep.name: classfiles_dir = dep.output_dir() for root, _, classfiles in os.walk(classfiles_dir): for classfile in classfiles: fastr_classfiles[os.path.relpath(join(root, classfile), classfiles_dir)] = join(root, classfile) self.subject.walk_deps(visit=dep_visit) # create the fastr.jar file with mx.Archiver(join(jars_dir, 'fastr.jar')) as arc: arc.zf.writestr("META-INF/MANIFEST.MF", "Manifest-Version: 1.0\n") for arcname, path in fastr_classfiles.iteritems(): with open(path, 'r') as f: contents = f.read() arc.zf.writestr(arcname, contents) # create the classpath string classpath = [] for _, _, jars in os.walk(jars_dir): for jar in jars: classpath.append(join("$R_HOME/bin/fastr_jars", jar)) classpath_string = ":".join(classpath) # replace the mx exec scripts with native Java launchers, setting the classpath from above bin_exec_dir = join(bin_dir, 'exec') r_launcher = join(self.subject.dir, 'src', 'R_launcher') template_dict = {'CLASSPATH': classpath_string} self._template(r_launcher, join(bin_exec_dir, 'R'), template_dict) shutil.rmtree(join(bin_dir, 'execRextras')) rscript_launcher = join(self.subject.dir, 'src', 'Rscript_launcher') self._template(rscript_launcher, join(bin_dir, 'Rscript'), template_dict)