def c1visualizer(args): """run the Cl Compiler Visualizer""" libpath = join(_suite.dir, 'lib') if mx.get_os() == 'windows': executable = join(libpath, 'c1visualizer', 'bin', 'c1visualizer.exe') else: executable = join(libpath, 'c1visualizer', 'bin', 'c1visualizer') # Check whether the current C1Visualizer installation is the up-to-date if exists(executable) and not exists(mx.library('C1VISUALIZER_DIST').get_path(resolve=False)): mx.log('Updating C1Visualizer') shutil.rmtree(join(libpath, 'c1visualizer')) archive = mx.library('C1VISUALIZER_DIST').get_path(resolve=True) if not exists(executable): zf = zipfile.ZipFile(archive, 'r') zf.extractall(libpath) if not exists(executable): mx.abort('C1Visualizer binary does not exist: ' + executable) if mx.get_os() != 'windows': # Make sure that execution is allowed. The zip file does not always specfiy that correctly os.chmod(executable, 0777) mx.run([executable])
def _run_netbeans_app(app_name, env=None, args=None): args = [] if args is None else args dist = app_name.upper() + '_DIST' name = app_name.lower() extractPath = join(_suite.get_output_root()) if mx.get_os() == 'windows': executable = join(extractPath, name, 'bin', name + '.exe') else: executable = join(extractPath, name, 'bin', name) # Check whether the current installation is up-to-date if exists(executable) and not exists(mx.library(dist).get_path(resolve=False)): mx.log('Updating ' + app_name) shutil.rmtree(join(extractPath, name)) archive = mx.library(dist).get_path(resolve=True) if not exists(executable): zf = zipfile.ZipFile(archive, 'r') zf.extractall(extractPath) if not exists(executable): mx.abort(app_name + ' binary does not exist: ' + executable) if mx.get_os() != 'windows': # Make sure that execution is allowed. The zip file does not always specfiy that correctly os.chmod(executable, 0777) mx.run([executable]+args, env=env)
def _parseVmArgs(jdk, args, addDefaultArgs=True): args = mx.expand_project_in_args(args, insitu=False) jacocoArgs = mx_gate.get_jacoco_agent_args() if jacocoArgs: args = jacocoArgs + args # Support for -G: options def translateGOption(arg): if arg.startswith('-G:+'): if '=' in arg: mx.abort('Mixing + and = in -G: option specification: ' + arg) arg = '-Dgraal.' + arg[len('-G:+'):] + '=true' elif arg.startswith('-G:-'): if '=' in arg: mx.abort('Mixing - and = in -G: option specification: ' + arg) arg = '-Dgraal.' + arg[len('-G:+'):] + '=false' elif arg.startswith('-G:'): if '=' not in arg: mx.abort('Missing "=" in non-boolean -G: option specification: ' + arg) arg = '-Dgraal.' + arg[len('-G:'):] return arg args = map(translateGOption, args) if '-G:+PrintFlags' in args and '-Xcomp' not in args: mx.warn('Using -G:+PrintFlags may have no effect without -Xcomp as Graal initialization is lazy') bcp = [] if _jvmciModes[_vm.jvmciMode]: if _add_jvmci_to_bootclasspath: bcp.append(mx.library('JVMCI').classpath_repr()) bcp.extend([d.get_classpath_repr() for d in _bootClasspathDists]) if bcp: args = ['-Xbootclasspath/p:' + os.pathsep.join(bcp)] + args # Remove JVMCI from class path. It's only there to support compilation. cpIndex, cp = mx.find_classpath_arg(args) if cp: jvmciLib = mx.library('JVMCI').path cp = os.pathsep.join([e for e in cp.split(os.pathsep) if e != jvmciLib]) args[cpIndex] = cp # Set the default JVMCI compiler jvmciCompiler = _compilers[-1] args = ['-Djvmci.compiler=' + jvmciCompiler] + args if '-version' in args: ignoredArgs = args[args.index('-version') + 1:] if len(ignoredArgs) > 0: mx.log("Warning: The following options will be ignored by the vm because they come after the '-version' argument: " + ' '.join(ignoredArgs)) return jdk.processArgs(args, addDefaultArgs=addDefaultArgs)
def igv(args): """run the Ideal Graph Visualizer""" logFile = '.ideal_graph_visualizer.log' with open(join(_suite.dir, logFile), 'w') as fp: mx.logv('[Ideal Graph Visualizer log is in ' + fp.name + ']') nbplatform = join(_suite.dir, 'src', 'share', 'tools', 'IdealGraphVisualizer', 'nbplatform') # Remove NetBeans platform if it is earlier than the current supported version if exists(nbplatform): updateTrackingFile = join(nbplatform, 'platform', 'update_tracking', 'org-netbeans-core.xml') if not exists(updateTrackingFile): mx.log('Could not find \'' + updateTrackingFile + '\', removing NetBeans platform') shutil.rmtree(nbplatform) else: dom = xml.dom.minidom.parse(updateTrackingFile) currentVersion = mx.VersionSpec(dom.getElementsByTagName('module_version')[0].getAttribute('specification_version')) supportedVersion = mx.VersionSpec('3.43.1') if currentVersion < supportedVersion: mx.log('Replacing NetBeans platform version ' + str(currentVersion) + ' with version ' + str(supportedVersion)) shutil.rmtree(nbplatform) elif supportedVersion < currentVersion: mx.log('Supported NetBeans version in igv command should be updated to ' + str(currentVersion)) if not exists(nbplatform): mx.logv('[This execution may take a while as the NetBeans platform needs to be downloaded]') env = _igvBuildEnv() # make the jar for Batik 1.7 available. env['IGV_BATIK_JAR'] = mx.library('BATIK').get_path(True) if mx.run(['ant', '-f', mx._cygpathU2W(join(_suite.dir, 'src', 'share', 'tools', 'IdealGraphVisualizer', 'build.xml')), '-l', mx._cygpathU2W(fp.name), 'run'], env=env, nonZeroIsFatal=False): mx.abort("IGV ant build & launch failed. Check '" + logFile + "'. You can also try to delete 'src/share/tools/IdealGraphVisualizer/nbplatform'.")
def igv(args): """run the Ideal Graph Visualizer""" env = dict(os.environ) # make the jar for Batik 1.7 available. env['IGV_BATIK_JAR'] = mx.library('BATIK').get_path(True) env['jdkhome'] = _igvJdk() _run_netbeans_app('IdealGraphVisualizer', env, args)
def jacocoreport(args): """create a JaCoCo coverage report Creates the report from the 'jacoco.exec' file in the current directory. Default output directory is 'coverage', but an alternative can be provided as an argument.""" jacocoreport = mx.library("JACOCOREPORT", True) out = 'coverage' if len(args) == 1: out = args[0] elif len(args) > 1: mx.abort('jacocoreport takes only one argument : an output directory') includes = list(_jacoco_includes) for p in mx.projects(): projsetting = getattr(p, 'jacoco', '') if projsetting == 'include' or projsetting == '': includes.append(p.name) includedirs = set() for p in mx.projects(): projsetting = getattr(p, 'jacoco', '') if projsetting == 'exclude': continue for include in includes: if include in p.dir: includedirs.add(p.dir) for i in includedirs: bindir = i + '/bin' mx.ensure_dir_exists(bindir) mx.run_java(['-jar', jacocoreport.get_path(True), '--in', 'jacoco.exec', '--out', out] + sorted(includedirs))
def getDacapo(name, dacapoArgs=None, extraVmArguments=None): dacapo = mx.get_env('DACAPO_CP') if dacapo is None: l = mx.library('DACAPO', False) if l is not None: dacapo = l.get_path(True) else: mx.abort('DaCapo 9.12 jar file must be specified with DACAPO_CP environment variable or as DACAPO library') if not isfile(dacapo) or not dacapo.endswith('.jar'): mx.abort('Specified DaCapo jar file does not exist or is not a jar file: ' + dacapo) dacapoSuccess = re.compile(r"^===== DaCapo 9\.12 ([a-zA-Z0-9_]+) PASSED in ([0-9]+) msec =====", re.MULTILINE) dacapoFail = re.compile(r"^===== DaCapo 9\.12 ([a-zA-Z0-9_]+) FAILED (warmup|) =====", re.MULTILINE) dacapoTime = re.compile(r"===== DaCapo 9\.12 (?P<benchmark>[a-zA-Z0-9_]+) PASSED in (?P<time>[0-9]+) msec =====") dacapoTime1 = re.compile(r"===== DaCapo 9\.12 (?P<benchmark>[a-zA-Z0-9_]+) completed warmup 1 in (?P<time>[0-9]+) msec =====") dacapoMatcher = ValuesMatcher(dacapoTime, {'group' : 'DaCapo', 'name' : '<benchmark>', 'score' : '<time>'}) dacapoMatcher1 = ValuesMatcher(dacapoTime1, {'group' : 'DaCapo-1stRun', 'name' : '<benchmark>', 'score' : '<time>'}) # Use ipv4 stack for dacapos; tomcat+solaris+ipv6_interface fails (see also: JDK-8072384) return Test("DaCapo-" + name, ['-jar', mx._cygpathU2W(dacapo), name] + _noneAsEmptyList(dacapoArgs), [dacapoSuccess], [dacapoFail], [dacapoMatcher, dacapoMatcher1], ['-Xms2g', '-XX:+' + gc, '-XX:-UseCompressedOops', "-Djava.net.preferIPv4Stack=true", '-G:+ExitVMOnException'] + _noneAsEmptyList(extraVmArguments))
def dacapoPath(self): dacapo = mx.get_env("DACAPO_CP") if dacapo: return dacapo lib = mx.library("DACAPO", False) if lib: return lib.get_path(True) return None
def daCapoPath(self): dacapo = mx.get_env(self.daCapoClasspathEnvVarName()) if dacapo: return dacapo lib = mx.library(self.daCapoLibraryName(), False) if lib: return lib.get_path(True) return None
def pullTestSuite(library, destDir, **kwargs): """downloads and unpacks a test suite""" mx.ensure_dir_exists(destDir) localPath = mx.library(library).get_path(True) mx_sulong.tar(localPath, destDir, **kwargs) os.remove(localPath) sha1Path = localPath + '.sha1' if os.path.exists(sha1Path): os.remove(sha1Path)
def jol(args): """Java Object Layout""" joljar = mx.library('JOL_INTERNALS').get_path(resolve=True) candidates = mx.findclass(args, logToConsole=False, matcher=lambda s, classname: s == classname or classname.endswith('.' + s) or classname.endswith('$' + s)) if len(candidates) > 0: candidates = mx.select_items(sorted(candidates)) else: # mx.findclass can be mistaken, don't give up yet candidates = args mx.run_java(['-javaagent:' + joljar, '-cp', os.pathsep.join([mx.classpath(jdk=mx.get_jdk()), joljar]), "org.openjdk.jol.MainObjectInternals"] + candidates)
def _sigtest_check(checktype, args, suite=None, projects=None): """run sigtest against Java projects with API""" sigtestlib = mx.library('SIGTEST').get_path(resolve=True) nonTestProjects = [p for p in mx.projects() if _should_test_project(p)] if not nonTestProjects: return 1 javaCompliance = max([p.javaCompliance for p in nonTestProjects]) class OutputCapture: def __init__(self): self.data = "" def __call__(self, data): self.data += data failed = None for p in nonTestProjects: sigtestResults = p.dir + os.sep + 'snapshot.sigtest' if not os.path.exists(sigtestResults): continue jdk = mx.get_jdk(javaCompliance) cmd = ['-cp', mx._cygpathU2W(sigtestlib), 'com.sun.tdk.signaturetest.SignatureTest', '-Static', '-Mode', 'bin', '-FileName', sigtestResults, '-ClassPath', mx.classpath(p, jdk=jdk) + os.pathsep + jdk.bootclasspath(), ] if checktype != 'all': cmd.append('-b') for pkg in mx._find_packages(p): cmd = cmd + ['-PackageWithoutSubpackages', pkg] out = OutputCapture() print 'Checking ' + checktype + ' signature changes against ' + sigtestResults exitcode = mx.run_java(cmd, nonZeroIsFatal=False, jdk=mx.get_jdk(javaCompliance), out=out, err=out) mx.ensure_dir_exists(p.get_output_root()) with open(p.get_output_root() + os.path.sep + 'sigtest-junit.xml', 'w') as f: f.write('<?xml version="1.0" encoding="UTF-8" ?>\n') f.write('<testsuite tests="1" name="' + p.name + '.sigtest.' + checktype + '">\n') f.write('<testcase classname="' + p.name + '" name="sigtest.' + checktype + '">\n') if exitcode != 95: print out.data failed = sigtestResults f.write('<failure type="SignatureCheck"><![CDATA[\n') f.write(out.data) f.write(']]></failure>') else: f.write('<system-err><![CDATA[\n') f.write(out.data) f.write(']]></system-err>') f.write('</testcase>\n') f.write('</testsuite>\n') if failed: mx.abort('Signature error in ' + failed) else: print 'OK.' return 0
def _parseVmArgs(jdk, args, addDefaultArgs=True): args = mx.expand_project_in_args(args, insitu=False) jacocoArgs = mx_gate.get_jacoco_agent_args() if jacocoArgs: args = jacocoArgs + args # Support for -G: options def translateGOption(arg): if arg.startswith("-G:+"): if "=" in arg: mx.abort("Mixing + and = in -G: option specification: " + arg) arg = "-Dgraal." + arg[len("-G:+") :] + "=true" elif arg.startswith("-G:-"): if "=" in arg: mx.abort("Mixing - and = in -G: option specification: " + arg) arg = "-Dgraal." + arg[len("-G:+") :] + "=false" elif arg.startswith("-G:"): if "=" not in arg: mx.abort('Missing "=" in non-boolean -G: option specification: ' + arg) arg = "-Dgraal." + arg[len("-G:") :] return arg args = map(translateGOption, args) if "-G:+PrintFlags" in args and "-Xcomp" not in args: mx.warn("Using -G:+PrintFlags may have no effect without -Xcomp as Graal initialization is lazy") bcp = [mx.distribution("truffle:TRUFFLE_API").classpath_repr()] if _jvmciModes[_vm.jvmciMode]: bcp.extend([d.get_classpath_repr() for d in _bootClasspathDists]) args = ["-Xbootclasspath/p:" + os.pathsep.join(bcp)] + args # Remove JVMCI from class path. It's only there to support compilation. cpIndex, cp = mx.find_classpath_arg(args) if cp: jvmciLib = mx.library("JVMCI").path cp = os.pathsep.join([e for e in cp.split(os.pathsep) if e != jvmciLib]) args[cpIndex] = cp # Set the default JVMCI compiler jvmciCompiler = _compilers[-1] args = ["-Djvmci.compiler=" + jvmciCompiler] + args if "-version" in args: ignoredArgs = args[args.index("-version") + 1 :] if len(ignoredArgs) > 0: mx.log( "Warning: The following options will be ignored by the vm because they come after the '-version' argument: " + " ".join(ignoredArgs) ) return jdk.processArgs(args, addDefaultArgs=addDefaultArgs)
def hcfdis(args): """disassemble HexCodeFiles embedded in text files Run a tool over the input files to convert all embedded HexCodeFiles to a disassembled format.""" parser = ArgumentParser(prog='mx hcfdis') parser.add_argument('-m', '--map', help='address to symbol map applied to disassembler output') parser.add_argument('files', nargs=REMAINDER, metavar='files...') args = parser.parse_args(args) path = mx.library('HCFDIS').get_path(resolve=True) mx.run_java(['-cp', path, 'com.oracle.max.hcfdis.HexCodeFileDis'] + args.files) if args.map is not None: addressRE = re.compile(r'0[xX]([A-Fa-f0-9]+)') with open(args.map) as fp: lines = fp.read().splitlines() symbols = dict() for l in lines: addressAndSymbol = l.split(' ', 1) if len(addressAndSymbol) == 2: address, symbol = addressAndSymbol if address.startswith('0x'): address = long(address, 16) symbols[address] = symbol for f in args.files: with open(f) as fp: lines = fp.read().splitlines() updated = False for i in range(0, len(lines)): l = lines[i] for m in addressRE.finditer(l): sval = m.group(0) val = long(sval, 16) sym = symbols.get(val) if sym: l = l.replace(sval, sym) updated = True lines[i] = l if updated: mx.log('updating ' + f) with open('new_' + f, "w") as fp: for l in lines: print >> fp, l
def get_jacoco_agent_args(): """ Gets the args to be added to a VM command line for injecting the JaCoCo agent if use of JaCoCo has been requested otherwise returns None. """ if _jacoco == "on" or _jacoco == "append": jacocoagent = mx.library("JACOCOAGENT", True) includes = list(_jacoco_includes) baseExcludes = [] for p in mx.projects(): projsetting = getattr(p, "jacoco", "") if projsetting == "exclude": baseExcludes.append(p.name) if projsetting == "include": includes.append(p.name + ".*") def _filter(l): # filter out specific classes which are already covered by a baseExclude package return [clazz for clazz in l if not any([clazz.startswith(package) for package in baseExcludes])] excludes = [] for p in mx.projects(): if p.isJavaProject(): excludes += _filter( p.find_classes_with_annotations(None, _jacoco_excluded_annotations, includeInnerClasses=True).keys() ) excludes += _filter( p.find_classes_with_matching_source_line( None, lambda line: "JaCoCo Exclude" in line, includeInnerClasses=True ).keys() ) excludes += [package + ".*" for package in baseExcludes] agentOptions = { "append": "true" if _jacoco == "append" else "false", "bootclasspath": "true", "includes": ":".join(includes), "excludes": ":".join(excludes), "destfile": "jacoco.exec", } return [ "-javaagent:" + jacocoagent.get_path(True) + "=" + ",".join([k + "=" + v for k, v in agentOptions.items()]) ] return None
def findbugs(args, fbArgs=None, suite=None, projects=None): """run FindBugs against non-test Java projects""" findBugsHome = mx.get_env('FINDBUGS_HOME', None) if suite is None: suite = mx._primary_suite if findBugsHome: findbugsJar = join(findBugsHome, 'lib', 'findbugs.jar') else: findbugsLib = join(mx._mx_suite.get_output_root(), 'findbugs-3.0.0') if not exists(findbugsLib): tmp = tempfile.mkdtemp(prefix='findbugs-download-tmp', dir=mx._mx_suite.dir) try: findbugsDist = mx.library('FINDBUGS_DIST').get_path(resolve=True) with zipfile.ZipFile(findbugsDist) as zf: candidates = [e for e in zf.namelist() if e.endswith('/lib/findbugs.jar')] assert len(candidates) == 1, candidates libDirInZip = os.path.dirname(candidates[0]) zf.extractall(tmp) shutil.copytree(join(tmp, libDirInZip), findbugsLib) finally: shutil.rmtree(tmp) findbugsJar = join(findbugsLib, 'findbugs.jar') assert exists(findbugsJar) nonTestProjects = [p for p in mx.projects() if _should_test_project(p)] if not nonTestProjects: return 0 outputDirs = map(mx._cygpathU2W, [p.output_dir() for p in nonTestProjects]) javaCompliance = max([p.javaCompliance for p in nonTestProjects]) jdk = mx.get_jdk(javaCompliance) if jdk.javaCompliance >= "1.9": mx.log('FindBugs does not yet support JDK9 - skipping') return 0 findbugsResults = join(suite.dir, 'findbugs.results') if fbArgs is None: fbArgs = defaultFindbugsArgs() cmd = ['-jar', mx._cygpathU2W(findbugsJar)] + fbArgs cmd = cmd + ['-auxclasspath', mx._separatedCygpathU2W(mx.classpath([p.name for p in nonTestProjects], jdk=jdk)), '-output', mx._cygpathU2W(findbugsResults), '-exitcode'] + args + outputDirs exitcode = mx.run_java(cmd, nonZeroIsFatal=False, jdk=jdk) if exitcode != 0: with open(findbugsResults) as fp: mx.log(fp.read()) os.unlink(findbugsResults) return exitcode
def getScalaDacapo(name, dacapoArgs=[]): dacapo = mx.get_env('DACAPO_SCALA_CP') if dacapo is None: l = mx.library('DACAPO_SCALA', False) if l is not None: dacapo = l.get_path(True) else: mx.abort('Scala DaCapo 0.1.0 jar file must be specified with DACAPO_SCALA_CP environment variable or as DACAPO_SCALA library') if not isfile(dacapo) or not dacapo.endswith('.jar'): mx.abort('Specified Scala DaCapo jar file does not exist or is not a jar file: ' + dacapo) dacapoSuccess = re.compile(r"^===== DaCapo 0\.1\.0(-SNAPSHOT)? ([a-zA-Z0-9_]+) PASSED in ([0-9]+) msec =====$", re.MULTILINE) dacapoFail = re.compile(r"^===== DaCapo 0\.1\.0(-SNAPSHOT)? ([a-zA-Z0-9_]+) FAILED (warmup|) =====$", re.MULTILINE) dacapoTime = re.compile(r"===== DaCapo 0\.1\.0(-SNAPSHOT)? (?P<benchmark>[a-zA-Z0-9_]+) PASSED in (?P<time>[0-9]+) msec =====") dacapoMatcher = ValuesMatcher(dacapoTime, {'group' : "Scala-DaCapo", 'name' : '<benchmark>', 'score' : '<time>'}) return Test("Scala-DaCapo-" + name, ['-jar', dacapo, name] + dacapoArgs, [dacapoSuccess], [dacapoFail], [dacapoMatcher], ['-Xms2g', '-XX:+'+gc, '-XX:-UseCompressedOops'])
def jackpot(args, suite=None, nonZeroIsFatal=False): """run Jackpot 3.0 against non-test Java projects""" jackpotHome = mx.get_env('JACKPOT_HOME', None) if jackpotHome: jackpotJar = join(jackpotHome, 'jackpot.jar') else: jackpotJar = mx.library('JACKPOT').get_path(resolve=True) assert exists(jackpotJar) if suite is None: suite = mx._primary_suite nonTestProjects = [p for p in mx.projects() if _should_test_project(p)] if not nonTestProjects: return 0 groups = [] for p in nonTestProjects: javacClasspath = [] deps = [] p.walk_deps(visit=lambda dep, edge: deps.append(dep) if dep.isLibrary() or dep.isJavaProject() else None) annotationProcessorOnlyDeps = [] if len(p.annotation_processors()) > 0: for apDep in p.annotation_processors(): if not apDep in deps: deps.append(apDep) annotationProcessorOnlyDeps.append(apDep) for dep in deps: if dep == p: continue if dep in annotationProcessorOnlyDeps: continue javacClasspath.append(dep.classpath_repr(resolve=True)) javaCompliance = p.javaCompliance groups = groups + ['--group', "--classpath " + mx._separatedCygpathU2W(_escape_string(os.pathsep.join(javacClasspath))) + " --source " + str(p.javaCompliance) + " " + " ".join([_escape_string(d) for d in p.source_dirs()])] cmd = ['-classpath', mx._cygpathU2W(jackpotJar), 'org.netbeans.modules.jackpot30.cmdline.Main'] cmd = cmd + ['--fail-on-warnings', '--progress'] + args + groups return mx.run_java(cmd, nonZeroIsFatal=nonZeroIsFatal, jdk=mx.get_jdk(javaCompliance))
def getDacapo(name, dacapoArgs=None): dacapo = mx.get_env('DACAPO_CP') if dacapo is None: l = mx.library('DACAPO', False) if l is not None: dacapo = l.get_path(True) else: mx.abort('DaCapo 9.12 jar file must be specified with DACAPO_CP environment variable or as DACAPO library') if not isfile(dacapo) or not dacapo.endswith('.jar'): mx.abort('Specified DaCapo jar file does not exist or is not a jar file: ' + dacapo) dacapoSuccess = re.compile(r"^===== DaCapo 9\.12 ([a-zA-Z0-9_]+) PASSED in ([0-9]+) msec =====$", re.MULTILINE) dacapoFail = re.compile(r"^===== DaCapo 9\.12 ([a-zA-Z0-9_]+) FAILED (warmup|) =====$", re.MULTILINE) dacapoTime = re.compile(r"===== DaCapo 9\.12 (?P<benchmark>[a-zA-Z0-9_]+) PASSED in (?P<time>[0-9]+) msec =====") dacapoTime1 = re.compile(r"===== DaCapo 9\.12 (?P<benchmark>[a-zA-Z0-9_]+) completed warmup 1 in (?P<time>[0-9]+) msec =====") dacapoMatcher = ValuesMatcher(dacapoTime, {'group' : 'DaCapo', 'name' : '<benchmark>', 'score' : '<time>'}) dacapoMatcher1 = ValuesMatcher(dacapoTime1, {'group' : 'DaCapo-1stRun', 'name' : '<benchmark>', 'score' : '<time>'}) return Test("DaCapo-" + name, ['-jar', dacapo, name] + _noneAsEmptyList(dacapoArgs), [dacapoSuccess], [dacapoFail], [dacapoMatcher, dacapoMatcher1], ['-Xms2g', '-XX:+' + gc, '-XX:-UseCompressedOops'])
def _alldeps(args): parser = ArgumentParser(prog='mx mxt-alldeps') parser.add_argument('--kind', action='store', help='project, dist or library', default='project') parser.add_argument('--name', action='store', help='name of entity', required=True) parser.add_argument('--includeLibs', action='store_true', help='includeLibs') parser.add_argument('--ignoreSelf', action='store_true', help='ignoreSelf') args = parser.parse_args(args) entity = None if args.kind == 'project': entity = mx.project(args.name) elif args.kind == 'library': entity = mx.library(args.name) elif args.kind == 'dist': entity = mx.distribution(args.name) else: mx.abort('illegal kind: ' + args.kind) in_deps = [] deps = entity.all_deps(in_deps, includeLibs=args.includeLibs, includeSelf=not args.ignoreSelf) print 'alldeps:' for d in deps: print d.__class__.__name__, ":", d.name
def _sigtest_generate(args, suite=None, projects=None): """run sigtest generator for Java projects with API""" sigtestlib = mx.library('SIGTEST').get_path(resolve=True) nonTestProjects = [p for p in mx.projects() if _should_test_project(p)] if not nonTestProjects: return 0 javaCompliance = max([p.javaCompliance for p in nonTestProjects]) for p in nonTestProjects: sigtestResults = p.dir + os.sep + 'snapshot.sigtest' cmd = ['-cp', mx._cygpathU2W(sigtestlib), 'com.sun.tdk.signaturetest.Setup', '-Static', '-FileName', sigtestResults, '-ClassPath', mx.classpath(p) + os.pathsep + mx.get_jdk(javaCompliance).bootclasspath(), ] for pkg in mx.find_packages(p): cmd = cmd + ['-PackageWithoutSubpackages', pkg] exitcode = mx.run_java(cmd, nonZeroIsFatal=False, jdk=mx.get_jdk(javaCompliance)) if exitcode != 95: mx.abort('Exit code was ' + str(exitcode) + ' while generating ' + sigtestResults) if not exists(sigtestResults): mx.abort('Cannot generate ' + sigtestResults) mx.log("Sigtest snapshot generated to " + sigtestResults) return 0
def get_jacoco_agent_args(): ''' Gets the args to be added to a VM command line for injecting the JaCoCo agent if use of JaCoCo has been requested otherwise returns None. ''' if _jacoco == 'on' or _jacoco == 'append': jacocoagent = mx.library("JACOCOAGENT", True) includes = list(_jacoco_includes) baseExcludes = [] for p in mx.projects(): projsetting = getattr(p, 'jacoco', '') if projsetting == 'exclude': baseExcludes.append(p.name) if projsetting == 'include': includes.append(p.name + '.*') def _filter(l): # filter out specific classes which are already covered by a baseExclude package return [clazz for clazz in l if not any([clazz.startswith(package) for package in baseExcludes])] excludes = [] for p in mx.projects(): if p.isJavaProject(): excludes += _filter(p.find_classes_with_annotations(None, _jacoco_excluded_annotations, includeInnerClasses=True).keys()) excludes += _filter(p.find_classes_with_matching_source_line(None, lambda line: 'JaCoCo Exclude' in line, includeInnerClasses=True).keys()) excludes += [package + '.*' for package in baseExcludes] agentOptions = { 'append' : 'true' if _jacoco == 'append' else 'false', 'bootclasspath' : 'true', 'includes' : ':'.join(includes), 'excludes' : ':'.join(excludes), 'destfile' : 'jacoco.exec' } return ['-javaagent:' + jacocoagent.get_path(True) + '=' + ','.join([k + '=' + v for k, v in agentOptions.items()])] return None
def gate_body(args, tasks): with Task('Vm: GraalVM dist names', tasks, tags=['names']) as t: if t: mx_sdk_vm.verify_graalvm_configs(suites=['vm', 'vm-enterprise']) with Task('Vm: Basic GraalVM Tests', tasks, tags=[VmGateTasks.compiler]) as t: if t and mx_sdk_vm_impl.has_component('GraalVM compiler'): # 1. the build must be a GraalVM # 2. the build must be JVMCI-enabled since the 'GraalVM compiler' component is registered mx_sdk_vm_impl.check_versions( mx_sdk_vm_impl.graalvm_output(), graalvm_version_regex=mx_sdk_vm_impl.graalvm_version_regex, expect_graalvm=True, check_jvmci=True) if mx_sdk_vm_impl.has_component('LibGraal'): libgraal_location = mx_sdk_vm_impl.get_native_image_locations( 'LibGraal', 'jvmcicompiler') if libgraal_location is None: mx.warn( "Skipping libgraal tests: no library enabled in the LibGraal component" ) else: extra_vm_arguments = [ '-XX:+UseJVMCICompiler', '-XX:+UseJVMCINativeLibrary', '-XX:JVMCILibPath=' + dirname(libgraal_location) ] if args.extra_vm_argument: extra_vm_arguments += args.extra_vm_argument import mx_compiler # run avrora on the GraalVM binary itself with Task('LibGraal Compiler:GraalVM DaCapo-avrora', tasks, tags=[VmGateTasks.libgraal]) as t: if t: java_exe = join(mx_sdk_vm_impl.graalvm_home(), 'bin', 'java') mx.run([ java_exe, '-XX:+UseJVMCICompiler', '-XX:+UseJVMCINativeLibrary', '-jar', mx.library('DACAPO').get_path(True), 'avrora', '-n', '1' ]) # Ensure that fatal errors in libgraal route back to HotSpot vmargs = [ '-XX:+UseJVMCICompiler', '-XX:+UseJVMCINativeLibrary', '-XX:+PrintFlagsFinal', '-Dlibgraal.CrashAt=length,hashCode', '-Dlibgraal.CrashAtIsFatal=true' ] cmd = ["dacapo:avrora", "--tracker=none", "--" ] + vmargs + ["--", "--preserve"] out = mx.OutputCapture() exitcode, bench_suite, _ = mx_benchmark.gate_mx_benchmark( cmd, out=out, err=out, nonZeroIsFatal=False) if exitcode == 0: if 'CrashAtIsFatal: no fatalError function pointer installed' in out.data: # Executing a VM that does not configure fatal errors handling # in libgraal to route back through the VM. pass else: mx.abort( 'Expected following benchmark to result in non-zero exit code: ' + ' '.join(cmd)) else: if len(bench_suite.scratchDirs()) == 0: mx.abort( "No scratch dir found despite error being expected!" ) latest_scratch_dir = bench_suite.scratchDirs()[-1] seen_libjvmci_log = False hs_errs = glob.glob( join(latest_scratch_dir, 'hs_err_pid*.log')) if not hs_errs: mx.abort( 'Expected a file starting with "hs_err_pid" in test directory. Entries found=' + str(listdir(latest_scratch_dir))) for hs_err in hs_errs: mx.log("Verifying content of {}".format( join(latest_scratch_dir, hs_err))) with open(join(latest_scratch_dir, hs_err)) as fp: contents = fp.read() if 'libjvmci' in hs_err: seen_libjvmci_log = True if 'Fatal error: Forced crash' not in contents: mx.abort( 'Expected "Fatal error: Forced crash" to be in contents of ' + hs_err + ':' + linesep + contents) else: if 'Fatal error in JVMCI' not in contents: mx.abort( 'Expected "Fatal error in JVMCI" to be in contents of ' + hs_err + ':' + linesep + contents) if 'JVMCINativeLibraryErrorFile' in out.data and not seen_libjvmci_log: mx.abort( 'Expected a file matching "hs_err_pid*_libjvmci.log" in test directory. Entries found=' + str(listdir(latest_scratch_dir))) # Only clean up scratch dir on success for scratch_dir in bench_suite.scratchDirs(): mx.log( "Cleaning up scratch dir after gate task completion: {}" .format(scratch_dir)) mx.rmtree(scratch_dir) with Task('LibGraal Compiler:CTW', tasks, tags=[VmGateTasks.libgraal]) as t: if t: mx_compiler.ctw([ '-DCompileTheWorld.Config=Inline=false ' + ' '.join( mx_compiler._compiler_error_options(prefix='')), '-esa', '-XX:+EnableJVMCI', '-DCompileTheWorld.MultiThreaded=true', '-Dgraal.InlineDuringParsing=false', '-Dgraal.TrackNodeSourcePosition=true', '-DCompileTheWorld.Verbose=false', '-DCompileTheWorld.HugeMethodLimit=4000', '-DCompileTheWorld.MaxCompiles=150000', '-XX:ReservedCodeCacheSize=300m', ], extra_vm_arguments) mx_compiler.compiler_gate_benchmark_runner( tasks, extra_vm_arguments, prefix='LibGraal Compiler:') with Task('LibGraal Truffle:unittest', tasks, tags=[VmGateTasks.libgraal]) as t: if t: def _unittest_config_participant(config): vmArgs, mainClass, mainClassArgs = config def is_truffle_fallback(arg): fallback_args = [ "-Dtruffle.TruffleRuntime=com.oracle.truffle.api.impl.DefaultTruffleRuntime", "-Dgraalvm.ForcePolyglotInvalid=true" ] return arg in fallback_args newVmArgs = [ arg for arg in vmArgs if not is_truffle_fallback(arg) ] return (newVmArgs, mainClass, mainClassArgs) mx_unittest.add_config_participant( _unittest_config_participant) excluded_tests = environ.get("TEST_LIBGRAAL_EXCLUDE") if excluded_tests: with NamedTemporaryFile(prefix='blacklist.', mode='w', delete=False) as fp: fp.file.writelines( [l + '\n' for l in excluded_tests.split()]) unittest_args = ["--blacklist", fp.name] else: unittest_args = [] unittest_args = unittest_args + [ "--enable-timing", "--verbose" ] compiler_log_file = "graal-compiler.log" mx_unittest.unittest(unittest_args + extra_vm_arguments + [ "-Dpolyglot.engine.AllowExperimentalOptions=true", "-Dpolyglot.engine.CompileImmediately=true", "-Dpolyglot.engine.BackgroundCompilation=false", "-Dpolyglot.engine.TraceCompilation=true", "-Dpolyglot.log.file={0}".format(compiler_log_file), "-Dgraalvm.locatorDisabled=true", "truffle" ]) if exists(compiler_log_file): remove(compiler_log_file) else: mx.warn("Skipping libgraal tests: component not enabled") gate_substratevm(tasks) gate_sulong(tasks) gate_python(tasks) gate_svm_sl_tck(tasks) gate_svm_truffle_tck_js(tasks)
def _sigtest_check(checktype, args, suite=None, projects=None): """run sigtest against Java projects with API""" sigtestlib = mx.library('SIGTEST').get_path(resolve=True) nonTestProjects = [p for p in mx.projects() if _should_test_project(p)] if not nonTestProjects: return 1 javaCompliance = max([p.javaCompliance for p in nonTestProjects]) class OutputCapture: def __init__(self): self.data = "" def __call__(self, data): self.data += data failed = None for p in nonTestProjects: sigtestResults = p.dir + os.sep + 'snapshot.sigtest' if not os.path.exists(sigtestResults): continue jdk = mx.get_jdk(javaCompliance) cmd = [ '-cp', mx._cygpathU2W(sigtestlib), 'com.sun.tdk.signaturetest.SignatureTest', '-BootCP', '-Static', '-Mode', 'bin', '-FileName', sigtestResults, '-ClassPath', mx.classpath(p, jdk=jdk), ] if args.human: cmd.append('-H') if checktype != 'all': cmd.append('-b') for pkg in mx._find_packages(p): cmd = cmd + ['-PackageWithoutSubpackages', pkg] out = OutputCapture() print('Checking ' + checktype + ' signature changes against ' + sigtestResults) exitcode = mx.run_java(cmd, nonZeroIsFatal=False, jdk=mx.get_jdk(javaCompliance), out=out, err=out) mx.ensure_dir_exists(p.get_output_root()) with open(p.get_output_root() + os.path.sep + 'sigtest-junit.xml', 'w') as f: f.write('<?xml version="1.0" encoding="UTF-8" ?>\n') f.write('<testsuite tests="1" name="' + p.name + '.sigtest.' + checktype + '">\n') f.write('<testcase classname="' + p.name + '" name="sigtest.' + checktype + '">\n') if exitcode != 95: print(out.data) failed = sigtestResults f.write('<failure type="SignatureCheck"><![CDATA[\n') f.write(out.data) f.write(']]></failure>') else: f.write('<system-err><![CDATA[\n') f.write(out.data) f.write(']]></system-err>') f.write('</testcase>\n') f.write('</testsuite>\n') if failed: print( '\nThe signature check detected changes in the API by comparing it with previous signature files.' ) print( 'To fix this restore the original API or regenerate the signature files with:' ) print('mx sigtest --generate') mx.abort('Signature error in ' + failed) else: print('OK.') return 0
def harness_path(self): lib = mx.library(self.renaissance_harness_lib_name()) if lib: return lib.get_path(True) return None
def _get_lib_path(_lib_name): return mx.library(_lib_name).get_path(resolve=True)
def get_jacoco_agent_path(resolve): return mx.library('JACOCOAGENT_0.8.4', True).get_path(resolve)
def _gate_dacapo(name, iterations, extraVMarguments=None): vmargs = ['-Xms2g', '-XX:+UseSerialGC', '-XX:-UseCompressedOops', '-Djava.net.preferIPv4Stack=true', '-Dgraal.ExitVMOnException=true'] + _remove_empty_entries(extraVMarguments) dacapoJar = mx.library('DACAPO').get_path(True) _gate_java_benchmark(vmargs + ['-jar', dacapoJar, name, '-n', str(iterations)], r'^===== DaCapo 9\.12 ([a-zA-Z0-9_]+) PASSED in ([0-9]+) msec =====')
def _gate_scala_dacapo(name, iterations, extraVMarguments=None): vmargs = ['-Xms2g', '-XX:+UseSerialGC', '-XX:-UseCompressedOops', '-Dgraal.ExitVMOnException=true'] + _noneAsEmptyList(extraVMarguments) scalaDacapoJar = mx.library('DACAPO_SCALA').get_path(True) _gate_java_benchmark(vmargs + ['-jar', scalaDacapoJar, name, '-n', str(iterations)], r'^===== DaCapo 0\.1\.0(-SNAPSHOT)? ([a-zA-Z0-9_]+) PASSED in ([0-9]+) msec =====')
def build(self): pre_ts = GraalNodeJsBuildTask._get_newest_ts(self.subject.getResults(), fatalIfMissing=False) build_env = os.environ.copy() _setEnvVar( 'PATH', '%s%s%s' % (join(_suite.mxDir, 'python2'), pathsep, build_env['PATH']), build_env) debug = ['--debug'] if self._debug_mode else [] shared_library = ['--enable-shared-library'] if hasattr( self.args, 'sharedlibrary') and self.args.sharedlibrary else [] newest_config_file_ts = GraalNodeJsBuildTask._get_newest_ts( _config_files, fatalIfMissing=True) newest_generated_config_file_ts = GraalNodeJsBuildTask._get_newest_ts( _generated_config_files, fatalIfMissing=False) # Lazily generate config files only if `configure` and `configure.py` are older than the files they generate. # If we don't do this, the `Makefile` always considers `config.gypi` out of date, triggering a second, unnecessary configure. lazy_generator = ['--lazy-generator' ] if newest_generated_config_file_ts.isNewerThan( newest_config_file_ts) else [] if _is_windows: processDevkitRoot(env=build_env) _setEnvVar( 'PATH', pathsep.join([build_env['PATH']] + [ mx.library(lib_name).get_path(True) for lib_name in ('NASM', 'NINJA') ]), build_env) extra_flags = ['--ninja', '--dest-cpu=x64', '--without-etw'] else: extra_flags = [] _mxrun(python_cmd() + [ join(_suite.dir, 'configure'), '--partly-static', '--without-dtrace', '--without-inspector', '--without-node-snapshot', '--without-node-code-cache', '--java-home', _java_home(forBuild=True) ] + debug + shared_library + lazy_generator + extra_flags, cwd=_suite.dir, print_cmd=True, env=build_env) quiet_build = mx.is_continuous_integration( ) and not mx.get_opts().verbose if _is_windows: verbose = ['-v'] if mx.get_opts().verbose else [] # The custom env is not used to resolve the location of the executable _mxrun([join(mx.library('NINJA').get_path(True), 'ninja.exe')] + verbose + ['-j%d' % self.parallelism, '-C', self._build_dir], print_cmd=True, quiet_if_successful=quiet_build, env=build_env) else: verbose = 'V={}'.format('1' if mx.get_opts().verbose else '') _mxrun([mx.gmake_cmd(), '-j%d' % self.parallelism, verbose], cwd=_suite.dir, print_cmd=True, quiet_if_successful=quiet_build, env=build_env) # put headers for native modules into out/headers _setEnvVar('HEADERS_ONLY', '1', build_env) _mxrun(python_cmd() + [ join('tools', 'install.py'), 'install', join('out', 'headers'), sep ], quiet_if_successful=not mx.get_opts().verbose, env=build_env) post_ts = GraalNodeJsBuildTask._get_newest_ts( self.subject.getResults(), fatalIfMissing=True) mx.logv( 'Newest time-stamp before building: {}\nNewest time-stamp after building: {}\nHas built? {}' .format(pre_ts, post_ts, post_ts.isNewerThan(pre_ts))) built = post_ts.isNewerThan(pre_ts) if built and _current_os == 'darwin': nodePath = join(self._build_dir, 'node') _mxrun([ 'install_name_tool', '-add_rpath', join(_java_home(), 'jre', 'lib'), '-add_rpath', join(_java_home(), 'lib'), nodePath ], print_cmd=True, env=build_env) return built
def renaissance_additional_lib(self, lib): return mx.library(lib).get_path(True)
def additional_lib(self, lib): return mx.library(lib).get_path(True)
def daCapoPath(self): lib = mx.library(self.daCapoLibraryName(), False) if lib: return lib.get_path(True) return None
def getLibraryDirectory(self): return mx.library("WRK2", True).get_path(True)
def get_sigtest_jar(self): """ Returns the proper version of the SIGTEST jar used by `mx sigtest`. """ return mx.library('SIGTEST_1_2').get_path(resolve=True)
def _gate_scala_dacapo(name, iterations, extraVMarguments=None): vmargs = ['-Xms2g', '-XX:+UseSerialGC', '-XX:-UseCompressedOops', '-Dgraal.ExitVMOnException=true'] + _remove_empty_entries(extraVMarguments) scalaDacapoJar = mx.library('DACAPO_SCALA').get_path(True) _gate_java_benchmark(vmargs + ['-jar', scalaDacapoJar, name, '-n', str(iterations)], r'^===== DaCapo 0\.1\.0(-SNAPSHOT)? ([a-zA-Z0-9_]+) PASSED in ([0-9]+) msec =====')
def bench(self, vm, cwd=None, extraVmOpts=None, vmbuild=None): """ Run this program as a benchmark. Copied from sanitycheck.Test to extend benchmarking for non-JVMs. """ if vm in self.ignoredVMs: return {} if cwd is None: cwd = self.defaultCwd parser = OutputParser() for successRE in self.successREs: parser.addMatcher(ValuesMatcher(successRE, {'passed' : '1'})) for failureRE in self.failureREs: parser.addMatcher(ValuesMatcher(failureRE, {'failed' : '1'})) for scoreMatcher in self.scoreMatchers: parser.addMatcher(scoreMatcher) if self.benchmarkCompilationRate: if vm == 'graal': bps = re.compile(r"ParsedBytecodesPerSecond@final: (?P<rate>[0-9]+)") ibps = re.compile(r"InlinedBytecodesPerSecond@final: (?P<rate>[0-9]+)") parser.addMatcher(ValuesMatcher(bps, {'group' : 'ParsedBytecodesPerSecond', 'name' : self.name, 'score' : '<rate>'})) parser.addMatcher(ValuesMatcher(ibps, {'group' : 'InlinedBytecodesPerSecond', 'name' : self.name, 'score' : '<rate>'})) else: ibps = re.compile(r"(?P<compiler>[\w]+) compilation speed: +(?P<rate>[0-9]+) bytes/s {standard") parser.addMatcher(ValuesMatcher(ibps, {'group' : 'InlinedBytecodesPerSecond', 'name' : '<compiler>:' + self.name, 'score' : '<rate>'})) startDelim = 'START: ' + self.name endDelim = 'END: ' + self.name outputfile = os.environ.get('BENCH_OUTPUT', None) if outputfile: # Used only to debug output parsing with open(outputfile) as fp: output = fp.read() start = output.find(startDelim) end = output.find(endDelim, start) if start == -1 and end == -1: return {} output = output[start + len(startDelim + os.linesep): end] mx.log(startDelim) mx.log(output) mx.log(endDelim) else: tee = Tee() mx.log(startDelim) # zippy result = -1 if vm == 'cpython2': result = mx.run(['python'] + self.cmd[-2:], out=tee.eat) elif vm == 'cpython': result = mx.run(['python3'] + self.cmd[-2:], out=tee.eat) elif vm == 'jython': result = mx_graal.vm(['-jar', mx.library('JYTHON').path] + self.cmd[-2:], vm = 'original', out=tee.eat) elif vm == 'pypy': result = mx.run(['pypy'] + self.cmd[-2:], out=tee.eat) elif vm == 'pypy3': result = mx.run(['pypy3'] + self.cmd[-2:], out=tee.eat) else: result = mx_graal.vm(self.vmOpts + _noneAsEmptyList(extraVmOpts) + self.cmd, vm, nonZeroIsFatal=False, out=tee.eat, err=subprocess.STDOUT, cwd=cwd, vmbuild=vmbuild) if result != 0: mx.abort("Benchmark failed (non-zero retcode)") # wait for subprocess to finish time.sleep(.5) mx.log(endDelim) output = tee.output.getvalue() groups = {} passed = False for valueMap in parser.parse(output): assert (valueMap.has_key('name') and valueMap.has_key('score') and valueMap.has_key('group')) or valueMap.has_key('passed') or valueMap.has_key('failed'), valueMap if valueMap.get('failed') == '1': mx.abort("Benchmark failed") if valueMap.get('passed') == '1': passed = True groupName = valueMap.get('group') if groupName: group = groups.setdefault(groupName, {}) name = valueMap.get('name') score = valueMap.get('score') if name and score: group[name] = score if not passed: mx.abort("Benchmark failed (not passed)") return groups
def _gate_dacapo(name, iterations, extraVMarguments=None): vmargs = ['-Xms2g', '-XX:+UseSerialGC', '-XX:-UseCompressedOops', '-Djava.net.preferIPv4Stack=true', '-Dgraal.ExitVMOnException=true'] + _noneAsEmptyList(extraVMarguments) dacapoJar = mx.library('DACAPO').get_path(True) _gate_java_benchmark(vmargs + ['-jar', dacapoJar, name, '-n', str(iterations)], r'^===== DaCapo 9\.12 ([a-zA-Z0-9_]+) PASSED in ([0-9]+) msec =====')
def sonarqube_upload(args): """run SonarQube scanner and upload JaCoCo results""" sonarqube_cli = mx.library("SONARSCANNER_CLI_3_3_0_1492", True) parser = ArgumentParser(prog='mx sonarqube-upload') parser.add_argument('--exclude-generated', action='store_true', help='Exclude generated source files') parser.add_argument('--skip-coverage', action='store_true', default=False, help='Do not upload coverage reports') args, sonar_args = mx.extract_VM_args(args, useDoubleDash=True, defaultAllVMArgs=True) args, other_args = parser.parse_known_args(args) java_props, other_args = _parse_java_properties(other_args) def _check_required_prop(prop): if prop not in java_props: mx.abort("Required property '{prop}' not present. (Format is '-D{prop}=<value>')".format(prop=prop)) _check_required_prop('sonar.projectKey') _check_required_prop('sonar.host.url') basedir = mx.primary_suite().dir # collect excluded projects excludes, includes = _jacoco_excludes_includes_projects(limit_to_primary=True) # collect excluded classes exclude_classes = _jacoco_exclude_classes(includes) java_bin = [] java_src = [] java_libs = [] def _visit_deps(dep, edge): if dep.isJARDistribution() or dep.isLibrary(): java_libs.append(dep.classpath_repr()) mx.walk_deps(includes, visit=_visit_deps) # collect all sources and binaries -- do exclusion later for p in includes: java_src.extend(p.source_dirs()) if not args.exclude_generated: gen_dir = p.source_gen_dir() if os.path.exists(gen_dir): java_src.append(gen_dir) java_bin.append(p.output_dir()) java_src = [os.path.relpath(s, basedir) for s in java_src] java_bin = [os.path.relpath(b, basedir) for b in java_bin] # Overlayed sources and classes must be excluded jdk_compliance = mx.get_jdk().javaCompliance overlayed_sources = [] overlayed_classfiles = {} for p in includes: if hasattr(p, "multiReleaseJarVersion") and jdk_compliance not in p.javaCompliance: # JDK9+ overlays for srcDir in p.source_dirs(): for root, _, files in os.walk(srcDir): for name in files: if name.endswith('.java') and name != 'package-info.java': overlayed_sources.append(join(os.path.relpath(root, basedir), name)) elif hasattr(p, "overlayTarget"): # JDK8 overlays target = mx.project(p.overlayTarget) overlay_sources = [] for srcDir in p.source_dirs(): for root, _, files in os.walk(srcDir): for name in files: if name.endswith('.java') and name != 'package-info.java': overlay_sources.append(join(os.path.relpath(root, srcDir), name)) print(p, target, overlay_sources) for srcDir in target.source_dirs(): for root, _, files in os.walk(srcDir): for name in files: if name.endswith('.java') and name != 'package-info.java': s = join(os.path.relpath(root, srcDir), name) if s in overlay_sources: overlayed = join(os.path.relpath(root, basedir), name) overlayed_sources.append(overlayed) for s in overlay_sources: classfile = join(os.path.relpath(target.output_dir(), basedir), s[:-len('java')] + 'class') with open(classfile, 'rb') as fp: overlayed_classfiles[classfile] = fp.read() exclude_dirs = [] for p in excludes: exclude_dirs.extend(p.source_dirs()) exclude_dirs.append(p.source_gen_dir()) javaCompliance = max([p.javaCompliance for p in includes]) if includes else mx.JavaCompliance('1.7') jacoco_exec = JACOCO_EXEC if not os.path.exists(jacoco_exec) and not args.skip_coverage: mx.abort('No JaCoCo report file found: ' + jacoco_exec) def _add_default_prop(key, value): if key not in java_props: java_props[key] = value # default properties _add_default_prop('sonar.java.source', str(javaCompliance)) _add_default_prop('sonar.projectBaseDir', basedir) if not args.skip_coverage: _add_default_prop('sonar.jacoco.reportPaths', jacoco_exec) _add_default_prop('sonar.sources', ','.join(java_src)) _add_default_prop('sonar.java.binaries', ','.join(java_bin)) _add_default_prop('sonar.java.libraries', ','.join(java_libs)) exclude_patterns = [os.path.relpath(e, basedir) + '**' for e in exclude_dirs] + \ overlayed_sources + \ list(set([os.path.relpath(match[0], basedir) for _, match in exclude_classes.items()])) if exclude_patterns: _add_default_prop('sonar.exclusions', ','.join(exclude_patterns)) _add_default_prop('sonar.coverage.exclusions', ','.join(exclude_patterns)) _add_default_prop('sonar.verbose', 'true' if mx._opts.verbose else 'false') with tempfile.NamedTemporaryFile(suffix="-sonarqube.properties", mode="w+") as fp: # prepare properties file fp.writelines(('{}={}\n'.format(k, v) for k, v in java_props.items())) fp.flush() # Since there's no options to exclude individual classes, # we temporarily delete the overlayed class files instead. for classfile in overlayed_classfiles: os.remove(classfile) try: # run sonarqube cli java_args = other_args + ['-Dproject.settings=' + fp.name, '-jar', sonarqube_cli.get_path(True)] + sonar_args exit_code = mx.run_java(java_args, nonZeroIsFatal=False) finally: # Restore temporarily deleted class files for classfile, data in overlayed_classfiles.items(): with open(classfile, 'wb') as cf: cf.write(data) if exit_code != 0: fp.seek(0) mx.abort('SonarQube scanner terminated with non-zero exit code: {}\n Properties file:\n{}'.format( exit_code, ''.join((' ' + l for l in fp.readlines()))))
def findbugs(args, fbArgs=None, suite=None, projects=None): """run FindBugs against non-test Java projects""" findBugsHome = mx.get_env('FINDBUGS_HOME', None) if suite is None: suite = mx._primary_suite if findBugsHome: findbugsJar = join(findBugsHome, 'lib', 'findbugs.jar') else: findbugsLib = join(mx._mx_suite.get_output_root(), 'findbugs-3.0.0') if not exists(findbugsLib): tmp = tempfile.mkdtemp(prefix='findbugs-download-tmp', dir=mx._mx_suite.dir) try: findbugsDist = mx.library('FINDBUGS_DIST').get_path(resolve=True) with zipfile.ZipFile(findbugsDist) as zf: candidates = [e for e in zf.namelist() if e.endswith('/lib/findbugs.jar')] assert len(candidates) == 1, candidates libDirInZip = os.path.dirname(candidates[0]) zf.extractall(tmp) shutil.copytree(join(tmp, libDirInZip), findbugsLib) finally: shutil.rmtree(tmp) findbugsJar = join(findbugsLib, 'findbugs.jar') assert exists(findbugsJar) nonTestProjects = [p for p in mx.projects() if _should_test_project(p)] if not nonTestProjects: return 0 ignoredClasses = set() for p in nonTestProjects: ignore = getattr(p, 'findbugsIgnoresGenerated', False) if not isinstance(ignore, bool): mx.abort('Value of attribute "findbugsIgnoresGenerated" must be True or False', context=p) if ignore is True: sourceDir = p.source_gen_dir() for root, _, files in os.walk(sourceDir): for name in files: if name.endswith('.java') and '-info' not in name: pkg = root[len(sourceDir) + 1:].replace(os.sep, '.') cls = pkg + '.' + name[:-len('.java')] ignoredClasses.add(cls) with tempfile.NamedTemporaryFile(suffix='.xml', prefix='findbugs_exclude_filter.', mode='w', delete=False) as fp: findbugsExcludeFilterFile = fp.name xmlDoc = mx.XMLDoc() xmlDoc.open('FindBugsFilter') for cls in ignoredClasses: xmlDoc.open('Match') xmlDoc.element('Class', attributes={'name' : '~' + cls + '.*'}) xmlDoc.close('Match') xmlDoc.close('FindBugsFilter') xml = xmlDoc.xml(indent=' ', newl='\n') print >> fp, xml outputDirs = map(mx._cygpathU2W, [p.output_dir() for p in nonTestProjects]) javaCompliance = max([p.javaCompliance for p in nonTestProjects]) jdk = mx.get_jdk(javaCompliance) if jdk.javaCompliance >= "1.9": mx.log('FindBugs does not yet support JDK9 - skipping') return 0 findbugsResults = join(suite.dir, 'findbugs.results') if fbArgs is None: fbArgs = defaultFindbugsArgs() cmd = ['-jar', mx._cygpathU2W(findbugsJar)] + fbArgs cmd = cmd + ['-exclude', findbugsExcludeFilterFile] cmd = cmd + ['-auxclasspath', mx._separatedCygpathU2W(mx.classpath([p.name for p in nonTestProjects], jdk=jdk)), '-output', mx._cygpathU2W(findbugsResults), '-exitcode'] + args + outputDirs try: exitcode = mx.run_java(cmd, nonZeroIsFatal=False, jdk=jdk) finally: os.unlink(findbugsExcludeFilterFile) if exitcode != 0: with open(findbugsResults) as fp: mx.log(fp.read()) os.unlink(findbugsResults) return exitcode
def get_sigtest_jar(self): return mx.library('SIGTEST_1_3').get_path(resolve=True)