def wikidoc(args): """generate Confluence Wiki format for package-info.java files""" # Ensure the wiki doclet is up to date mx.build(['--projects', 'com.oracle.max.tools']) # the WikiDoclet cannot see the -classpath argument passed to javadoc so we pass the # full list of projects as an explicit argument, thereby enabling it to map classes # to projects, which is needed to generate Wiki links to the source code. # There is no virtue in running the doclet on dependent projects as there are # no generated links between Wiki pages toolsDir = mx.project('com.oracle.max.tools').output_dir() baseDir = mx.project('com.oracle.max.base').output_dir() dp = os.pathsep.join([toolsDir, baseDir]) project_list = ','.join(p.name for p in mx.sorted_deps()) for a in [ '-docletpath', dp, '-doclet', 'com.oracle.max.tools.javadoc.wiki.WikiDoclet', '-projects', project_list ]: args.append('--arg') args.append('@' + a) mx.javadoc(args, parser=ArgumentParser('mx wikidoc'), docDir='wikidoc', includeDeps=False, stdDoclet=False)
def validate_parser(grammar_project, grammar_path, create_command, args=None, out=None): def read_file(path): with open(path, "r") as f: return f.readlines() parser_path = grammar_path.replace(".g4", "Parser.java") lexer_path = grammar_path.replace(".g4", "Lexer.java") parser = mx.project(grammar_project).source_dirs()[0] + "/" + parser_path lexer = mx.project(grammar_project).source_dirs()[0] + "/" + lexer_path parser_before = read_file(parser) lexer_before = read_file(lexer) create_command([], out) parser_after = read_file(parser) lexer_after = read_file(lexer) if (parser_before != parser_after or lexer_before != lexer_after): with open(parser, "w") as f: f.writelines(parser_before) with open(lexer, "w") as f: f.writelines(lexer_before) mx.abort( "Parser generated from " + grammar_path + " does not match content of " + parser_path + " or " + lexer_path + "." + " Make sure the grammar files are up to date with the generated code. You can regenerate the generated code using mx." )
def _espresso_gate_runner(args, tasks): # Jackpot configuration is inherited from Truffle. with Task('Jackpot', tasks, tags=[EspressoTags.jackpot]) as t: if t: jackpot(['--fail-on-warnings'], suite=None, nonZeroIsFatal=True) with Task('Espresso: GraalVM dist names', tasks, tags=['names']) as t: if t: mx_sdk_vm.verify_graalvm_configs(suites=['espresso']) mokapot_header_gate_name = 'Verify consistency of mokapot headers' with Task(mokapot_header_gate_name, tasks, tags=[EspressoTags.verify]) as t: if t: import mx_sdk_vm_impl run_instructions = "$ mx --dynamicimports=/substratevm --native-images=lib:espresso gate --all-suites --task '{}'".format(mokapot_header_gate_name) if mx_sdk_vm_impl._skip_libraries(espresso_library_config): mx.abort("""\ The registration of the Espresso library ('lib:espresso') is skipped. Please run this gate as follows: {}""".format(run_instructions)) errors = False mokapot_dir = join(mx.project('com.oracle.truffle.espresso.mokapot').dir, 'include') libespresso_dir = mx.project(mx_sdk_vm_impl.GraalVmNativeImage.project_name(espresso_library_config)).get_output_root() for header in ['libespresso_dynamic.h', 'graal_isolate_dynamic.h']: committed_header = join(mokapot_dir, header) if not mx.exists(committed_header): mx.abort("Cannot locate '{}'. Was the file moved or renamed?".format(committed_header)) generated_header = join(libespresso_dir, header) if not mx.exists(generated_header): mx.abort("Cannot locate '{}'. Did you forget to build? Example:\n'mx --dynamicimports=/substratevm --native-images=lib:espresso build'".format(generated_header)) committed_header_copyright = [] with open(committed_header, 'r') as committed_header_file: for line in committed_header_file.readlines(): if line == '/*\n' or line.startswith(' *') or line == '*/\n': committed_header_copyright.append(line) else: break with open(generated_header, 'r') as generated_header_file: generated_header_lines = [] for line in generated_header_file.readlines(): # Ignore definitions that are not needed for Espresso if not line.startswith("typedef") or "(*Espresso_" in line or "__graal" in line or "(*graal_" in line: generated_header_lines.append(line) else: newline = generated_header_lines.pop() # Remove newline before ignored declaration assert newline == "\n" errors = errors or mx.update_file(committed_header, ''.join(committed_header_copyright + generated_header_lines), showDiff=True) if errors: mx.abort("""\ One or more header files in the include dir of the mokapot project ('{committed}/') do not match those generated by Native Image ('{generated}/'). To fix the issue, run this gate locally: {instructions} And adapt the code to the modified headers in '{committed}'. """.format(committed=os.path.relpath(mokapot_dir, _suite.vc_dir), generated=os.path.relpath(libespresso_dir, _suite.vc_dir), instructions=run_instructions))
def rbcheck(args): '''check FastR builtins against GnuR''' parser = ArgumentParser(prog='mx rbcheck') parser.add_argument('--check-internal', action='store_const', const='--check-internal', help='check .Internal functions') parser.add_argument('--unknown-to-gnur', action='store_const', const='--unknown-to-gnur', help='list builtins not in GnuR FUNCTAB') parser.add_argument('--todo', action='store_const', const='--todo', help='show unimplemented') parser.add_argument('--no-eval-args', action='store_const', const='--no-eval-args', help='list functions that do not evaluate their args') parser.add_argument('--visibility', action='store_const', const='--visibility', help='list visibility specification') parser.add_argument('--printGnuRFunctions', action='store', help='ask GnuR to "print" value of functions') parser.add_argument('--packageBase', action='store', help='directory to be recursively scanned for R sources (used to get frequencies for builtins)') parser.add_argument('--interactive', action='store_const', const='--interactive', help='interactive querying of the word frequencies') args = parser.parse_args(args) class_map = mx.project('com.oracle.truffle.r.nodes').find_classes_with_matching_source_line(None, lambda line: "@RBuiltin" in line, True) classes = [] for className, path in class_map.iteritems(): classNameX = className.split("$")[0] if '$' in className else className if not classNameX.endswith('Factory'): classes.append([className, path]) class_map = mx.project('com.oracle.truffle.r.nodes.builtin').find_classes_with_matching_source_line(None, lambda line: "@RBuiltin" in line, True) for className, path in class_map.iteritems(): classNameX = className.split("$")[0] if '$' in className else className if not classNameX.endswith('Factory'): classes.append([className, path]) (_, testfile) = tempfile.mkstemp(".classes", "mx") os.close(_) with open(testfile, 'w') as f: for c in classes: f.write(c[0] + ',' + c[1][0] + '\n') analyzeArgs = [] if args.check_internal: analyzeArgs.append(args.check_internal) if args.unknown_to_gnur: analyzeArgs.append(args.unknown_to_gnur) if args.todo: analyzeArgs.append(args.todo) if args.no_eval_args: analyzeArgs.append(args.no_eval_args) if args.visibility: analyzeArgs.append(args.visibility) if args.interactive: analyzeArgs.append(args.interactive) if args.printGnuRFunctions: analyzeArgs.append('--printGnuRFunctions') analyzeArgs.append(args.printGnuRFunctions) if args.packageBase: analyzeArgs.append('--packageBase') analyzeArgs.append(args.packageBase) analyzeArgs.append(testfile) cp = mx.classpath('com.oracle.truffle.r.test') mx.run_java(['-cp', cp, 'com.oracle.truffle.r.test.tools.AnalyzeRBuiltin'] + analyzeArgs)
def jttgen(args): """(re)generate harness and run scheme for the JavaTester tests Run the JavaTester to update the JavaTesterRunScheme.java and JavaTesterTests.java files in the com.sun.max.vm.jtrun.all package.""" testDirs = [join(mx.project('com.oracle.max.vm.tests').dir, 'src'), join(mx.project('com.oracle.max.tests').dir, 'src')] tests = [] for testDir in testDirs: for name in os.listdir(join(testDir, 'jtt')): if name != 'hotspot' and name != 'fail': tests.append(join(testDir, 'jtt', name)) return mx.run_java(['-cp', mx.classpath('com.oracle.max.vm.tests'), 'test.com.sun.max.vm.compiler.JavaTester', '-scenario=target', '-run-scheme-package=all', '-native-tests'] + tests)
def checkheaders(args): """check Java source headers against any required pattern""" failures = {} for p in mx.projects(): if not p.isJavaProject(): continue csConfig = join(mx.project(p.checkstyleProj).dir, '.checkstyle_checks.xml') if not exists(csConfig): mx.log('Cannot check headers for ' + p.name + ' - ' + csConfig + ' does not exist') continue dom = xml.dom.minidom.parse(csConfig) for module in dom.getElementsByTagName('module'): if module.getAttribute('name') == 'RegexpHeader': for prop in module.getElementsByTagName('property'): if prop.getAttribute('name') == 'header': value = prop.getAttribute('value') matcher = re.compile(value, re.MULTILINE) for sourceDir in p.source_dirs(): for root, _, files in os.walk(sourceDir): for name in files: if name.endswith('.java') and name != 'package-info.java': f = join(root, name) with open(f) as fp: content = fp.read() if not matcher.match(content): failures[f] = csConfig for n, v in failures.iteritems(): mx.log('{0}: header does not match RegexpHeader defined in {1}'.format(n, v)) return len(failures)
def checkheaders(args): """check Java source headers against any required pattern""" failures = {} for p in mx.projects(): if not p.isJavaProject(): continue csConfig = join(mx.project(p.checkstyleProj).dir, ".checkstyle_checks.xml") if not exists(csConfig): mx.log("Cannot check headers for " + p.name + " - " + csConfig + " does not exist") continue dom = xml.dom.minidom.parse(csConfig) for module in dom.getElementsByTagName("module"): if module.getAttribute("name") == "RegexpHeader": for prop in module.getElementsByTagName("property"): if prop.getAttribute("name") == "header": value = prop.getAttribute("value") matcher = re.compile(value, re.MULTILINE) for sourceDir in p.source_dirs(): for root, _, files in os.walk(sourceDir): for name in files: if name.endswith(".java") and name != "package-info.java": f = join(root, name) with open(f) as fp: content = fp.read() if not matcher.match(content): failures[f] = csConfig for n, v in failures.iteritems(): mx.log("{0}: header does not match RegexpHeader defined in {1}".format(n, v)) return len(failures)
def jaotc_test(args): """run (acceptance) tests for the AOT compiler (jaotc)""" all_tests = ['HelloWorld', 'java.base', 'javac'] parser = ArgumentParser(prog='mx jaotc-test') parser.add_argument("--list", default=None, action="store_true", help="Print the list of available jaotc tests.") parser.add_argument('tests', help='tests to run (omit to run all tests)', nargs=ZERO_OR_MORE) args = parser.parse_args(args) if args.list: print "The following jaotc tests are available:\n" for name in all_tests: print " " + name return tests = args.tests or all_tests for test in tests: mx.log('Testing `{}`'.format(test)) if test == 'HelloWorld': test_class( classpath=mx.classpath('JAOTC_TEST'), main_class='jdk.tools.jaotc.test.HelloWorld' ) elif test == 'javac': test_javac('jdk.tools.jaotc') elif test == 'java.base': test_modules( classpath=mx.project('jdk.tools.jaotc.test').output_dir(), main_class='jdk.tools.jaotc.test.HelloWorld', modules=['java.base'] ) else: mx.abort('Unknown jaotc test: {}'.format(test))
def _alldeps(args): parser = ArgumentParser(prog='mx mxt-alldeps') parser.add_argument('--kind', action='store', help='project, dist or library', default='project') parser.add_argument('--name', action='store', help='name of entity', required=True) parser.add_argument('--includeLibs', action='store_true', help='includeLibs') parser.add_argument('--ignoreSelf', action='store_true', help='ignoreSelf') args = parser.parse_args(args) entity = None if args.kind == 'project': entity = mx.project(args.name) elif args.kind == 'library': entity = mx.library(args.name) elif args.kind == 'dist': entity = mx.distribution(args.name) else: mx.abort('illegal kind: ' + args.kind) in_deps = [] deps = entity.all_deps(in_deps, includeLibs=args.includeLibs, includeSelf=not args.ignoreSelf) print('alldeps:') for d in deps: print(d.__class__.__name__, ":", d.name)
def create_parser(grammar_project, grammar_package, grammar_name, copyright_template, args=None, out=None): """create the DSL expression parser using antlr""" grammar_dir = os.path.join( mx.project(grammar_project).source_dirs()[0], *grammar_package.split(".")) + os.path.sep mx.run_java( mx.get_runtime_jvm_args(['ANTLR4_COMPLETE']) + ["org.antlr.v4.Tool", "-package", grammar_package, "-no-listener"] + args + [grammar_dir + grammar_name + ".g4"], out=out) for filename in [ grammar_dir + grammar_name + "Lexer.java", grammar_dir + grammar_name + "Parser.java" ]: with open(filename, 'r') as content_file: content = content_file.read() # remove first line content = "\n".join(content.split("\n")[1:]) # modify SuppressWarnings to remove useless entries content = PTRN_SUPPRESS_WARNINGS.sub('@SuppressWarnings("all")', content) # remove useless casts content = PTRN_LOCALCTXT_CAST.sub('_localctx', content) content = PTRN_TOKEN_CAST.sub('_errHandler.recoverInline(this)', content) # add copyright header content = copyright_template.format(content) with open(filename, 'w') as content_file: content_file.write(content)
def getSearchPathOption(lib_args=None): if lib_args is None: lib_args = ['-lgmp', '-lgfortran'] lib_names = [] lib_aliases = { '-lc': ['libc.so.6', 'libc.dylib'], '-lstdc++': ['libstdc++.so.6', 'libstdc++.6.dylib'], '-lgmp': ['libgmp.so.10', 'libgmp.10.dylib'], '-lgfortran': ['libgfortran.so.3', 'libgfortran.3.dylib'], '-lpcre': ['libpcre.so.3', 'libpcre.dylib'] } osStr = mx.get_os() index = {'linux': 0, 'darwin': 1}[mx.get_os()] if index is None: mx.log_error("{0} not supported!".format(osStr)) for lib_arg in ['-lc', '-lstdc++'] + lib_args: if lib_arg in lib_aliases: lib_arg = lib_aliases[lib_arg][index] else: lib_arg = lib_arg[2:] lib_names.append(lib_arg) libpath = join( mx.project('com.oracle.truffle.llvm.test.native').getOutput(), 'bin') for path, _, files in os.walk(libpath): for f in files: if f.endswith('.so'): lib_names.append(join(path, f)) return '-Dsulong.DynamicNativeLibraryPath=' + ':'.join(lib_names)
def _graal_nodejs_post_gate_runner(args, tasks): _setEnvVar('NODE_INTERNAL_ERROR_CHECK', 'true') with Task('UnitTests', tasks, tags=[GraalNodeJsTags.allTests, GraalNodeJsTags.unitTests]) as t: if t: commonArgs = ['-ea'] unitTestDir = join('test', 'graal') mx.run(['rm', '-rf', 'node_modules', 'build'], cwd=unitTestDir) npm(['--scripts-prepend-node-path=auto', 'install', '--nodedir=' + _suite.dir] + commonArgs, cwd=unitTestDir) npm(['--scripts-prepend-node-path=auto', 'test'] + commonArgs, cwd=unitTestDir) npm(['--scripts-prepend-node-path=auto', 'test', '-Dtruffle.js.NashornJavaInterop=true'] + commonArgs, cwd=unitTestDir) with Task('TestNpm', tasks, tags=[GraalNodeJsTags.allTests]) as t: if t: tmpdir = tempfile.mkdtemp() try: npm(['--scripts-prepend-node-path=auto', 'install', '--nodedir=' + _suite.dir, 'microtime'], cwd=tmpdir) node(['-e', 'print(require("microtime").now());'], cwd=tmpdir) finally: shutil.rmtree(tmpdir) with Task('JniProfilerTests', tasks, tags=[GraalNodeJsTags.allTests, GraalNodeJsTags.jniProfilerTests]) as t: if t: commonArgs = ['-ea'] unitTestDir = join(mx.project('com.oracle.truffle.trufflenode.jniboundaryprofiler').dir, 'tests') mx.run(['rm', '-rf', 'node_modules', 'build'], cwd=unitTestDir) npm(['--scripts-prepend-node-path=auto', 'install', '--nodedir=' + _suite.dir] + commonArgs, cwd=unitTestDir) node(['-profile-native-boundary', '-Dtruffle.js.NashornJavaInterop=true', 'test.js'] + commonArgs, cwd=unitTestDir)
def jaotc_test(args): """run (acceptance) tests for the AOT compiler (jaotc)""" _check_jaotc_support() all_tests = ['HelloWorld', 'java.base', 'javac'] parser = ArgumentParser(prog='mx jaotc-test') parser.add_argument("--list", default=None, action="store_true", help="Print the list of available jaotc tests.") parser.add_argument('tests', help='tests to run (omit to run all tests)', nargs=ZERO_OR_MORE) args = parser.parse_args(args) if args.list: print("The following jaotc tests are available:\n") for name in all_tests: print(" " + name) return tests = args.tests or all_tests for test in tests: mx.log('Testing `{}`'.format(test)) if test == 'HelloWorld': test_class(classpath=mx.classpath('JAOTC_TEST'), main_class='jdk.tools.jaotc.test.HelloWorld') elif test == 'javac': test_javac('jdk.tools.jaotc') elif test == 'java.base': test_modules( classpath=mx.project('jdk.tools.jaotc.test').output_dir(), main_class='jdk.tools.jaotc.test.HelloWorld', modules=['java.base']) else: mx.abort('Unknown jaotc test: {}'.format(test))
def mx_post_parse_cmd_line(opts): mx_fastr_dists.mx_post_parse_cmd_line(opts) if mx.suite("sulong", fatalIfMissing=False) and not _fastr_suite.isBinarySuite(): # native.recommended runs FastR, it already has a build dependency to the FASTR distribution # if we are running with sulong we also need the SULONG distribution rec = mx.project('com.oracle.truffle.r.native.recommended') rec.buildDependencies += [mx.distribution('SULONG')]
def _graal_nodejs_post_gate_runner(args, tasks): _setEnvVar('NODE_INTERNAL_ERROR_CHECK', 'true') with Task('UnitTests', tasks, tags=[GraalNodeJsTags.allTests, GraalNodeJsTags.unitTests]) as t: if t: _setEnvVar('NODE_JVM_CLASSPATH', mx.distribution('graal-js:TRUFFLE_JS_TESTS').path) commonArgs = ['-ea', '-esa'] unitTestDir = join('test', 'graal') mx.run(['rm', '-rf', 'node_modules', 'build'], cwd=unitTestDir) npm(['--scripts-prepend-node-path=auto', 'install', '--nodedir=' + _suite.dir] + commonArgs, cwd=unitTestDir) npm(['--scripts-prepend-node-path=auto', 'test'] + commonArgs, cwd=unitTestDir) with Task('TestNpm', tasks, tags=[GraalNodeJsTags.allTests]) as t: if t: tmpdir = tempfile.mkdtemp() try: npm(['init', '-y'], cwd=tmpdir) npm(['--scripts-prepend-node-path=auto', 'install', '--nodedir=' + _suite.dir, '--build-from-source', 'microtime'], cwd=tmpdir) node(['-e', 'console.log(require("microtime").now());'], cwd=tmpdir) finally: mx.rmtree(tmpdir) with Task('JniProfilerTests', tasks, tags=[GraalNodeJsTags.allTests, GraalNodeJsTags.jniProfilerTests]) as t: if t: commonArgs = ['-ea', '-esa'] unitTestDir = join(mx.project('com.oracle.truffle.trufflenode.jniboundaryprofiler').dir, 'tests') mx.run(['rm', '-rf', 'node_modules', 'build'], cwd=unitTestDir) npm(['--scripts-prepend-node-path=auto', 'install', '--nodedir=' + _suite.dir] + commonArgs, cwd=unitTestDir) node(['-profile-native-boundary', 'test.js'] + commonArgs, cwd=unitTestDir)
def gnu_rtests(args, env=None): ''' run tests of the internally built GNU R under tests subdirectory ''' os.chdir(_fastr_suite.dir) # Packages install fails otherwise # mx_fastr_pkgs.installpkgs(['--pkg-pattern', '^MASS$']) # required by tests/Examples/base-Ex.R np = mx.project('com.oracle.truffle.r.native') tst = join(np.dir, 'gnur', 'tests') tstsrc = join(tst, 'src') tstlog = join(tst, 'log') shutil.rmtree(tstlog, True) os.mkdir(tstlog) diffname = join(tstlog, 'all.diff') diff = open(diffname, 'a') try: for subd in ['Examples', '']: logd = join(tstlog, subd) if subd != '': os.mkdir(logd) os.chdir(logd) srcd = join(tstsrc, subd) for f in sorted(os.listdir(srcd)): if f.endswith('.R'): print 'Running {} explicitly by FastR CMD BATCH ...'.format( f) mx.run( [r_path(), '--vanilla', 'CMD', 'BATCH', join(srcd, f)] + args, nonZeroIsFatal=False, env=env, timeout=90) outf = f + 'out' if os.path.isfile(outf): outff = outf + '.fastr' os.rename(outf, outff) print 'Running {} explicitly by GnuR CMD BATCH ...'.format( f) mx.run([ join(_gnur_path(), 'R'), '--vanilla', 'CMD', 'BATCH', join(srcd, f) ] + args, nonZeroIsFatal=False, env=env, timeout=90) if os.path.isfile(outf): outfg = outf + '.gnur' os.rename(outf, outfg) diff.write('\nRdiff {} {}:\n'.format(outfg, outff)) diff.flush() subprocess.Popen( [r_path(), 'CMD', 'Rdiff', outfg, outff], stdout=diff, stderr=diff, shell=False) diff.flush() diff.close() print 'FastR to GnuR diff was written to {}'.format(diffname) finally: shutil.rmtree(join(_fastr_suite.dir, 'deparse'), True)
def _unittest(args, annotations, prefixCp="", blacklist=None, whitelist=None, verbose=False, fail_fast=False, enable_timing=False, regex=None, color=False, eager_stacktrace=False, gc_after_test=False, suite=None): testfile = os.environ.get('MX_TESTFILE', None) if testfile is None: (_, testfile) = tempfile.mkstemp(".testclasses", "mxtool") os.close(_) mainClass = 'com.oracle.mxtool.junit.MxJUnitWrapper' if not exists(join(mx.project('com.oracle.mxtool.junit').output_dir(), mainClass.replace('.', os.sep) + '.class')): mx.build(['--only', 'com.oracle.mxtool.junit']) coreCp = mx.classpath(['com.oracle.mxtool.junit']) coreArgs = [] if verbose: coreArgs.append('-JUnitVerbose') if fail_fast: coreArgs.append('-JUnitFailFast') if enable_timing: coreArgs.append('-JUnitEnableTiming') if color: coreArgs.append('-JUnitColor') if eager_stacktrace: coreArgs.append('-JUnitEagerStackTrace') if gc_after_test: coreArgs.append('-JUnitGCAfterTest') def harness(unittestCp, vmLauncher, vmArgs): prefixArgs = ['-esa', '-ea'] if gc_after_test: prefixArgs.append('-XX:-DisableExplicitGC') with open(testfile) as fp: testclasses = [l.rstrip() for l in fp.readlines()] cp = prefixCp + coreCp + os.pathsep + unittestCp # suppress menubar and dock when running on Mac vmArgs = prefixArgs + ['-Djava.awt.headless=true'] + vmArgs + ['-cp', mx._separatedCygpathU2W(cp)] # Execute Junit directly when one test is being run. This simplifies # replaying the VM execution in a native debugger (e.g., gdb). mainClassArgs = coreArgs + (testclasses if len(testclasses) == 1 else ['@' + mx._cygpathU2W(testfile)]) config = (vmArgs, mainClass, mainClassArgs) for p in _config_participants: config = p(config) vmLauncher.launcher(*config) vmLauncher = _vm_launcher if vmLauncher is None: jdk = mx.get_jdk() def _run_vm(vmArgs, mainClass, mainClassArgs): mx.run_java(vmArgs + [mainClass] + mainClassArgs, jdk=jdk) vmLauncher = _VMLauncher('default VM launcher', _run_vm, jdk) try: _run_tests(args, harness, vmLauncher, annotations, testfile, blacklist, whitelist, regex, mx.suite(suite) if suite else None) finally: if os.environ.get('MX_TESTFILE') is None: os.remove(testfile)
def mx_post_parse_cmd_line(opts): mx_fastr_dists.mx_post_parse_cmd_line(opts) mx_subst.results_substitutions.register_no_arg('graalvm_version', mx.suite('sdk').release_version()) if mx.suite("sulong", fatalIfMissing=False) and not _fastr_suite.isBinarySuite(): # native.recommended runs FastR, it already has a build dependency to the FASTR distribution # if we are running with sulong we also need the SULONG_NATIVE distribution rec = mx.project('com.oracle.truffle.r.native.recommended') rec.buildDependencies += [mx.distribution('SULONG_NATIVE')]
def _ap(args): parser = ArgumentParser(prog='mx mxt-proj-ap-path') parser.add_argument('--project', action='store', help='name of entity') args = parser.parse_args(args) project = mx.project(args.project) result = project.annotation_processors_path() print 'annotation_processors_path for: ', args.project comps = result.split(':') for comp in comps: print comp
def _ap(args): parser = ArgumentParser(prog='mx mxt-proj-ap-path') parser.add_argument('--project', action='store', help='name of entity') args = parser.parse_args(args) project = mx.project(args.project) result = project.annotation_processors_path() print('annotation_processors_path for: ', args.project) comps = result.split(':') for comp in comps: print(comp)
def getBitcodeLibrariesOption(*args): libraries = [] if 'SULONG_NO_LIBRARY' not in os.environ: libpath = join( mx.project('com.oracle.truffle.llvm.libraries').getOutput(), 'bin') for path, _, files in os.walk(libpath): for f in files: if f.endswith('.bc'): libraries.append(join(path, f)) return ['-Dsulong.DynamicBitcodeLibraries=' + ':'.join(libraries)] if libraries else []
def wikidoc(args): """generate Confluence Wiki format for package-info.java files""" # Ensure the wiki doclet is up to date mx.build(['--projects', 'com.oracle.max.tools']) # the WikiDoclet cannot see the -classpath argument passed to javadoc so we pass the # full list of projects as an explicit argument, thereby enabling it to map classes # to projects, which is needed to generate Wiki links to the source code. # There is no virtue in running the doclet on dependent projects as there are # no generated links between Wiki pages toolsDir = mx.project('com.oracle.max.tools').output_dir() baseDir = mx.project('com.oracle.max.base').output_dir() dp = os.pathsep.join([toolsDir, baseDir]) project_list = ','.join(p.name for p in mx.sorted_deps()) for a in ['-docletpath', dp, '-doclet', 'com.oracle.max.tools.javadoc.wiki.WikiDoclet', '-projects', project_list]: args.append('--arg') args.append('@' + a) mx.javadoc(args, parser=ArgumentParser('mx wikidoc'), docDir='wikidoc', includeDeps=False, stdDoclet=False)
def testgen(args): '''generate the expected output for unit tests''' # check we are in the home directory if os.getcwd() != _fastr_suite.dir: mx.abort('must run rtestgen from FastR home directory') def _test_srcdir(): tp = 'com.oracle.truffle.r.test' return join(mx.project(tp).dir, 'src', tp.replace('.', sep)) def need_version_check(): vardef = os.environ.has_key('FASTR_TESTGEN_GNUR') varval = os.environ['FASTR_TESTGEN_GNUR'] if vardef else None version_check = vardef and varval != 'internal' if version_check: rpath = join(varval, 'bin', 'R') else: rpath = None return version_check, rpath version_check, rpath = need_version_check() if version_check: # check the version of GnuR against FastR try: fastr_version = subprocess.check_output([ mx.get_jdk().java, mx.get_runtime_jvm_args('com.oracle.truffle.r.runtime'), 'com.oracle.truffle.r.runtime.RVersionNumber' ]) gnur_version = subprocess.check_output([rpath, '--version']) if not gnur_version.startswith(fastr_version): mx.abort( 'R version is incompatible with FastR, please update to ' + fastr_version) except subprocess.CalledProcessError: mx.abort('RVersionNumber.main failed') tests = _all_generated_unit_tests() # now just invoke unittst with the appropriate options mx.log("generating expected output for packages: ") for pkg in tests: mx.log(" " + str(pkg)) os.environ["TZDIR"] = "/usr/share/zoneinfo/" _unset_conflicting_envs() mx_unittest.unittest([ '-Dfastr.test.gen.expected=' + _test_srcdir(), '-Dfastr.test.gen.expected.quiet', '-Dfastr.test.project.output.dir=' + mx.project('com.oracle.truffle.r.test').output_dir() ] + tests)
def runAndReturnStdOut(self, benchmarks: List[str], bmSuiteArgs: List[str]) -> Tuple[int, OutputCapture, Dict[str, str]]: if len(benchmarks) > 1: mx.abort("Only one benchmark can be run at a time") bench_name = benchmarks[0] bench_path = Path(mx.project("altrep.benchmarks").dir).joinpath(bench_name + ".r") bench_runner_source = generate_bench_runner_source(self._bench_args, bench_path) with tempfile.NamedTemporaryFile() as bench_runner_script: bench_runner_script.write(bytes(bench_runner_source, "UTF-8")) bench_runner_script.seek(0) bench_runner_args = [ "--slave", "--vanilla", "-f", bench_runner_script.name ] retcode = -1 output_capture = AltrepBenchmarkSuite.OutputCapture() if self._bench_args.driver == AltrepBenchmarkSuite.Driver.FASTR: extra_vm_args = [] extra_vm_args += ["-Dgraal.TraceTruffleCompilation=true"] extra_vm_args += ["-Dgraal.CompilationFailureAction=ExitVM"] extra_vm_args += ["-da"] extra_vm_args += ["-dsa"] #os.environ["MX_R_GLOBAL_ARGS"] = " ".join(extra_vm_args) self.vm_args = extra_vm_args fastr_args = ["--R.PrintErrorStacktracesToFile=true"] self.run_args = fastr_args + bench_runner_args if mx.suite("compiler", fatalIfMissing=False): os.environ["DEFAULT_DYNAMIC_IMPORTS"] = "graal/compiler" retcode = _fastr_suite.extensions.do_run_r(self.run_args, "R", nonZeroIsFatal=False, extraVmArgs=extra_vm_args, out=output_capture, err=output_capture) else: # Run in GNU-R self.vm_args = [] self.run_args = bench_runner_args gnur_bin = str(self._gnur_dir.joinpath("bin", "R").absolute()) retcode = mx.run([gnur_bin] + self.run_args, out=output_capture, err=output_capture) return retcode, output_capture, None
def test_javac(project_name, opts_set): """(jaotc-)Compiles the `jdk.compiler` module and compiles (mx) project_name using `javac` (+ AOT module).""" # jaotc uses ':' as separator. modules = ':'.join(['jdk.compiler']) for common_opts in opts_set: out_dir = tempfile.mkdtemp() try: mx.log('(jaotc) Compiling module(s) {} with {}'.format( modules, ' '.join(common_opts))) with mktemp_libfile() as lib_module: lib_module.file.close() run_jaotc(['-J' + opt for opt in common_opts] + [ '--exit-on-error', '--info', '--module', modules, '--output', lib_module.name ]) aot_opts = [ '-XX:+UnlockDiagnosticVMOptions', '-XX:+UseAOTStrictLoading', '-XX:AOTLibrary=' + lib_module.name ] project = mx.project(project_name) java_files = collect_java_sources(project.source_dirs()) javac_args = mx.JavacCompiler(jdk).prepare( sourceFiles=java_files, project=project, outputDir=out_dir, classPath=mx.classpath(project, includeSelf=False, jdk=jdk, ignoreStripped=True), sourceGenDir=project.source_gen_dir(), jnigenDir=project.jni_gen_dir(), processorPath=project.annotation_processors_path(jdk), disableApiRestrictions=True, warningsAsErrors=False, showTasks=False, postCompileActions=[], forceDeprecationAsWarning=False) mx_compiler.run_vm(common_opts + aot_opts + ['com.sun.tools.javac.Main'] + javac_args) finally: shutil.rmtree(out_dir)
def create_parser(grammar_project, grammar_package, grammar_name, copyright_template, args=None, out=None): """create the DSL expression parser using antlr""" grammar_dir = mx.project(grammar_project).source_dirs()[0] + "/" + grammar_package.replace(".", "/") + "/" mx.run_java(mx.get_runtime_jvm_args(['ANTLR4_COMPLETE']) + ["org.antlr.v4.Tool", "-package", grammar_package, "-no-listener"] + args + [grammar_dir + grammar_name + ".g4"], out=out) for filename in [grammar_dir + grammar_name + "Lexer.java", grammar_dir + grammar_name + "Parser.java"]: with open(filename, 'r') as content_file: content = content_file.read() # remove first line content = "\n".join(content.split("\n")[1:]) # modify SuppressWarnings to remove useless entries content = PTRN_SUPPRESS_WARNINGS.sub('@SuppressWarnings("all")', content) # remove useless casts content = PTRN_LOCALCTXT_CAST.sub('_localctx', content) content = PTRN_TOKEN_CAST.sub('_errHandler.recoverInline(this)', content) # add copyright header content = copyright_template.format(content) with open(filename, 'w') as content_file: content_file.write(content)
def nativebuild(args): ''' force the build of part or all of the native project ''' parser = ArgumentParser(prog='nativebuild') parser.add_argument('--all', action='store_true', help='clean and build everything, else just ffi') args = parser.parse_args(args) nativedir = mx.project('com.oracle.truffle.r.native').dir if args.all: return subprocess.call(['make clean && make'], shell=True, cwd=nativedir) else: ffidir = join(nativedir, 'fficall') jni_done = join(ffidir, 'jni.done') jniboot_done = join(ffidir, 'jniboot.done') if os.path.exists(jni_done): os.remove(jni_done) if os.path.exists(jniboot_done): os.remove(jniboot_done) return mx.build(['--no-java'])
def _graal_nodejs_post_gate_runner(args, tasks): _setEnvVar('NODE_INTERNAL_ERROR_CHECK', 'true') with Task('UnitTests', tasks, tags=[GraalNodeJsTags.allTests, GraalNodeJsTags.unitTests]) as t: if t: _setEnvVar('NODE_JVM_CLASSPATH', mx.distribution('graal-js:TRUFFLE_JS_TESTS').path) commonArgs = ['-ea', '-esa'] unitTestDir = join('test', 'graal') for dir_name in 'node_modules', 'build': p = join(unitTestDir, dir_name) if exists(p): mx.rmtree(p) npm(['--scripts-prepend-node-path=auto', 'install', '--nodedir=' + _suite.dir] + commonArgs, cwd=unitTestDir) npm(['--scripts-prepend-node-path=auto', 'test'] + commonArgs, cwd=unitTestDir) with Task('TestNpm', tasks, tags=[GraalNodeJsTags.allTests, GraalNodeJsTags.windows]) as t: if t: tmpdir = tempfile.mkdtemp() try: npm(['init', '-y'], cwd=tmpdir) npm(['install', '--scripts-prepend-node-path=true', 'microtime'], cwd=tmpdir) node(['-e', 'console.log(require("microtime").now());'], cwd=tmpdir) finally: mx.rmtree(tmpdir, ignore_errors=True) with Task('TestNpx', tasks, tags=[GraalNodeJsTags.allTests, GraalNodeJsTags.windows]) as t: if t: npx(['cowsay', 'GraalVM rules!']) with Task('JniProfilerTests', tasks, tags=[GraalNodeJsTags.allTests, GraalNodeJsTags.jniProfilerTests]) as t: if t: commonArgs = ['-ea', '-esa'] unitTestDir = join(mx.project('com.oracle.truffle.trufflenode.jniboundaryprofiler').dir, 'tests') for dir_name in 'node_modules', 'build': p = join(unitTestDir, dir_name) if exists(p): mx.rmtree(p) npm(['--scripts-prepend-node-path=auto', 'install', '--nodedir=' + _suite.dir] + commonArgs, cwd=unitTestDir) node(['-profile-native-boundary', 'test.js'] + commonArgs, cwd=unitTestDir) with Task('TestNodeInstrument', tasks, tags=[GraalNodeJsTags.allTests, GraalNodeJsTags.windows]) as t: if t: testnodeInstrument([])
def _alldeps(args): parser = ArgumentParser(prog='mx mxt-alldeps') parser.add_argument('--kind', action='store', help='project, dist or library', default='project') parser.add_argument('--name', action='store', help='name of entity', required=True) parser.add_argument('--includeLibs', action='store_true', help='includeLibs') parser.add_argument('--ignoreSelf', action='store_true', help='ignoreSelf') args = parser.parse_args(args) entity = None if args.kind == 'project': entity = mx.project(args.name) elif args.kind == 'library': entity = mx.library(args.name) elif args.kind == 'dist': entity = mx.distribution(args.name) else: mx.abort('illegal kind: ' + args.kind) in_deps = [] deps = entity.all_deps(in_deps, includeLibs=args.includeLibs, includeSelf=not args.ignoreSelf) print 'alldeps:' for d in deps: print d.__class__.__name__, ":", d.name
def test_javac(project_name): """(jaotc-)Compiles the `jdk.compiler` module and compiles (mx) project_name using `javac` (+ AOT module).""" # jaotc uses ':' as separator. modules = ':'.join(['jdk.compiler']) for common_opts in common_opts_variants: out_dir = tempfile.mkdtemp() try: mx.log('(jaotc) Compiling module(s) {} with {}'.format(modules, ' '.join(common_opts))) with mktemp_libfile() as lib_module: run_jaotc(['-J' + opt for opt in common_opts] + ['--info', '--module', modules, '--output', lib_module.name]) aot_opts = [ '-XX:+UnlockDiagnosticVMOptions', '-XX:+UseAOTStrictLoading', '-XX:AOTLibrary=' + lib_module.name ] project = mx.project(project_name) java_files = collect_java_sources(project.source_dirs()) javac_args = mx.JavacCompiler(jdk).prepare( sourceFiles=java_files, project=project, outputDir=out_dir, classPath=mx.classpath(project, includeSelf=False, jdk=jdk, ignoreStripped=True), sourceGenDir=project.source_gen_dir(), jnigenDir=project.jni_gen_dir(), processorPath=project.annotation_processors_path(jdk), disableApiRestrictions=True, warningsAsErrors=False, showTasks=False, postCompileActions=[], forceDeprecationAsWarning=False) mx_compiler.run_vm(common_opts + aot_opts + ['com.sun.tools.javac.Main'] + javac_args) finally: shutil.rmtree(out_dir)
def _unittest(args, annotations, prefixCp="", blacklist=None, whitelist=None, verbose=False, very_verbose=False, fail_fast=False, enable_timing=False, regex=None, color=False, eager_stacktrace=False, gc_after_test=False, suite=None): testfile = os.environ.get('MX_TESTFILE', None) if testfile is None: (_, testfile) = tempfile.mkstemp(".testclasses", "mxtool") os.close(_) mainClass = 'com.oracle.mxtool.junit.MxJUnitWrapper' if not exists( join( mx.project('com.oracle.mxtool.junit').output_dir(), mainClass.replace('.', os.sep) + '.class')): mx.build(['--only', 'com.oracle.mxtool.junit']) coreCp = mx.classpath(['com.oracle.mxtool.junit']) coreArgs = [] if very_verbose: coreArgs.append('-JUnitVeryVerbose') elif verbose: coreArgs.append('-JUnitVerbose') if fail_fast: coreArgs.append('-JUnitFailFast') if enable_timing: coreArgs.append('-JUnitEnableTiming') if color: coreArgs.append('-JUnitColor') if eager_stacktrace: coreArgs.append('-JUnitEagerStackTrace') if gc_after_test: coreArgs.append('-JUnitGCAfterTest') def harness(unittestCp, vmLauncher, vmArgs): prefixArgs = ['-esa', '-ea'] if gc_after_test: prefixArgs.append('-XX:-DisableExplicitGC') with open(testfile) as fp: testclasses = [l.rstrip() for l in fp.readlines()] cp = prefixCp + coreCp + os.pathsep + unittestCp # suppress menubar and dock when running on Mac vmArgs = prefixArgs + ['-Djava.awt.headless=true'] + vmArgs + [ '-cp', mx._separatedCygpathU2W(cp) ] # Execute Junit directly when one test is being run. This simplifies # replaying the VM execution in a native debugger (e.g., gdb). mainClassArgs = coreArgs + (testclasses if len(testclasses) == 1 else ['@' + mx._cygpathU2W(testfile)]) config = (vmArgs, mainClass, mainClassArgs) for p in _config_participants: config = p(config) vmLauncher.launcher(*config) vmLauncher = _vm_launcher if vmLauncher is None: jdk = mx.get_jdk() def _run_vm(vmArgs, mainClass, mainClassArgs): mx.run_java(vmArgs + [mainClass] + mainClassArgs, jdk=jdk) vmLauncher = _VMLauncher('default VM launcher', _run_vm, jdk) try: _run_tests(args, harness, vmLauncher, annotations, testfile, blacklist, whitelist, regex, mx.suite(suite) if suite else None) finally: if os.environ.get('MX_TESTFILE') is None: os.remove(testfile)
def _junit_r_harness(args, vmArgs, jdk, junitArgs): # always pass the directory where the expected output file should reside runlistener_arg = 'expected=' + _test_srcdir() # there should not be any unparsed arguments at this stage if args.remainder: mx.abort('unexpected arguments: ' + str(args.remainder).strip('[]') + '; did you forget --tests') def add_arg_separator(): # can't update in Python 2.7 arg = runlistener_arg if len(arg) > 0: arg += ',' return arg if args.gen_fastr_output: runlistener_arg = add_arg_separator() runlistener_arg += 'gen-fastr=' + args.gen_fastr_output if args.check_expected_output: args.gen_expected_output = True runlistener_arg = add_arg_separator() runlistener_arg += 'check-expected' if args.gen_expected_output: runlistener_arg = add_arg_separator() runlistener_arg += 'gen-expected' if args.keep_trailing_whitespace: runlistener_arg = add_arg_separator() runlistener_arg += 'keep-trailing-whitespace' if args.gen_expected_quiet: runlistener_arg = add_arg_separator() runlistener_arg += 'gen-expected-quiet' if args.gen_diff_output: runlistener_arg = add_arg_separator() runlistener_arg += 'gen-diff=' + args.gen_diff_output if args.trace_tests: runlistener_arg = add_arg_separator() runlistener_arg += 'trace-tests' # if args.test_methods: # runlistener_arg = add_arg_separator() # runlistener_arg = 'test-methods=' + args.test_methods runlistener_arg = add_arg_separator() runlistener_arg += 'test-project-output-dir=' + mx.project('com.oracle.truffle.r.test').output_dir() # use a custom junit.RunListener runlistener = 'com.oracle.truffle.r.test.TestBase$RunListener' if len(runlistener_arg) > 0: runlistener += ':' + runlistener_arg junitArgs += ['--runlistener', runlistener] # on some systems a large Java stack seems necessary vmArgs += ['-Xss12m'] # no point in printing errors to file when running tests (that contain errors on purpose) vmArgs += ['-DR:-PrintErrorStacktracesToFile'] setREnvironment() return mx.run_java(vmArgs + junitArgs, nonZeroIsFatal=False, jdk=jdk)
def _test_srcdir(): tp = 'edu.uci.python.test' return join(mx.project(tp).dir, 'src', tp.replace('.', sep))
def substitution_path(): path = mx.project('com.oracle.svm.bench').classpath_repr() if not mx.exists(path): mx.abort('Path to substitutions for scala dacapo not present: ' + path + '. Did you build all of substratevm?') return path
def _unittest( args, annotations, prefixCp="", blacklist=None, whitelist=None, verbose=False, fail_fast=False, enable_timing=False, regex=None, color=False, eager_stacktrace=False, gc_after_test=False, suite=None, ): testfile = os.environ.get("MX_TESTFILE", None) if testfile is None: (_, testfile) = tempfile.mkstemp(".testclasses", "mxtool") os.close(_) mainClass = "com.oracle.mxtool.junit.MxJUnitWrapper" if not exists(join(mx.project("com.oracle.mxtool.junit").output_dir(), mainClass.replace(".", os.sep) + ".class")): mx.build(["--only", "com.oracle.mxtool.junit"]) coreCp = mx.classpath(["com.oracle.mxtool.junit"]) coreArgs = [] if verbose: coreArgs.append("-JUnitVerbose") if fail_fast: coreArgs.append("-JUnitFailFast") if enable_timing: coreArgs.append("-JUnitEnableTiming") if color: coreArgs.append("-JUnitColor") if eager_stacktrace: coreArgs.append("-JUnitEagerStackTrace") if gc_after_test: coreArgs.append("-JUnitGCAfterTest") def harness(projectsCp, vmLauncher, vmArgs): prefixArgs = ["-esa", "-ea"] if gc_after_test: prefixArgs.append("-XX:-DisableExplicitGC") with open(testfile) as fp: testclasses = [l.rstrip() for l in fp.readlines()] cp = prefixCp + coreCp + os.pathsep + projectsCp # suppress menubar and dock when running on Mac vmArgs = prefixArgs + ["-Djava.awt.headless=true"] + vmArgs + ["-cp", mx._separatedCygpathU2W(cp)] # Execute Junit directly when one test is being run. This simplifies # replaying the VM execution in a native debugger (e.g., gdb). mainClassArgs = coreArgs + (testclasses if len(testclasses) == 1 else ["@" + mx._cygpathU2W(testfile)]) config = (vmArgs, mainClass, mainClassArgs) for p in _config_participants: config = p(config) _, launcher = vmLauncher launcher(*config) vmLauncher = _vm_launcher if vmLauncher is None: vmLauncher = ( "default VM launcher", lambda vmArgs, mainClass, mainClassArgs: mx.run_java(vmArgs + [mainClass] + mainClassArgs), ) try: _run_tests( args, harness, vmLauncher, annotations, testfile, blacklist, whitelist, regex, mx.suite(suite) if suite else None, ) finally: if os.environ.get("MX_TESTFILE") is None: os.remove(testfile)
def runTestSuite(testsuiteproject, args, testClasses=None, vmArgs=None): """compile and run external testsuite projects""" project = mx.project(testsuiteproject) assert isinstance(project, SulongTestSuite) project.runTestSuite(testClasses, vmArgs)
def _test_srcdir(): tp = 'com.oracle.truffle.r.test' return join(mx.project(tp).dir, 'src', tp.replace('.', sep))
def _cran_test_project_dir(): return mx.project(_cran_test_project()).dir
def mx_post_parse_cmd_line(opts): if mx.suite("tools", fatalIfMissing=False) and not _suite.isBinarySuite(): mx.project('truffleruby-bin').buildDependencies += [ mx.distribution('CHROMEINSPECTOR'), mx.distribution('TRUFFLE_PROFILER') ]
def _gnur_path(): np = mx.project('com.oracle.truffle.r.native') return join(np.dir, 'gnur', r_version(), 'bin')
def _packages_test_project_dir(): return mx.project(_packages_test_project()).dir
def sonarqube_upload(args): """run SonarQube scanner and upload JaCoCo results""" sonarqube_cli = mx.library("SONARSCANNER_CLI_4_2_0_1873", True) parser = ArgumentParser(prog='mx sonarqube-upload') parser.add_argument('--exclude-generated', action='store_true', help='Exclude generated source files') parser.add_argument('--skip-coverage', action='store_true', default=False, help='Do not upload coverage reports') args, sonar_args = mx.extract_VM_args(args, useDoubleDash=True, defaultAllVMArgs=True) args, other_args = parser.parse_known_args(args) java_props, other_args = _parse_java_properties(other_args) def _check_required_prop(prop): if prop not in java_props: mx.abort("Required property '{prop}' not present. (Format is '-D{prop}=<value>')".format(prop=prop)) _check_required_prop('sonar.projectKey') _check_required_prop('sonar.host.url') basedir = mx.primary_suite().dir # collect excluded projects excludes, includes = _jacoco_excludes_includes_projects(limit_to_primary=True) # collect excluded classes exclude_classes = _jacoco_exclude_classes(includes) java_bin = [] java_src = [] java_libs = [] def _visit_deps(dep, edge): if dep.isJARDistribution() or dep.isLibrary(): java_libs.append(dep.classpath_repr()) mx.walk_deps(includes, visit=_visit_deps) # collect all sources and binaries -- do exclusion later for p in includes: java_src.extend(p.source_dirs()) if not args.exclude_generated: gen_dir = p.source_gen_dir() if os.path.exists(gen_dir): java_src.append(gen_dir) java_bin.append(p.output_dir()) java_src = [os.path.relpath(s, basedir) for s in java_src] java_bin = [os.path.relpath(b, basedir) for b in java_bin] # Overlayed sources and classes must be excluded jdk_compliance = mx.get_jdk().javaCompliance overlayed_sources = [] overlayed_classfiles = {} for p in includes: if hasattr(p, "multiReleaseJarVersion") and jdk_compliance not in p.javaCompliance: # JDK9+ overlays for srcDir in p.source_dirs(): for root, _, files in os.walk(srcDir): for name in files: if name.endswith('.java') and name != 'package-info.java': overlayed_sources.append(join(os.path.relpath(root, basedir), name)) elif hasattr(p, "overlayTarget"): # JDK8 overlays target = mx.project(p.overlayTarget) overlay_sources = [] for srcDir in p.source_dirs(): for root, _, files in os.walk(srcDir): for name in files: if name.endswith('.java') and name != 'package-info.java': overlay_sources.append(join(os.path.relpath(root, srcDir), name)) print(p, target, overlay_sources) for srcDir in target.source_dirs(): for root, _, files in os.walk(srcDir): for name in files: if name.endswith('.java') and name != 'package-info.java': s = join(os.path.relpath(root, srcDir), name) if s in overlay_sources: overlayed = join(os.path.relpath(root, basedir), name) overlayed_sources.append(overlayed) for s in overlay_sources: classfile = join(os.path.relpath(target.output_dir(), basedir), s[:-len('java')] + 'class') with open(classfile, 'rb') as fp: overlayed_classfiles[classfile] = fp.read() exclude_dirs = [] for p in excludes: exclude_dirs.extend(p.source_dirs()) exclude_dirs.append(p.source_gen_dir()) javaCompliance = max([p.javaCompliance for p in includes]) if includes else mx.JavaCompliance('1.7') jacoco_exec = get_jacoco_dest_file() if not os.path.exists(jacoco_exec) and not args.skip_coverage: mx.abort('No JaCoCo report file found: ' + jacoco_exec) def _add_default_prop(key, value): if key not in java_props: java_props[key] = value # default properties _add_default_prop('sonar.java.source', str(javaCompliance)) _add_default_prop('sonar.projectBaseDir', basedir) if not args.skip_coverage: _add_default_prop('sonar.jacoco.reportPaths', jacoco_exec) _add_default_prop('sonar.sources', ','.join(java_src)) _add_default_prop('sonar.java.binaries', ','.join(java_bin)) _add_default_prop('sonar.java.libraries', ','.join(java_libs)) exclude_patterns = [os.path.relpath(e, basedir) + '**' for e in exclude_dirs] + \ overlayed_sources + \ list(set([os.path.relpath(match[0], basedir) for _, match in exclude_classes.items()])) if exclude_patterns: _add_default_prop('sonar.exclusions', ','.join(exclude_patterns)) _add_default_prop('sonar.coverage.exclusions', ','.join(exclude_patterns)) _add_default_prop('sonar.verbose', 'true' if mx._opts.verbose else 'false') with tempfile.NamedTemporaryFile(suffix="-sonarqube.properties", mode="w+") as fp: # prepare properties file fp.writelines(('{}={}\n'.format(k, v) for k, v in java_props.items())) fp.flush() # Since there's no options to exclude individual classes, # we temporarily delete the overlayed class files instead. for classfile in overlayed_classfiles: os.remove(classfile) try: # run sonarqube cli java_args = other_args + ['-Dproject.settings=' + fp.name, '-jar', sonarqube_cli.get_path(True)] + sonar_args exit_code = mx.run_java(java_args, nonZeroIsFatal=False) finally: # Restore temporarily deleted class files for classfile, data in overlayed_classfiles.items(): with open(classfile, 'wb') as cf: cf.write(data) if exit_code != 0: fp.seek(0) mx.abort('SonarQube scanner terminated with non-zero exit code: {}\n Properties file:\n{}'.format( exit_code, ''.join((' ' + l for l in fp.readlines()))))
def _cran_test_project(): return mx.project('com.oracle.truffle.r.test.cran').dir
def jaotc_classpath(): return mx.project('jdk.tools.jaotc.test').output_dir()