Exemplo n.º 1
0
def cinterfacetutorial(native_image, args=None):
    """Build and run the tutorial for the C interface"""

    args = [] if args is None else args
    tutorial_proj = mx.dependency('com.oracle.svm.tutorial')
    cSourceDir = join(tutorial_proj.dir, 'native')
    buildDir = join(svmbuild_dir(), tutorial_proj.name, 'build')

    # clean / create output directory
    if exists(buildDir):
        remove_tree(buildDir)
    mkpath(buildDir)

    # Build the shared library from Java code
    native_image(['--shared', '-H:Path=' + buildDir, '-H:Name=libcinterfacetutorial',
                  '-H:CLibraryPath=' + tutorial_proj.dir, '-cp', tutorial_proj.output_dir()] + args)

    # Build the C executable
    mx.run(['cc', '-g', join(cSourceDir, 'cinterfacetutorial.c'),
            '-I' + buildDir,
            '-L' + buildDir, '-lcinterfacetutorial',
            '-ldl', '-Wl,-rpath,' + buildDir,
            '-o', join(buildDir, 'cinterfacetutorial')])

    # Start the C executable
    mx.run([buildDir + '/cinterfacetutorial'])
Exemplo n.º 2
0
def genInlineAssemblyParser(args=None, out=None):
    """generate inline assembly parser and scanner if corresponding grammer is new"""
    generatedParserDir = _inlineAssemblySrcDir + _inlineAssemblyPackageName.replace(".", "/") + "/"
    generatedFiles = [generatedParserDir + "Parser.java", generatedParserDir + "Scanner.java"]
    configFiles = [
        _inlineAssemblySrcDir + "InlineAssembly.atg",
        _inlineAssemblySrcDir + "Parser.frame",
        _inlineAssemblySrcDir + "Scanner.frame",
        _inlineAssemblySrcDir + "Copyright.frame",
    ]
    isAllGeneratedFilesExists = all([isfile(fileName) for fileName in generatedFiles])
    latestGeneratedFile = (
        sorted(generatedFiles, key=os.path.getmtime, reverse=True)[0] if isAllGeneratedFilesExists else ""
    )
    latestConfigFile = sorted(configFiles, key=os.path.getmtime, reverse=True)[0]
    # If any auto-generated file is missing or any config file is updated after last auto-generation then regenerate the files
    if not isAllGeneratedFilesExists or os.path.getmtime(latestConfigFile) >= os.path.getmtime(latestGeneratedFile):
        localCocoJarFile = _suite.dir + "/lib/Coco.jar"
        if not isfile(localCocoJarFile):
            jarFileUrls = ["https://lafo.ssw.uni-linz.ac.at/pub/sulong-deps/Coco.jar"]
            mx.download(localCocoJarFile, jarFileUrls)
        command = [
            mx.get_jdk(tag="jvmci").java,
            "-jar",
            localCocoJarFile,
            "-package",
            _inlineAssemblyPackageName,
            "-o",
            generatedParserDir,
            _inlineAssemblyGrammer,
        ]
        mx.run(command)
        # Files get generated in Windows file format. Convert them to avoid style check failure during regression testing
        dos2unix(generatedFiles)
Exemplo n.º 3
0
def _update_JDK9_STUBS_library():
    """
    Sets the "path" and "sha1" attributes of the "JDK9_STUBS" library.
    """
    jdk9InternalLib = _suite.suiteDict['libraries']['JDK9_STUBS']
    jarInputDir = join(_suite.get_output_root(), 'jdk9-stubs')
    jarPath = join(_suite.get_output_root(), 'jdk9-stubs.jar')

    stubs = [
        ('jdk.internal.misc', 'VM', """package jdk.internal.misc;
public class VM {
    public static String getSavedProperty(String key) {
        throw new InternalError("should not reach here");
    }
}
""")
    ]

    if not exists(jarPath):
        sourceFiles = []
        for (package, className, source) in stubs:
            sourceFile = join(jarInputDir, package.replace('.', os.sep), className + '.java')
            mx.ensure_dir_exists(os.path.dirname(sourceFile))
            with open(sourceFile, 'w') as fp:
                fp.write(source)
            sourceFiles.append(sourceFile)
        jdk = mx.get_jdk(tag='default')
        mx.run([jdk.javac, '-d', jarInputDir] + sourceFiles)
        mx.run([jdk.jar, 'cf', jarPath, '.'], cwd=jarInputDir)

    jdk9InternalLib['path'] = jarPath
    jdk9InternalLib['sha1'] = mx.sha1OfFile(jarPath)
Exemplo n.º 4
0
def helloworld_internal(native_image, path=svmbuild_dir(), javac_command=None, args=None):
    if javac_command is None:
        javac_command = ['javac']
    args = [] if args is None else args
    mkpath(path)
    hello_file = join(path, 'HelloWorld.java')
    output = 'Hello from Substrate VM'
    with open(hello_file, 'w') as fp:
        fp.write('public class HelloWorld { public static void main(String[] args) { System.out.println("' + output + '"); } }')

    # Run javac. We sometimes run with an image so annotation processing must be disabled because it requires dynamic
    #  class loading, and we need to set the bootclasspath manually because our build directory does not contain any
    # .jar files.
    mx.run(javac_command + [hello_file])

    native_image(["-H:Path=" + path, '-cp', path, 'HelloWorld'] + args)

    expectedOutput = [output + '\n']
    actualOutput = []
    def _collector(x):
        actualOutput.append(x)
        mx.log(x)

    mx.run([join(path, 'helloworld')], out=_collector)

    if actualOutput != expectedOutput:
        raise Exception('Wrong output: ' + str(actualOutput) + "  !=  " + str(expectedOutput))
Exemplo n.º 5
0
def _gnur_install_test(pkgs, gnur_libinstall, gnur_install_tmp):
    gnur_packages = join(_fastr_suite_dir(), 'gnur.packages')
    with open(gnur_packages, 'w') as f:
        for pkg in pkgs:
            f.write(pkg)
            f.write('\n')
    env = os.environ.copy()
    env["TMPDIR"] = gnur_install_tmp
    env['R_LIBS_USER'] = gnur_libinstall
    env["TZDIR"] = "/usr/share/zoneinfo/"

    args = []
    if _graalvm():
        args += [_gnur_rscript()]
    args += [_installpkgs_script()]
    args += ['--pkg-filelist', gnur_packages]
    args += ['--run-tests']
# GNU R will abort the entire run otherwise if a failure occurs
#    args += ['--run-mode', 'internal']
    args += ['--ignore-blacklist']
    args += ['--testdir', 'test.gnur']
    _log_step('BEGIN', 'install/test', 'GnuR')
    if _graalvm():
        mx.run(args, nonZeroIsFatal=False, env=env)
    else:
        mx_fastr.gnu_rscript(args, env=env)
    _log_step('END', 'install/test', 'GnuR')
Exemplo n.º 6
0
def _run_netbeans_app(app_name, env=None, args=None):
    args = [] if args is None else args
    dist = app_name.upper() + '_DIST'
    name = app_name.lower()
    extractPath = join(_suite.get_output_root())
    if mx.get_os() == 'windows':
        executable = join(extractPath, name, 'bin', name + '.exe')
    else:
        executable = join(extractPath, name, 'bin', name)

    # Check whether the current installation is up-to-date
    if exists(executable) and not exists(mx.library(dist).get_path(resolve=False)):
        mx.log('Updating ' + app_name)
        shutil.rmtree(join(extractPath, name))

    archive = mx.library(dist).get_path(resolve=True)

    if not exists(executable):
        zf = zipfile.ZipFile(archive, 'r')
        zf.extractall(extractPath)

    if not exists(executable):
        mx.abort(app_name + ' binary does not exist: ' + executable)

    if mx.get_os() != 'windows':
        # Make sure that execution is allowed. The zip file does not always specfiy that correctly
        os.chmod(executable, 0777)
    mx.run([executable]+args, env=env)
Exemplo n.º 7
0
def pullInstallDragonEgg(args=None):
    """downloads and installs dragonegg (assumes that compatible GCC and G++ versions are installed)"""
    toolDir = join(_toolDir, "tools/dragonegg")
    mx.ensure_dir_exists(toolDir)
    url = "https://lafo.ssw.uni-linz.ac.at/pub/sulong-deps/dragonegg-3.2.src.tar.gz"
    localPath = pullsuite(toolDir, [url])
    tar(localPath, toolDir)
    os.remove(localPath)
    if mx.get_os() == "darwin":
        gccToolDir = join(_toolDir, "tools/gcc")
        url = "https://lafo.ssw.uni-linz.ac.at/pub/sulong-deps/gcc-4.6.4.tar.gz"
        localPath = pullsuite(gccToolDir, [url])
        tar(localPath, gccToolDir)
        os.remove(localPath)
        mx.run(
            ["patch", "-p1", _toolDir + "tools/dragonegg/dragonegg-3.2.src/Makefile", "mx.sulong/dragonegg-mac.patch"]
        )
    os.environ["GCC"] = getGCC()
    os.environ["CXX"] = getGPP()
    os.environ["CC"] = getGCC()
    pullLLVMBinaries()
    os.environ["LLVM_CONFIG"] = findLLVMProgram("llvm-config")
    print os.environ["LLVM_CONFIG"]
    compileCommand = ["make"]
    return mx.run(compileCommand, cwd=_toolDir + "tools/dragonegg/dragonegg-3.2.src")
Exemplo n.º 8
0
def _runmake(args):
    """run the JDK make process

To build hotspot and import it into the JDK: "mx make hotspot import-hotspot"
{0}"""

    jdkBuildDir = _get_jdk_build_dir()
    if not exists(jdkBuildDir):
        # JDK9 must be bootstrapped with a JDK8
        compliance = mx.JavaCompliance('8')
        jdk8 = mx.get_jdk(compliance.exactMatch, versionDescription=compliance.value)
        cmd = ['sh', 'configure', '--with-debug-level=' + _vm.debugLevel, '--with-native-debug-symbols=external', '--disable-precompiled-headers', '--with-jvm-features=graal',
               '--with-jvm-variants=' + _vm.jvmVariant, '--disable-warnings-as-errors', '--with-boot-jdk=' + jdk8.home, '--with-jvm-features=graal']
        mx.run(cmd, cwd=_jdkSourceRoot)
    cmd = [mx.gmake_cmd(), 'CONF=' + _vm.debugLevel]
    if mx.get_opts().verbose:
        cmd.append('LOG=debug')
    cmd.extend(args)
    if mx.get_opts().use_jdk_image and 'images' not in args:
        cmd.append('images')

    if not mx.get_opts().verbose:
        mx.log('--------------- make execution ----------------------')
        mx.log('Working directory: ' + _jdkSourceRoot)
        mx.log('Command line: ' + ' '.join(cmd))
        mx.log('-----------------------------------------------------')

    mx.run(cmd, cwd=_jdkSourceRoot)
Exemplo n.º 9
0
def c1visualizer(args):
    """run the Cl Compiler Visualizer"""
    libpath = join(_suite.dir, 'lib')
    if mx.get_os() == 'windows':
        executable = join(libpath, 'c1visualizer', 'bin', 'c1visualizer.exe')
    else:
        executable = join(libpath, 'c1visualizer', 'bin', 'c1visualizer')

    # Check whether the current C1Visualizer installation is the up-to-date
    if exists(executable) and not exists(mx.library('C1VISUALIZER_DIST').get_path(resolve=False)):
        mx.log('Updating C1Visualizer')
        shutil.rmtree(join(libpath, 'c1visualizer'))

    archive = mx.library('C1VISUALIZER_DIST').get_path(resolve=True)

    if not exists(executable):
        zf = zipfile.ZipFile(archive, 'r')
        zf.extractall(libpath)

    if not exists(executable):
        mx.abort('C1Visualizer binary does not exist: ' + executable)

    if mx.get_os() != 'windows':
        # Make sure that execution is allowed. The zip file does not always specfiy that correctly
        os.chmod(executable, 0777)

    mx.run([executable])
Exemplo n.º 10
0
def jdkartifactstats(args):
    """show stats about JDK deployed Graal artifacts"""
    artifacts = {}
    jdkDir = get_jvmci_jdk().home
    def _getDeployedJars():
        if JVMCI_VERSION < 9:
            for root, _, filenames in os.walk(join(jdkDir, 'jre', 'lib')):
                for f in filenames:
                    if f.endswith('.jar') and not f.endswith('.stripped.jar'):
                        yield join(root, f)
        else:
            for jdkDist in jdkDeployedDists:
                dist = jdkDist.dist()
                if isinstance(jdkDist, JvmciJDKDeployedDist):
                    yield dist.path

    for jar in _getDeployedJars():
        f = basename(jar)
        if 'truffle' in f:
            if 'enterprise' in f:
                artifacts.setdefault('GraalEnterpriseTruffle', []).append(jar)
            else:
                artifacts.setdefault('GraalTruffle', []).append(jar)
        elif 'enterprise' in f:
            artifacts.setdefault('GraalEnterprise', []).append(jar)
        elif 'jvmci' in f:
            artifacts.setdefault('JVMCI', []).append(jar)
        elif 'graal' in f:
            artifacts.setdefault('Graal', []).append(jar)
        else:
            mx.logv('ignored: ' + jar)

    print '{:>10}  {:>10}  {:>10}  {}'.format('All', 'NoVars', 'None', 'Jar')
    for category in sorted(artifacts.viewkeys()):
        jars = artifacts[category]
        if jars:
            totals = (0, 0, 0)
            print
            for j in jars:
                gSize = os.path.getsize(j)
                stripped = j[:-len('.jar')] + '.stripped.jar'
                mx.run([mx.get_jdk().pack200, '--repack', '--quiet', '-J-Djava.util.logging.config.file=', '-DLocalVariableTypeTable=strip', '-DLocalVariableTable=strip', stripped, j])
                gLinesSourceSize = os.path.getsize(stripped)
                mx.run([mx.get_jdk().pack200, '--repack', '--quiet', '-J-Djava.util.logging.config.file=', '-G', stripped, j])
                gNoneSize = os.path.getsize(stripped)
                os.remove(stripped)
                print '{:10,}  {:10,}  {:10,}  {}:{}'.format(gSize, gLinesSourceSize, gNoneSize, category, basename(j))
                t1, t2, t3 = totals
                totals = (t1 + gSize, t2 + gLinesSourceSize, t3 + gNoneSize)
            t1, t2, t3 = totals
            print '{:10,}  {:10,}  {:10,}  {}'.format(t1, t2, t3, category)

    jvmLib = join(jdkDir, relativeVmLibDirInJdk(), get_vm(), mx.add_lib_suffix(mx.add_lib_prefix('jvm')))
    print
    if exists(jvmLib):
        print '{:10,}  {}'.format(os.path.getsize(jvmLib), jvmLib)
    else:
        print '{:>10}  {}'.format('<missing>', jvmLib)
Exemplo n.º 11
0
 def build(self):
     cwd = _suite.dir
     maven_repo_arg, env = mavenSetup()
     mx.log("Building jruby-core with Maven")
     mx.run_maven(['-q', '-DskipTests', maven_repo_arg, '-pl', 'core,lib'], cwd=cwd, env=env)
     # Install Bundler
     gem_home = join(_suite.dir, 'lib', 'ruby', 'gems', 'shared')
     env['GEM_HOME'] = gem_home
     env['GEM_PATH'] = gem_home
     mx.run(['bin/jruby', 'bin/gem', 'install', 'bundler', '-v', '1.10.6'], cwd=cwd, env=env)
Exemplo n.º 12
0
 def build(self):
     cwd = _suite.dir
     maven_repo_arg, env = mavenSetup()
     mx.log("Building jruby-core with Maven")
     quiet = [] if mx.get_opts().verbose else ['-q']
     mx.run_maven(quiet + ['-DskipTests', maven_repo_arg, '-Dcreate.sources.jar', '-pl', 'core,lib'], cwd=cwd, env=env)
     # Install Bundler
     gem_home = join(_suite.dir, 'lib', 'ruby', 'gems', 'shared')
     env['GEM_HOME'] = gem_home
     env['GEM_PATH'] = gem_home
     mx.run(['bin/jruby', 'bin/gem', 'install', 'bundler', '-v', '1.10.6'], cwd=cwd, env=env)
Exemplo n.º 13
0
def clangBench(args=None):
    """ Executes a benchmark with the system default Clang"""
    _, inputFiles = extract_compiler_args(args)
    _, ext = os.path.splitext(inputFiles[0])
    if ext == '.c':
        mx.run(['clang'] + args + standardLinkerCommands())
    elif ext == '.cpp':
        mx.run(['clang++'] + args + standardLinkerCommands())
    else:
        exit(ext + " is not supported!")
    return mx.run(['./a.out'])
Exemplo n.º 14
0
def _configs():
    class Configs:
        def __init__(self):
            self.configs = dict()

        def eat(self, line):
            (k, v) = line.split('#')
            self.configs[k] = v.rstrip()
    c = Configs()
    mx.run([mx.java().java, '-client', '-Xmx40m', '-Xms40m', '-XX:NewSize=30m', '-cp', mx.classpath(resolve=False), 'test.com.sun.max.vm.MaxineTesterConfiguration'], out=c.eat)
    return c.configs
Exemplo n.º 15
0
def gccBench(args=None):
    """ executes a benchmark with the system default GCC version"""
    _, inputFiles = extract_compiler_args(args)
    _, ext = os.path.splitext(inputFiles[0])
    if ext == '.c':
        mx.run(['gcc', '-std=gnu99'] + args + standardLinkerCommands())
    elif ext == '.cpp':
        mx.run(['g++'] + args + standardLinkerCommands())
    else:
        exit(ext + " is not supported!")
    return mx.run(['./a.out'])
Exemplo n.º 16
0
def mavenSetup():
    mavenDir = join(_suite.dir, 'mxbuild', 'mvn')
    maven_repo_arg = '-Dmaven.repo.local=' + mavenDir
    env = os.environ.copy()
    env['JRUBY_BUILD_MORE_QUIET'] = 'true'
    # HACK: since the maven executable plugin does not configure the
    # java executable that is used we unfortunately need to append it to the PATH
    javaHome = os.getenv('JAVA_HOME')
    if javaHome:
        env["PATH"] = javaHome + '/bin' + os.pathsep + env["PATH"]
        mx.logv('Setting PATH to {}'.format(os.environ["PATH"]))
    mx.run(['java', '-version'])
    return maven_repo_arg, env
Exemplo n.º 17
0
def inspectoragent(args):
    """launch the Inspector agent

    Launch the Inspector agent.

    The agent listens on a given port for an incoming connection from
    a remote Inspector process."""

    cmd = mx.java().format_cmd(['-cp', mx.classpath(), 'com.sun.max.tele.channel.agent.InspectorAgent'] + args)
    if mx.get_os() == 'darwin':
        # The -E option propagates the environment variables into the sudo process
        mx.run(['sudo', '-E', '-p', 'Debugging is a privileged operation on Mac OS X.\nPlease enter your "sudo" password:'] + cmd)
    else:
        mx.run(cmd)
Exemplo n.º 18
0
def pullInstallDragonEgg(args=None):
    """downloads and installs dragonegg (assumes that GCC 4.6 is on the path)"""
    if hasDragoneggGCCInstalled():
        toolDir = join(_toolDir, "tools/dragonegg")
        mx.ensure_dir_exists(toolDir)
        url = 'http://llvm.org/releases/3.2/dragonegg-3.2.src.tar.gz'
        localPath = pullsuite(toolDir, [url])
        tar(localPath, toolDir)
        os.remove(localPath)
        os.environ['GCC'] = 'gcc-4.6'
        os.environ['LLVM_CONFIG'] = _toolDir + 'tools/llvm/bin/llvm-config'
        compileCommand = ['make']
        mx.run(compileCommand, cwd=_toolDir + 'tools/dragonegg/dragonegg-3.2.src')
    else:
        print 'could not find gcc-4.6, skip installing dragonegg!'
Exemplo n.º 19
0
def mavenSetup():
    buildPack = join(_suite.dir, 'jruby-build-pack/maven')
    mavenDir = buildPack if isdir(buildPack) else join(_suite.dir, 'mxbuild/mvn')
    maven_repo_arg = '-Dmaven.repo.local=' + mavenDir
    env = os.environ.copy()
    if not mx.get_opts().verbose:
        env['JRUBY_BUILD_MORE_QUIET'] = 'true'
    # HACK: since the maven executable plugin does not configure the
    # java executable that is used we unfortunately need to prepend it to the PATH
    javaHome = os.getenv('JAVA_HOME')
    if javaHome:
        env["PATH"] = javaHome + '/bin' + os.pathsep + env["PATH"]
        mx.logv('Setting PATH to {}'.format(os.environ["PATH"]))
    mx.run(['java', '-version'], env=env)
    return maven_repo_arg, env
Exemplo n.º 20
0
def run_java(jdk, args, nonZeroIsFatal=True, out=None, err=None, cwd=None, timeout=None, env=None, addDefaultArgs=True):

    args = _parseVmArgs(jdk, args, addDefaultArgs=addDefaultArgs)

    jvmciModeArgs = _jvmciModes[_vm.jvmciMode]
    cmd = [jdk.java] + ['-' + get_vm()] + jvmciModeArgs + args
    return mx.run(cmd, nonZeroIsFatal=nonZeroIsFatal, out=out, err=err, cwd=cwd)
Exemplo n.º 21
0
def js_image_test(binary, bench_location, name, warmup_iterations, iterations, timeout=None, bin_args=None):
    bin_args = bin_args if bin_args is not None else []
    jsruncmd = [binary] + bin_args + [join(bench_location, 'harness.js'), '--', join(bench_location, name + '.js'),
                                      '--', '--warmup-iterations=' + str(warmup_iterations),
                                      '--iterations=' + str(iterations)]
    mx.log(' '.join(jsruncmd))

    passing = []

    stdoutdata = []
    def stdout_collector(x):
        stdoutdata.append(x)
        mx.log(x.rstrip())
    stderrdata = []
    def stderr_collector(x):
        stderrdata.append(x)
        mx.warn(x.rstrip())

    returncode = mx.run(jsruncmd, cwd=bench_location, out=stdout_collector, err=stderr_collector, nonZeroIsFatal=False, timeout=timeout)

    if returncode == mx.ERROR_TIMEOUT:
        print('INFO: TIMEOUT (> %d): %s' % (timeout, name))
    elif returncode >= 0:
        matches = 0
        for line in stdoutdata:
            if re.match(r'^\S+: *\d+(\.\d+)?\s*$', line):
                matches += 1
        if matches > 0:
            passing = stdoutdata

    if not passing:
        mx.abort('JS benchmark ' + name + ' failed')
Exemplo n.º 22
0
def igv(args):
    """run the Ideal Graph Visualizer"""
    logFile = '.ideal_graph_visualizer.log'
    with open(join(_suite.dir, logFile), 'w') as fp:
        mx.logv('[Ideal Graph Visualizer log is in ' + fp.name + ']')
        nbplatform = join(_suite.dir, 'src', 'share', 'tools', 'IdealGraphVisualizer', 'nbplatform')

        # Remove NetBeans platform if it is earlier than the current supported version
        if exists(nbplatform):
            updateTrackingFile = join(nbplatform, 'platform', 'update_tracking', 'org-netbeans-core.xml')
            if not exists(updateTrackingFile):
                mx.log('Could not find \'' + updateTrackingFile + '\', removing NetBeans platform')
                shutil.rmtree(nbplatform)
            else:
                dom = xml.dom.minidom.parse(updateTrackingFile)
                currentVersion = mx.VersionSpec(dom.getElementsByTagName('module_version')[0].getAttribute('specification_version'))
                supportedVersion = mx.VersionSpec('3.43.1')
                if currentVersion < supportedVersion:
                    mx.log('Replacing NetBeans platform version ' + str(currentVersion) + ' with version ' + str(supportedVersion))
                    shutil.rmtree(nbplatform)
                elif supportedVersion < currentVersion:
                    mx.log('Supported NetBeans version in igv command should be updated to ' + str(currentVersion))

        if not exists(nbplatform):
            mx.logv('[This execution may take a while as the NetBeans platform needs to be downloaded]')

        env = _igvBuildEnv()
        # make the jar for Batik 1.7 available.
        env['IGV_BATIK_JAR'] = mx.library('BATIK').get_path(True)
        if mx.run(['ant', '-f', mx._cygpathU2W(join(_suite.dir, 'src', 'share', 'tools', 'IdealGraphVisualizer', 'build.xml')), '-l', mx._cygpathU2W(fp.name), 'run'], env=env, nonZeroIsFatal=False):
            mx.abort("IGV ant build & launch failed. Check '" + logFile + "'. You can also try to delete 'src/share/tools/IdealGraphVisualizer/nbplatform'.")
Exemplo n.º 23
0
def cc(args):
    _log('fastr:cc', args)
    compiler = None
    sulong = _sulong()
    if sulong:
        analyzed_args = _analyze_args(args)
        if _is_linux():
            rc = sulong.compileWithGCC(analyzed_args.compile_args)
            if rc == 0 and analyzed_args.llvm_ir_file:
                if not analyzed_args.is_link:
                    rc = sulong.compileWithGCC(analyzed_args.emit_llvm_args)
        elif _is_darwin():
            rc = sulong.compileWithClang(analyzed_args.compile_args)
            if rc == 0 and analyzed_args.llvm_ir_file:
                if not analyzed_args.is_link:
                    rc = sulong.compileWithClang(analyzed_args.emit_llvm_args)
        else:
            mx.abort('unsupported platform')
        if rc == 0 and not analyzed_args.is_link and analyzed_args.llvm_ir_file:
            rc = _mem2reg_opt(analyzed_args.llvm_ir_file)
            if rc == 0:
                rc = _embed_ir(analyzed_args.llvm_ir_file)
    else:
        if _is_linux():
            compiler = 'gcc'
        elif _is_darwin():
            compiler = 'clang'
        else:
            mx.abort('unsupported platform')

        rc = mx.run([compiler] + args, nonZeroIsFatal=False)

    return rc
Exemplo n.º 24
0
def gnu_rscript(args, env=None):
    '''
    run the internally built GNU Rscript executable
    env arg is used by pkgtest
    '''
    cmd = [join(_gnur_path(), 'Rscript')] + args
    return mx.run(cmd, nonZeroIsFatal=False, env=env)
Exemplo n.º 25
0
    def run(self, cwd, args):
        # save current Directory
        self.currentDir = os.getcwd()
        os.chdir(_benchmarksDirectory())

        f = open(os.devnull, 'w')

        mx_sulong.ensureLLVMBinariesExist()
        benchmarkDir = args[0]

        # enter benchmark dir
        os.chdir(benchmarkDir)

        # create directory for executable of this vm
        if not os.path.exists(self.name()):
            os.makedirs(self.name())
        os.chdir(self.name())

        if os.path.exists('bench'):
            os.remove('bench')

        env = os.environ.copy()
        env['CFLAGS'] = ' '.join(_env_flags + ['-lm', '-lgmp'])
        env['LLVM_COMPILER'] = 'clang'
        env['CC'] = 'wllvm'
        env['VPATH'] = '..'

        cmdline = ['make', '-f', '../Makefile']

        mx.run(cmdline, out=f, err=f, env=env)
        mx.run(['extract-bc', 'bench'], out=f, err=f)
        mx_sulong.opt(['-o', 'bench.bc', 'bench.bc'] + ['-mem2reg', '-globalopt', '-simplifycfg', '-constprop', '-instcombine', '-dse', '-loop-simplify', '-reassociate', '-licm', '-gvn'], out=f, err=f)

        suTruffleOptions = [
            '-Dgraal.TruffleBackgroundCompilation=false',
            '-Dgraal.TruffleTimeThreshold=1000000',
            '-Dgraal.TruffleInliningMaxCallerSize=10000',
            '-Dgraal.TruffleCompilationExceptionsAreFatal=true',
            mx_subst.path_substitutions.substitute('-Dpolyglot.llvm.libraryPath=<path:SULONG_LIBS>'),
            '-Dpolyglot.llvm.libraries=libgmp.so.10']
        sulongCmdLine = suTruffleOptions + mx_sulong.getClasspathOptions() + ['-XX:-UseJVMCIClassLoader', "com.oracle.truffle.llvm.launcher.LLVMLauncher"] + ['bench.bc']
        result = self.host_vm().run(cwd, sulongCmdLine + args)

        # reset current Directory
        os.chdir(self.currentDir)

        return result
Exemplo n.º 26
0
 def runTool(self, args, errorMsg=None):
     try:
         if not mx.get_opts().verbose:
             f = open(os.devnull, 'w')
             ret = mx.run(args, out=f, err=f)
         else:
             f = None
             ret = mx.run(args)
     except SystemExit:
         ret = -1
         if errorMsg is None:
             print('\nError: Cannot run %s' % args)
         else:
             print('\nError: %s\n%s' % (errorMsg, ' '.join(args)))
     if f is not None:
         f.close()
     return ret
Exemplo n.º 27
0
def _jvmci_gate_runner(args, tasks):
    # Build release server VM now so we can run the unit tests
    with Task('BuildHotSpotJVMCIHosted: release', tasks) as t:
        if t: _runmultimake(['--jdk-jvm-variants', 'server', '--jdk-debug-levels', 'release'])

    # Run unit tests in hosted mode
    with VM(jvmVariant='server', debugLevel='release', jvmciMode='hosted'):
        with Task('JVMCI UnitTests: hosted-release', tasks) as t:
            if t: unittest(['--suite', 'jvmci', '--enable-timing', '--verbose', '--fail-fast'])

    # Build the other VM flavors
    with Task('BuildHotSpotJVMCIOthers: fastdebug', tasks) as t:
        if t: _runmultimake(['--jdk-jvm-variants', 'server', '--jdk-debug-levels', 'fastdebug'])

    with Task('CleanAndBuildIdealGraphVisualizer', tasks, disableJacoco=True) as t:
        if t and platform.processor() != 'sparc':
            buildxml = mx._cygpathU2W(join(_suite.dir, 'src', 'share', 'tools', 'IdealGraphVisualizer', 'build.xml'))
            mx.run(['ant', '-f', buildxml, '-q', 'clean', 'build'], env=_igvBuildEnv())
Exemplo n.º 28
0
    def run_java(self, args, vm=None, nonZeroIsFatal=True, out=None, err=None, cwd=None, timeout=None, env=None, addDefaultArgs=True):
        if vm is None:
            vm = 'server'

        args = self.parseVmArgs(args, addDefaultArgs=addDefaultArgs)

        jvmciModeArgs = _jvmciModes[_vm.jvmciMode]
        cmd = [self.java] + ['-' + vm] + jvmciModeArgs + args
        return mx.run(cmd, nonZeroIsFatal=nonZeroIsFatal, out=out, err=err, cwd=cwd)
Exemplo n.º 29
0
def _runmake(args):
    """run the JDK make process

To build hotspot and import it into the JDK: "mx make hotspot import-hotspot"
{0}"""

    jdkBuildDir = _get_jdk_build_dir()
    if not exists(jdkBuildDir):
        # JDK9 must be bootstrapped with a JDK8
        compliance = mx.JavaCompliance('8')
        jdk8 = mx.get_jdk(compliance.exactMatch, versionDescription=compliance.value)
        cmd = ['sh', 'configure', '--with-debug-level=' + _vm.debugLevel, '--with-native-debug-symbols=external', '--disable-precompiled-headers',
               '--with-jvm-variants=' + _vm.jvmVariant, '--disable-warnings-as-errors', '--with-boot-jdk=' + jdk8.home]
        mx.run(cmd, cwd=_jdkSourceRoot)
    cmd = [mx.gmake_cmd(), 'CONF=' + _vm.debugLevel]
    if mx.get_opts().verbose:
        cmd.append('LOG=debug')
    cmd.extend(args)
    if mx.get_opts().use_jdk_image and 'images' not in args:
        cmd.append('images')

    if not mx.get_opts().verbose:
        mx.log('--------------- make execution ----------------------')
        mx.log('Working directory: ' + _jdkSourceRoot)
        mx.log('Command line: ' + ' '.join(cmd))
        mx.log('-----------------------------------------------------')

    mx.run(cmd, cwd=_jdkSourceRoot)

    if 'images' in cmd:
        jdkImageDir = join(jdkBuildDir, 'images', 'jdk')

        # The OpenJDK build creates an empty cacerts file so copy one from
        # the default JDK (which is assumed to be an OracleJDK)
        srcCerts = join(mx.get_jdk(tag='default').home, 'lib', 'security', 'cacerts')
        if not exists(srcCerts):
            # Might be building with JDK8 which has cacerts under jre/
            srcCerts = join(mx.get_jdk(tag='default').home, 'jre', 'lib', 'security', 'cacerts')
        dstCerts = join(jdkImageDir, 'lib', 'security', 'cacerts')
        if srcCerts != dstCerts:
            shutil.copyfile(srcCerts, dstCerts)

        _create_jdk_bundle(jdkBuildDir, _vm.debugLevel, jdkImageDir)
Exemplo n.º 30
0
def fc(args):
    _log('fastr:fc', args)
    compiler = None
    sulong = _sulong()
    if sulong:
        analyzed_args = _analyze_args(args, dragonEgg=True)
        rc = mx.run([sulong.getGFortran()] + analyzed_args.compile_args, nonZeroIsFatal=False)
        if rc == 0:
            rc = sulong.dragonEggGFortran(analyzed_args.emit_llvm_args)
            if rc == 0 and analyzed_args.llvm_ir_file:
                # create bitcode from textual IR
                llvm_as = sulong.findLLVMProgram('llvm-as')
                llvm_bc_file = os.path.splitext(analyzed_args.llvm_ir_file)[0] + '.bc'
                rc = mx.run([llvm_as, analyzed_args.llvm_ir_file, '-o', llvm_bc_file])
                rc = _embed_ir(llvm_bc_file)
    else:
        compiler = 'gfortran'
        rc = mx.run([compiler] + args, nonZeroIsFatal=False)

    return rc
Exemplo n.º 31
0
def coverage_upload(args):
    parser = ArgumentParser(prog='mx coverage-upload')
    parser.add_argument('--upload-url', required=False, default=mx.get_env('COVERAGE_UPLOAD_URL'), help='Format is like rsync: user@host:/directory')
    parser.add_argument('--build-name', required=False, default=mx.get_env('BUILD_NAME'))
    parser.add_argument('--build-url', required=False, default=mx.get_env('BUILD_URL'))
    parser.add_argument('--build-number', required=False, default=mx.get_env('BUILD_NUMBER'))
    args, other_args = parser.parse_known_args(args)
    if not args.upload_url:
        parser.print_help()
        return
    remote_host, remote_basedir = args.upload_url.split(':')
    if not remote_host:
        mx.abort('Cannot determine remote host from {}'.format(args.upload_url))

    primary = mx.primary_suite()
    info = primary.vc.parent_info(primary.dir)
    rev = primary.vc.parent(primary.dir)
    if len(remote_basedir) > 0 and not remote_basedir.endswith('/'):
        remote_basedir += '/'
    remote_dir = '{}_{}_{}'.format(primary.name, datetime.datetime.fromtimestamp(info['author-ts']).strftime('%Y-%m-%d_%H_%M'), rev[:7])
    if args.build_name:
        remote_dir += '_' + args.build_name
    if args.build_number:
        remote_dir += '_' + args.build_number
    upload_dir = remote_basedir + remote_dir
    includes, excludes = _jacocoreport(['--omit-excluded'] + other_args)

    # Upload jar+sources
    coverage_sources = 'java_sources.tar.gz'
    coverage_binaries = 'java_binaries.tar.gz'

    with mx.Archiver(os.path.realpath(coverage_sources), kind='tgz') as sources, mx.Archiver(os.path.realpath(coverage_binaries), kind='tgz') as binaries:
        def _visit_deps(dep, edge):
            if dep.isJavaProject() and not dep.is_test_project():
                binaries.zf.add(dep.output_dir(), dep.name)
                for d in dep.source_dirs():
                    sources.zf.add(d, dep.name)
                if os.path.exists(dep.source_gen_dir()):
                    sources.zf.add(dep.source_gen_dir(), dep.name)
        mx.walk_deps(mx.projects(), visit=_visit_deps)

    files = [JACOCO_EXEC, 'coverage', coverage_sources, coverage_binaries]
    print("Syncing {} to {}:{}".format(" ".join(files), remote_host, upload_dir))
    mx.run([
        'bash',
        '-c',
        r'tar -czf - {files} | ssh {remote} bash -c \'"mkdir -p {remotedir} && cd {remotedir} && cat | tar -x{verbose}z && chmod -R 755 ."\''
            .format(
                files=" ".join(files),
                remote=remote_host,
                remotedir=upload_dir,
                verbose='v' if mx._opts.verbose else '')
    ])
    def upload_string(content, path):
        mx.run(['ssh', remote_host, 'bash', '-c', 'cat > "' + path + '"'], stdin=content)

    upload_string(json.dumps({
        'timestamp': time.time(),
        'suite': primary.name,
        'revision': rev,
        'directory': remote_dir,
        'build_name': args.build_name,
        'build_url': args.build_url,
        'jdk_version': str(mx.get_jdk().version),
        'build_number': args.build_number,
        'primary_info': info,
        'excludes': [str(e) for e in excludes],
        'includes': [str(i) for i in includes]}), upload_dir + '/description.json')
    mx.run(['ssh', remote_host, 'bash', '-c', r'"(echo \[; for i in {remote_basedir}/*/description.json; do if \[ -s \$i \];then cat \$i; echo ,; fi done; echo null\]) > {remote_basedir}/index.json"'.format(remote_basedir=remote_basedir)])
    upload_string("""<html>
<script language="javascript">
  function urlChange(url) {
    if (url.pathname !== "blank") {
      window.history.replaceState(null, null, url.pathname.replace("/coverage_upload/", "/coverage_upload/#"))
    }
  }
</script>
<frameset rows="40,*">
  <frame id="navigation" src="navigation.html"/>
  <frame id="content" src="" onload="urlChange(this.contentWindow.location);" />
</frameset>
</html>""", remote_basedir + '/index.html')
    js_library_url = rewriteurl("https://ajax.googleapis.com/ajax/libs/angularjs/1.7.7/angular.js")
    upload_string(r"""<html>
    <head>
        <script src="%js_library_url"></script>
        <script language="javascript">
        var App = angular.module('myApp', [])
            .controller('IndexCtrl', function IndexCtrl($scope, $http) {
                var hash = parent.window.location.hash;
                if(hash) {
                    hash = hash.substring(1, hash.length); // remove leading hash
                }
                $http.get('index.json').then(function(response, status) {
                    var data = response.data.filter(x => x != null);
                    /*
                        #GR-17399
                        Filter build that are unique per suite with revision as key and merge builds.
                    */
                    data = data
                        .filter(x => !x.hasOwnProperty('merge'))
                        .filter( // filter builds that are unique per suite with revision as key
                            x => !data
                                .filter(z => x != z && x.suite == z.suite) // exclude self build and build for other suites.
                                .map(z => z.revision) // map from array of build to array of revision
                                .includes(x.revision) // check if revision of x is index data.
                        ).concat(data.filter(x => x.hasOwnProperty('merge'))); // concat unique build with merged build.

                    data.sort((l,r) => r.timestamp - l.timestamp);
                    if(data.length > 0) {
                        var startdir;
                        if(hash) {
                            startdir = data.find(build => hash.includes(build.directory));
                            startdir.hash = hash;
                        }
                        if(!startdir) {
                            startdir = data[0];
                        }
                        $scope.directory = startdir;
                    }
                    $scope.data = data;
                });
                $scope.$watch('directory', (dir, olddir) => {
                    if(dir) {
                        var content = parent.document.getElementById("content");
                        var contentDocument = content.contentDocument || content.contentWindow.document;
                        var newpath;
                        if(olddir && olddir.suite === dir.suite) {
                            newpath = contentDocument.location.href.replace(olddir.directory, dir.directory);
                        } else {
                            newpath = dir.hasOwnProperty('hash') ? hash : dir.directory + "/coverage/";
                        }
                        contentDocument.location.href = newpath;
                        parent.window.history.replaceState(undefined, undefined, "#" + newpath.replace(/^.+coverage_upload\//, ""));
                    }
                });
                $scope.step = (i) => $scope.directory = $scope.data[$scope.data.indexOf($scope.directory)+i];
            });
        function copy(url) {
            var content = parent.document.getElementById("content");
            var contentDocument = content.contentDocument || content.contentWindow.document;
            var copyText = document.getElementById("copy");
            copyText.value = contentDocument.location.href.replace("coverage_upload/", "coverage_upload/#");
            copyText.select();
            document.execCommand("copy");
        }
        </script>
    </head>
    <body ng-app="myApp" ng-controller="IndexCtrl">
       <button ng-click="step(1)" ng-disabled="data.indexOf(directory) >= data.length-1">&lt;&lt;</button>
       <button ng-click="step(-1)" ng-disabled="data.indexOf(directory) <= 0">&gt;&gt;</button>
       <select ng-model="directory" ng-options="(i.primary_info['author-ts']*1000|date:'yy-MM-dd hh:mm') + ' ' + i.build_name + ' ' + i.revision.substr(0,8) group by i.suite for i in data"></select>
       <a href="{{directory.build_url}}" ng-if="directory.build_url" target="_blank">Build</a> Commit: {{directory.revision.substr(0,5)}}: {{directory.primary_info.description}}
       <input type="text" style="opacity: 0;width: 20;" id="copy" />
       <button style="float: right;" onclick="copy(window.location);">Share url</button>
    </body>
</html>""".replace("%js_library_url", js_library_url), remote_basedir + '/navigation.html')
Exemplo n.º 32
0
 def upload_string(content, path):
     mx.run(['ssh', remote_host, 'bash', '-c', 'cat > "' + path + '"'], stdin=content)
Exemplo n.º 33
0
def opt(args=None, version=None, out=None, err=None):
    """runs opt"""
    return mx.run([findLLVMProgram('opt', version)] + args, out=out, err=err)
Exemplo n.º 34
0
def gate_body(args, tasks):
    with Task('Vm: Basic GraalVM Tests', tasks,
              tags=[VmGateTasks.compiler]) as t:
        if t and mx_vm.has_component('Graal compiler'):
            # 1. a full open-source build should be built with an open-source JDK but we allow Oracle JDK in non-strict mode as it is common on developer machines
            # 2. the build must be a GraalVM
            # 3. the build must be JVMCI-enabled since the 'Graal compiler' component is registered
            version_regex = mx_vm.openjdk_version_regex if args.strict_mode else mx_vm.anyjdk_version_regex
            mx_vm.check_versions(
                mx_vm.graalvm_output(),
                version_regex,
                graalvm_version_regex=mx_vm.graalvm_version_regex,
                expect_graalvm=True,
                check_jvmci=True)

    with Task('Vm: Sulong tests', tasks, tags=[VmGateTasks.sulong]) as t:
        if t and mx_vm.has_component('Sulong', fatalIfMissing=True):
            pass

    with Task('Vm: Graal.js tests', tasks,
              tags=[VmGateTasks.graal_js_all]) as t:
        if t and mx_vm.has_component('Graal.js', fatalIfMissing=True):
            pass

    with Task('Vm: Graal.nodejs tests', tasks,
              tags=[VmGateTasks.graal_nodejs]) as t:
        if t and mx_vm.has_component('Graal.nodejs', fatalIfMissing=True):
            pass

    with Task('Vm: TruffleRuby tests', tasks,
              tags=[VmGateTasks.truffleruby]) as t:
        if t and mx_vm.has_component('TruffleRuby', fatalIfMissing=True):
            pass

    with Task('Vm: FastR tests', tasks, tags=[VmGateTasks.fastr]) as t:
        if t and mx_vm.has_component('FastR', fatalIfMissing=True):
            pass

    with Task('Vm: Graal.Python tests', tasks,
              tags=[VmGateTasks.graalpython]) as t:
        if t and mx_vm.has_component('Graal.Python', fatalIfMissing=True):
            pass

    if mx_vm.has_component('LibGraal'):
        libgraal_location = mx_vm.get_native_image_locations(
            'LibGraal', 'jvmcicompiler')
        if libgraal_location is None:
            mx.warn(
                "Skipping libgraal tests: no library enabled in the LibGraal component"
            )
        else:
            extra_vm_arguments = [
                '-XX:+UseJVMCICompiler', '-XX:+UseJVMCINativeLibrary',
                '-XX:JVMCILibPath=' + dirname(libgraal_location)
            ]
            if args.extra_vm_argument:
                extra_vm_arguments += args.extra_vm_argument
            import mx_compiler

            # run avrora on the GraalVM binary itself
            with Task('LibGraal Compiler:GraalVM DaCapo-avrora',
                      tasks,
                      tags=[VmGateTasks.libgraal]) as t:
                if t:
                    mx.run([
                        join(mx_vm.graalvm_home(), 'bin',
                             'java'), '-XX:+UseJVMCICompiler',
                        '-XX:+UseJVMCINativeLibrary', '-jar',
                        mx.library('DACAPO').get_path(True), 'avrora'
                    ])

            with Task('LibGraal Compiler:CTW',
                      tasks,
                      tags=[VmGateTasks.libgraal]) as t:
                if t:
                    mx_compiler.ctw([
                        '-DCompileTheWorld.Config=Inline=false CompilationFailureAction=ExitVM',
                        '-esa',
                        '-XX:+EnableJVMCI',
                        '-DCompileTheWorld.MultiThreaded=true',
                        '-Dgraal.InlineDuringParsing=false',
                        '-Dgraal.TrackNodeSourcePosition=true',
                        '-DCompileTheWorld.Verbose=false',
                        '-XX:ReservedCodeCacheSize=300m',
                    ], extra_vm_arguments)

            mx_compiler.compiler_gate_benchmark_runner(
                tasks, extra_vm_arguments, prefix='LibGraal Compiler:')

            with Task('LibGraal Truffle:unittest',
                      tasks,
                      tags=[VmGateTasks.libgraal]) as t:
                if t:

                    def _unittest_config_participant(config):
                        vmArgs, mainClass, mainClassArgs = config
                        newVmArgs = [
                            arg for arg in vmArgs if arg !=
                            "-Dtruffle.TruffleRuntime=com.oracle.truffle.api.impl.DefaultTruffleRuntime"
                        ]
                        return (newVmArgs, mainClass, mainClassArgs)

                    mx_unittest.add_config_participant(
                        _unittest_config_participant)
                    excluded_tests = environ.get("TEST_LIBGRAAL_EXCLUDE")
                    if excluded_tests:
                        with NamedTemporaryFile(prefix='blacklist.',
                                                mode='w',
                                                delete=False) as fp:
                            fp.file.writelines(
                                [l + '\n' for l in excluded_tests.split()])
                            unittest_args = ["--blacklist", fp.name]
                    else:
                        unittest_args = []
                    unittest_args = unittest_args + [
                        "--enable-timing", "--verbose"
                    ]
                    mx_unittest.unittest(unittest_args + extra_vm_arguments + [
                        "-Dgraal.TruffleCompileImmediately=true",
                        "-Dgraal.TruffleBackgroundCompilation=false", "truffle"
                    ])
    else:
        mx.warn("Skipping libgraal tests: component not enabled")

    gate_substratevm(tasks)
    gate_sulong(tasks)
    gate_ruby(tasks)
    gate_python(tasks)
Exemplo n.º 35
0
    def build(self):
        source_dir = self.subject.getSourceDir()
        output_dir = self.subject.getOutputDir()
        if not emcc_dir:
            mx.abort("No EMCC_DIR specified - the source programs will not be compiled to .wasm.")
        emcc_cmd = os.path.join(emcc_dir, "emcc")
        gcc_cmd = os.path.join(gcc_dir, "gcc")
        if mx.run([emcc_cmd, "-v"], nonZeroIsFatal=False) != 0:
            mx.abort("Could not check the emcc version.")
        if mx.run([gcc_cmd, "--version"], nonZeroIsFatal=False) != 0:
            mx.abort("Could not check the gcc version.")
        if not wabt_dir:
            mx.abort("Set WABT_DIR if you want the binary to include .wat files.")
        mx.log("Building files from the source dir: " + source_dir)
        cc_flags = ["-g2", "-O3"]
        include_flags = []
        if hasattr(self.project, "includeset"):
            include_flags = ["-I", os.path.join(_suite.dir, "includes", self.project.includeset)]
        emcc_flags = ["-s", "EXIT_RUNTIME=1", "-s", "STANDALONE_WASM", "-s", "WASM_BIGINT"] + cc_flags
        if self.project.isBenchmarkProject():
            emcc_flags = emcc_flags + ["-s", "EXPORTED_FUNCTIONS=" + str(benchmark_methods).replace("'", "\"") + ""]
        subdir_program_names = defaultdict(lambda: [])
        for root, filename in self.subject.getProgramSources():
            subdir = os.path.relpath(root, self.subject.getSourceDir())
            mx.ensure_dir_exists(os.path.join(output_dir, subdir))

            basename = remove_extension(filename)
            source_path = os.path.join(root, filename)
            output_wasm_path = os.path.join(output_dir, subdir, basename + ".wasm")
            output_js_path = os.path.join(output_dir, subdir, basename + ".js")
            timestampedSource = mx.TimeStampFile(source_path)
            timestampedOutput = mx.TimeStampFile(output_wasm_path)
            mustRebuild = timestampedSource.isNewerThan(timestampedOutput) or not timestampedOutput.exists()

            # Step 1: build the .wasm binary.
            if mustRebuild:
                if filename.endswith(".c"):
                    # This generates both a js file and a wasm file.
                    # See https://github.com/emscripten-core/emscripten/wiki/WebAssembly-Standalone
                    build_cmd_line = [emcc_cmd] + emcc_flags + [source_path, "-o", output_js_path] + include_flags
                    if mx.run(build_cmd_line, nonZeroIsFatal=False) != 0:
                        mx.abort("Could not build the wasm-only output of " + filename + " with emcc.")
                elif filename.endswith(".wat"):
                    # Step 1: compile the .wat file to .wasm.
                    wat2wasm_cmd = os.path.join(wabt_dir, "wat2wasm")
                    build_cmd_line = [wat2wasm_cmd, "-o", output_wasm_path, source_path]
                    if mx.run(build_cmd_line, nonZeroIsFatal=False) != 0:
                        mx.abort("Could not translate " + filename + " to binary format.")
                elif filename.endswith(".wasm"):
                    shutil.copyfile(source_path, output_wasm_path)

            else:
                mx.logv("skipping, file is up-to-date: " + source_path)

            # Step 2: copy the result file if it exists.
            result_path = os.path.join(root, basename + ".result")
            if os.path.isfile(result_path):
                result_output_path = os.path.join(output_dir, subdir, basename + ".result")
                shutil.copyfile(result_path, result_output_path)

            # Step 3: copy the opts file if it exists.
            opts_path = os.path.join(root, basename + ".opts")
            if os.path.isfile(opts_path):
                opts_output_path = os.path.join(output_dir, subdir, basename + ".opts")
                shutil.copyfile(opts_path, opts_output_path)

            output_wat_path = os.path.join(output_dir, subdir, basename + ".wat")
            if mustRebuild:
                if filename.endswith(".c"):
                    # Step 4: produce the .wat files, for easier debugging.
                    wasm2wat_cmd = os.path.join(wabt_dir, "wasm2wat")
                    if mx.run([wasm2wat_cmd, "-o", output_wat_path, output_wasm_path], nonZeroIsFatal=False) != 0:
                        mx.abort("Could not compile .wat file for " + filename)
                elif filename.endswith(".wat"):
                    # Step 4: copy the .wat file, for easier debugging.
                    wat_path = os.path.join(root, basename + ".wat")
                    shutil.copyfile(wat_path, output_wat_path)

            # Step 5: if this is a benchmark project, create native binaries too.
            if mustRebuild:
                if filename.endswith(".c"):
                    mx.ensure_dir_exists(os.path.join(output_dir, subdir, NATIVE_BENCH_DIR))
                    output_path = os.path.join(output_dir, subdir, NATIVE_BENCH_DIR, mx.exe_suffix(basename))
                    link_flags = ["-lm"]
                    gcc_cmd_line = [gcc_cmd] + cc_flags + [source_path, "-o", output_path] + include_flags + link_flags
                    if mx.run(gcc_cmd_line, nonZeroIsFatal=False) != 0:
                        mx.abort("Could not build the native binary of " + filename + ".")
                    os.chmod(output_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
                elif filename.endswith(".wat"):
                    mx.warn("The .wat files are not translated to native binaries: " + filename)

            # Remember the source name.
            subdir_program_names[subdir].append(basename)
        for subdir in subdir_program_names:
            with open(os.path.join(output_dir, subdir, "wasm_test_index"), "w") as f:
                for name in subdir_program_names[subdir]:
                    f.write(name)
                    f.write("\n")
Exemplo n.º 36
0
def graalpython_gate_runner(args, tasks):
    # JUnit tests
    with Task('GraalPython JUnit', tasks,
              tags=[GraalPythonTags.junit]) as task:
        if task:
            punittest(['--verbose'])

    # Unittests on JVM
    with Task('GraalPython Python unittests',
              tasks,
              tags=[GraalPythonTags.unittest]) as task:
        if task:
            if platform.system() != 'Darwin':
                # TODO: drop condition when python3 is available on darwin
                mx.log("Running tests with CPython")
                test_args = [_graalpytest_driver(), "-v", _graalpytest_root()]
                mx.run(["python3"] + test_args, nonZeroIsFatal=True)
            mx.run(["env"])
            run_python_unittests(python_gvm())

    with Task('GraalPython sandboxed tests',
              tasks,
              tags=[GraalPythonTags.unittest_sandboxed]) as task:
        if task:
            run_python_unittests(python_gvm(["sandboxed"]),
                                 args=["--llvm.managed"])

    with Task('GraalPython Jython emulation tests',
              tasks,
              tags=[GraalPythonTags.unittest_jython]) as task:
        if task:
            run_python_unittests(python_gvm(),
                                 args=["--python.EmulateJython"],
                                 paths=["test_interop.py"])

    with Task('GraalPython Python tests', tasks,
              tags=[GraalPythonTags.tagged]) as task:
        if task:
            with set_env(ENABLE_CPYTHON_TAGGED_UNITTESTS="true",
                         ENABLE_THREADED_GRAALPYTEST="true"):
                run_python_unittests(python_gvm(),
                                     args=["--python.WithThread=true"],
                                     paths=["test_tagged_unittests.py"])

    # Unittests on SVM
    with Task('GraalPython tests on SVM',
              tasks,
              tags=[GraalPythonTags.svmunit]) as task:
        if task:
            run_python_unittests(python_svm())

    with Task('GraalPython sandboxed tests on SVM',
              tasks,
              tags=[GraalPythonTags.svmunit_sandboxed]) as task:
        if task:
            run_python_unittests(python_svm(["sandboxed"]),
                                 args=["--llvm.managed"])

    with Task('GraalPython license header update',
              tasks,
              tags=[GraalPythonTags.license]) as task:
        if task:
            python_checkcopyrights([])

    with Task('GraalPython GraalVM shared-library build',
              tasks,
              tags=[GraalPythonTags.shared_object,
                    GraalPythonTags.graalvm]) as task:
        if task:
            run_shared_lib_test()

    with Task('GraalPython GraalVM sandboxed shared-library build',
              tasks,
              tags=[
                  GraalPythonTags.shared_object_sandboxed,
                  GraalPythonTags.graalvm_sandboxed
              ]) as task:
        if task:
            run_shared_lib_test(["sandboxed"])

    with Task('GraalPython GraalVM build',
              tasks,
              tags=[GraalPythonTags.svm, GraalPythonTags.graalvm]) as task:
        if task:
            svm_image = python_svm(["--version"])
            benchmark = os.path.join(PATH_MESO, "image-magix.py")
            out = mx.OutputCapture()
            mx.run([
                svm_image, "-v", "-S", "--log.python.level=FINEST", benchmark
            ],
                   nonZeroIsFatal=True,
                   out=mx.TeeOutputCapture(out))
            success = "\n".join([
                "[0, 0, 0, 0, 0, 0, 10, 10, 10, 0, 0, 10, 3, 10, 0, 0, 10, 10, 10, 0, 0, 0, 0, 0, 0]",
            ])
            if success not in out.data:
                mx.abort('Output from generated SVM image "' + svm_image +
                         '" did not match success pattern:\n' + success)
            # Test that stdlib paths are not cached on packages
            out = mx.OutputCapture()
            mx.run([
                svm_image, "-v", "-S", "--log.python.level=FINEST",
                "--python.StdLibHome=/foobar", "-c",
                "import encodings; print(encodings.__path__)"
            ],
                   out=mx.TeeOutputCapture(out))
            if "/foobar" not in out.data:
                mx.abort('Output from generated SVM image "' + svm_image +
                         '" did not have patched std lib path "/foobar"')
            # Test that stdlib paths are not cached on modules
            out = mx.OutputCapture()
            mx.run([
                svm_image, "-v", "-S", "--log.python.level=FINEST",
                "--python.StdLibHome=/foobar", "-c",
                "import encodings; print(encodings.__file__)"
            ],
                   out=mx.TeeOutputCapture(out))
            if "/foobar" not in out.data:
                mx.abort('Output from generated SVM image "' + svm_image +
                         '" did not have patched std lib path "/foobar"')
            # Finally, test that we can start even if the graalvm was moved
            out = mx.OutputCapture()
            graalvm_home = svm_image.replace(
                os.path.sep.join(["", "bin", "graalpython"]), "")
            new_graalvm_home = graalvm_home + "_new"
            shutil.move(graalvm_home, new_graalvm_home)
            launcher = os.path.join(new_graalvm_home, "bin", "graalpython")
            mx.log(launcher)
            mx.run([
                launcher, "--log.python.level=FINE", "-S", "-c",
                "print(b'abc'.decode('ascii'))"
            ],
                   out=mx.TeeOutputCapture(out),
                   err=mx.TeeOutputCapture(out))
            assert "Using preinitialized context." in out.data

    with Task('GraalPython GraalVM native embedding',
              tasks,
              tags=[
                  GraalPythonTags.svm, GraalPythonTags.graalvm,
                  GraalPythonTags.native_image_embedder
              ]) as task:
        if task:
            run_embedded_native_python_test()
Exemplo n.º 37
0
 def _native_image(args, **kwargs):
     mx.run([native_image_cmd] + args, **kwargs)
Exemplo n.º 38
0
def gnu_r(args):
    '''
    run the internally built GNU R executable'
    '''
    cmd = [join(_gnur_path(), 'bin', 'R')] + args
    return mx.run(cmd, nonZeroIsFatal=False)
Exemplo n.º 39
0
def _update_JVMCI_library():
    """
    Updates the "path" and "sha1" attributes of the "JVMCI" library to
    refer to a jvmci.jar created from the JVMCI classes in JDK9.
    """
    suiteDict = _suite.suiteDict
    jvmciLib = suiteDict['libraries']['JVMCI']
    d = join(_suite.get_output_root(), abspath(_jdk.home)[1:])
    path = join(d, 'jvmci.jar')

    explodedModule = join(_jdk.home, 'modules', 'jdk.vm.ci')
    if exists(explodedModule):
        jarInputs = {}
        newestJarInput = None
        for root, _, files in os.walk(explodedModule):
            relpath = root[len(explodedModule) + 1:]
            for f in files:
                arcname = join(relpath, f).replace(os.sep, '/')
                jarInput = join(root, f)
                jarInputs[arcname] = jarInput
                t = mx.TimeStampFile(jarInput)
                if newestJarInput is None or t.isNewerThan(newestJarInput):
                    newestJarInput = t
        if not exists(path) or newestJarInput.isNewerThan(path):
            with mx.Archiver(path, kind='zip') as arc:
                for arcname, jarInput in jarInputs.iteritems():
                    with open(jarInput, 'rb') as fp:
                        contents = fp.read()
                        arc.zf.writestr(arcname, contents)
    else:
        # Use the jdk.internal.jimage utility since it's the only way
        # to partially read .jimage files as the JDK9 jimage tool
        # does not support partial extraction.
        bootmodules = join(_jdk.home, 'lib', 'modules', 'bootmodules.jimage')
        if not exists(bootmodules):
            mx.abort('Could not find JVMCI classes at ' + bootmodules + ' or ' + explodedModule)
        if not exists(path) or mx.TimeStampFile(bootmodules).isNewerThan(path):
            mx.ensure_dir_exists(d)
            javaSource = join(d, 'ExtractJVMCI.java')
            with open(javaSource, 'w') as fp:
                print >> fp, """import java.io.FileOutputStream;
    import java.util.jar.JarEntry;
    import java.util.jar.JarOutputStream;
    import jdk.internal.jimage.BasicImageReader;

    public class ExtractJVMCI {
    public static void main(String[] args) throws Exception {
        BasicImageReader image = BasicImageReader.open(args[0]);
        String[] names = image.getEntryNames();
        if (names.length == 0) {
            return;
        }
        try (JarOutputStream jos = new JarOutputStream(new FileOutputStream(args[1]))) {
            for (String name : names) {
                if (name.startsWith("/jdk.vm.ci/")) {
                    String ename = name.substring("/jdk.vm.ci/".length());
                    JarEntry je = new JarEntry(ename);
                    jos.putNextEntry(je);
                    jos.write(image.getResource(name));
                    jos.closeEntry();
                }
            }
        }
    }
    }
    """
            mx.run([_jdk.javac, '-d', d, javaSource])
            mx.run([_jdk.java, '-cp', d, 'ExtractJVMCI', bootmodules, path])
            if not exists(path):
                mx.abort('Could not find the JVMCI classes in ' + bootmodules)

    jvmciLib['path'] = path
    jvmciLib['sha1'] = mx.sha1OfFile(path)
Exemplo n.º 40
0
def gnu_rtests(args, env=None):
    '''
    run tests of the internally built GNU R under tests subdirectory
    '''
    os.chdir(_fastr_suite.dir)  # Packages install fails otherwise
    #   mx_fastr_pkgs.installpkgs(['--pkg-pattern', '^MASS$']) # required by tests/Examples/base-Ex.R
    np = mx.project('com.oracle.truffle.r.native')
    tst = join(np.dir, 'gnur', 'tests')
    tstsrc = join(tst, 'src')
    tstlog = join(tst, 'log')
    shutil.rmtree(tstlog, True)
    os.mkdir(tstlog)
    diffname = join(tstlog, 'all.diff')
    diff = open(diffname, 'a')
    try:
        for subd in ['Examples', '']:
            logd = join(tstlog, subd)
            if subd != '':
                os.mkdir(logd)
            os.chdir(logd)
            srcd = join(tstsrc, subd)
            for f in sorted(os.listdir(srcd)):
                if f.endswith('.R'):
                    print(
                        'Running {} explicitly by FastR CMD BATCH ...'.format(
                            f))  # pylint: disable=superfluous-parens
                    mx.run(
                        [r_path(), '--vanilla', 'CMD', 'BATCH',
                         join(srcd, f)] + args,
                        nonZeroIsFatal=False,
                        env=env,
                        timeout=90)
                    outf = f + 'out'
                    if os.path.isfile(outf):
                        outff = outf + '.fastr'
                        os.rename(outf, outff)
                        print('Running {} explicitly by GnuR CMD BATCH ...'.
                              format(f))  # pylint: disable=superfluous-parens
                        mx.run([
                            join(_gnur_path(), 'bin', 'R'), '--vanilla', 'CMD',
                            'BATCH',
                            join(srcd, f)
                        ] + args,
                               nonZeroIsFatal=False,
                               env=env,
                               timeout=90)
                        if os.path.isfile(outf):
                            outfg = outf + '.gnur'
                            os.rename(outf, outfg)
                            diff.write('\nRdiff {} {}:\n'.format(outfg, outff))
                            diff.flush()
                            subprocess.Popen(
                                [r_path(), 'CMD', 'Rdiff', outfg, outff],
                                stdout=diff,
                                stderr=diff,
                                shell=False)
                            diff.flush()
        diff.close()
        print('FastR to GnuR diff was written to {}'.format(diffname))  # pylint: disable=superfluous-parens
    finally:
        shutil.rmtree(join(_fastr_suite.dir, 'deparse'), True)
Exemplo n.º 41
0
def dragonEggGPP(args=None):
    """executes G++ with dragonegg"""
    executeCommand = [getGPP(), "-fplugin=" + dragonEggPath(), '-fplugin-arg-dragonegg-emit-ir']
    return mx.run(executeCommand + args)
Exemplo n.º 42
0
 def run_java(self, args, out=None, err=None, cwd=None, nonZeroIsFatal=False):
     """Run 'java' workloads."""
     self.extract_vm_info(args)
     return mx.run([os.path.join(mx_sdk_vm_impl.graalvm_home(fatalIfMissing=True), 'bin', 'java')] + args, out=out, err=err, cwd=cwd, nonZeroIsFatal=nonZeroIsFatal)
Exemplo n.º 43
0
 def run_output(args, cwd=None):
     out = mx.OutputCapture()
     mx.run(args, cwd=cwd, out=out, err=out)
     return out.data
Exemplo n.º 44
0
    def run_java(self,
                 args,
                 out=None,
                 err=None,
                 cwd=None,
                 nonZeroIsFatal=False):
        if '-version' in args:
            return super(NativeImageVM,
                         self).run_java(args,
                                        out=out,
                                        err=err,
                                        cwd=cwd,
                                        nonZeroIsFatal=nonZeroIsFatal)
        else:
            image_cwd = os.path.abspath(cwd if cwd else os.getcwd())
            non_zero_is_fatal = self.is_gate or nonZeroIsFatal
            config = NativeImageVM.BenchmarkConfig()
            original_java_run_args = config.parse(args)
            executable, classpath_arguments, system_properties, image_run_args = NativeImageVM.extract_benchmark_arguments(
                original_java_run_args)
            executable_name = (os.path.splitext(os.path.basename(
                executable[1]))[0] if executable[0] == '-jar' else
                               executable[0]).lower()
            image_path = os.path.join(image_cwd, executable_name)
            config.profile_dir = mx.mkdtemp(suffix='profile',
                                            prefix='native-image')
            profile_path = os.path.join(config.profile_dir,
                                        executable_name + '.iprof')

            # Agent configuration and/or HotSpot profiling
            needs_config = (config.config_dir is None) and config.needs_config
            if needs_config or self.hotspot_pgo:
                hotspot_vm_args = ['-ea', '-esa'] if self.is_gate else []
                hotspot_run_args = []

                if needs_config:
                    config.config_dir = mx.mkdtemp(suffix='config',
                                                   prefix='native-image')
                    hotspot_vm_args += [
                        '-agentlib:native-image-agent=config-output-dir=' +
                        str(config.config_dir)
                    ]

                if self.hotspot_pgo:
                    hotspot_vm_args += [
                        '-Dgraal.PGOInstrument=' + profile_path
                    ]

                if config.extra_agent_run_args:
                    hotspot_run_args += config.extra_profile_run_args if self.hotspot_pgo and not self.is_gate else config.extra_agent_run_args
                else:
                    hotspot_run_args += image_run_args

                hotspot_args = hotspot_vm_args + classpath_arguments + executable + system_properties + hotspot_run_args

                mx.log(
                    'Running with HotSpot to get the configuration files and profiles. This could take a while:'
                )
                super(NativeImageVM,
                      self).run_java(hotspot_args,
                                     out=None,
                                     err=None,
                                     cwd=image_cwd,
                                     nonZeroIsFatal=non_zero_is_fatal)

            base_image_build_args = [
                os.path.join(mx_vm.graalvm_home(fatalIfMissing=True), 'bin',
                             'native-image')
            ]
            base_image_build_args += ['--no-fallback']
            base_image_build_args += [
                '-J-ea', '-J-esa', '-H:+VerifyGraalGraphs', '-H:+VerifyPhases',
                '-H:+TraceClassInitialization'
            ] if self.is_gate else []
            base_image_build_args += system_properties
            base_image_build_args += classpath_arguments
            base_image_build_args += executable
            base_image_build_args += [
                '-H:Name=' + executable_name, '-H:Path=' + image_cwd
            ]
            if needs_config:
                base_image_build_args += [
                    '-H:ConfigurationFileDirectories=' + config.config_dir
                ]
            base_image_build_args += config.extra_image_build_arguments

            # PGO instrumentation
            i = 0
            while i < self.pgo_instrumented_iterations:
                instrument_args = ['--pgo-instrument'] + (
                    [] if i == 0 and not self.hotspot_pgo else ['--pgo'])
                instrument_image_build_args = base_image_build_args + instrument_args
                mx.log('Building the instrumentation image with: ')
                mx.log(' ' + ' '.join([
                    pipes.quote(str(arg))
                    for arg in instrument_image_build_args
                ]))
                mx.run(instrument_image_build_args,
                       out=None,
                       err=None,
                       cwd=image_cwd,
                       nonZeroIsFatal=non_zero_is_fatal)

                image_run_cmd = [image_path]
                image_run_cmd += ['-XX:ProfilesDumpFile=' + profile_path]
                if config.extra_profile_run_args:
                    image_run_cmd += config.extra_profile_run_args
                else:
                    image_run_cmd += image_run_args + config.extra_run_args

                mx.log('Running the instrumented image with: ')
                mx.log(
                    ' ' +
                    ' '.join([pipes.quote(str(arg)) for arg in image_run_cmd]))
                mx.run(image_run_cmd,
                       out=out,
                       err=err,
                       cwd=image_cwd,
                       nonZeroIsFatal=non_zero_is_fatal)

                i += 1

            # Build the final image
            pgo_args = ['--pgo=' + profile_path] if self.pgo_instrumented_iterations > 0 or self.hotspot_pgo else []
            final_image_args = base_image_build_args + pgo_args
            mx.log('Building the final image with: ')
            mx.log(
                ' ' +
                ' '.join([pipes.quote(str(arg)) for arg in final_image_args]))
            mx.run(final_image_args,
                   out=None,
                   err=None,
                   cwd=image_cwd,
                   nonZeroIsFatal=non_zero_is_fatal)

            # Execute the benchmark
            image_run_cmd = [image_path
                             ] + image_run_args + config.extra_run_args
            mx.log('Running the produced native executable with: ')
            mx.log(' ' +
                   ' '.join([pipes.quote(str(arg)) for arg in image_run_cmd]))
            mx.run(image_run_cmd,
                   out=out,
                   err=err,
                   cwd=image_cwd,
                   nonZeroIsFatal=non_zero_is_fatal)
Exemplo n.º 45
0
def updategraalinopenjdk(args):
    """updates the Graal sources in OpenJDK"""
    parser = ArgumentParser(prog='mx updategraalinopenjdk')
    parser.add_argument(
        '--pretty',
        help=
        'value for --pretty when logging the changes since the last JDK* tag')
    parser.add_argument('jdkrepo', help='path to the local OpenJDK repo')
    parser.add_argument('version',
                        type=int,
                        help='Java version of the OpenJDK repo')

    args = parser.parse_args(args)

    if mx_compiler.jdk.javaCompliance.value < args.version:
        mx.abort(
            'JAVA_HOME/--java-home must be Java version {} or greater: {}'.
            format(args.version, mx_compiler.jdk))

    graal_modules = [
        # JDK module jdk.internal.vm.compiler is composed of sources from:
        GraalJDKModule(
            'jdk.internal.vm.compiler',
            # 1. Classes in the compiler suite under the org.graalvm namespace except for packages
            #    or projects whose names include "truffle", "management" or "core.llvm"
            [
                SuiteJDKInfo('compiler', ['org.graalvm'],
                             ['truffle', 'management', 'core.llvm']),
                # 2. Classes in the sdk suite under the org.graalvm.collections and org.graalvm.word namespaces
                SuiteJDKInfo(
                    'sdk', ['org.graalvm.collections', 'org.graalvm.word'], [])
            ]),
        # JDK module jdk.internal.vm.compiler.management is composed of sources from:
        GraalJDKModule(
            'jdk.internal.vm.compiler.management',
            # 1. Classes in the compiler suite under the org.graalvm.compiler.hotspot.management namespace
            [
                SuiteJDKInfo('compiler',
                             ['org.graalvm.compiler.hotspot.management'], [])
            ]),
        # JDK module jdk.aot is composed of sources from:
        GraalJDKModule(
            'jdk.aot',
            # 1. Classes in the compiler suite under the jdk.tools.jaotc namespace
            [SuiteJDKInfo('compiler', ['jdk.tools.jaotc'], [])]),
    ]

    # Strings to be replaced in files copied to OpenJDK.
    replacements = {
        'published by the Free Software Foundation.  Oracle designates this\n * particular file as subject to the "Classpath" exception as provided\n * by Oracle in the LICENSE file that accompanied this code.':
        'published by the Free Software Foundation.',
        _read_sibling_file('upl_substring.txt'):
        _read_sibling_file('gplv2_substring.txt')
    }

    # Strings that must not exist in OpenJDK source files. This is applied after replacements are made.
    blacklist = ['"Classpath" exception']

    jdkrepo = args.jdkrepo
    git_repo = _is_git_repo(jdkrepo)

    for m in graal_modules:
        m_src_dir = join(jdkrepo, 'src', m.name)
        if not exists(m_src_dir):
            mx.abort(jdkrepo + ' does not look like a JDK repo - ' +
                     m_src_dir + ' does not exist')

    def run_output(args, cwd=None):
        out = mx.OutputCapture()
        mx.run(args, cwd=cwd, out=out, err=out)
        return out.data

    for m in graal_modules:
        m_src_dir = join('src', m.name)
        mx.log('Checking ' + m_src_dir)
        if git_repo:
            out = run_output(['git', 'status', '-s', m_src_dir], cwd=jdkrepo)
        else:
            out = run_output(['hg', 'status', m_src_dir], cwd=jdkrepo)
        if out:
            mx.abort(jdkrepo + ' is not "clean":' + '\n' +
                     out[:min(200, len(out))] + '...')

    for dirpath, _, filenames in os.walk(join(jdkrepo, 'make')):
        for filename in filenames:
            if filename.endswith('.gmk'):
                rename_packages(join(dirpath, filename), True)

    java_package_re = re.compile(
        r"^\s*package\s+(?P<package>[a-zA-Z_][\w\.]*)\s*;$", re.MULTILINE)

    copied_source_dirs = []
    jdk_internal_vm_compiler_EXCLUDES = set()  # pylint: disable=invalid-name
    jdk_internal_vm_compiler_test_SRC = set()  # pylint: disable=invalid-name
    # Add org.graalvm.compiler.processor since it is only a dependency
    # for (most) Graal annotation processors and is not needed to
    # run Graal.
    jdk_internal_vm_compiler_EXCLUDES.add('org.graalvm.compiler.processor')
    for m in graal_modules:
        classes_dir = join(jdkrepo, 'src', m.name, 'share', 'classes')
        for info in m.suites:
            mx.log('Processing ' + m.name + ':' + info.name)
            for e in os.listdir(classes_dir):
                if any(inc in e for inc in info.includes) and not any(
                        ex in e for ex in info.excludes):
                    project_dir = join(classes_dir, e)
                    shutil.rmtree(project_dir)
                    mx.log('  removed ' + project_dir)
            suite = mx.suite(info.name)

            worklist = []

            for p in [e for e in suite.projects if e.isJavaProject()]:
                if any(inc in p.name for inc in info.includes) and not any(
                        ex in p.name for ex in info.excludes):
                    assert len(p.source_dirs()) == 1, p
                    version = 0
                    new_project_name = p.name
                    if hasattr(p, 'multiReleaseJarVersion'):
                        version = int(getattr(p, 'multiReleaseJarVersion'))
                        if version <= args.version:
                            base_project = _find_version_base_project(p)
                            new_project_name = base_project.name
                        else:
                            continue

                    for old_name, new_name in package_renamings.items():
                        if new_project_name.startswith(old_name):
                            new_project_name = new_project_name.replace(
                                old_name, new_name)

                    source_dir = p.source_dirs()[0]
                    target_dir = join(classes_dir, new_project_name, 'src')
                    copied_source_dirs.append(source_dir)

                    workitem = (version, p, source_dir, target_dir)
                    worklist.append(workitem)

            # Ensure versioned resources are copied in the right order
            # such that higher versions override lower versions.
            worklist = sorted(worklist)

            for version, p, source_dir, target_dir in worklist:
                first_file = True
                for dirpath, _, filenames in os.walk(source_dir):
                    for filename in filenames:
                        src_file = join(dirpath, filename)
                        dst_file = join(target_dir,
                                        os.path.relpath(src_file, source_dir))
                        with open(src_file) as fp:
                            contents = fp.read()
                        old_line_count = len(contents.split('\n'))
                        if filename.endswith('.java'):
                            for old_name, new_name in package_renamings.items(
                            ):
                                old_name_as_dir = old_name.replace('.', os.sep)
                                if old_name_as_dir in src_file:
                                    new_name_as_dir = new_name.replace(
                                        '.', os.sep)
                                    dst = src_file.replace(
                                        old_name_as_dir, new_name_as_dir)
                                    dst_file = join(
                                        target_dir,
                                        os.path.relpath(dst, source_dir))
                                contents = contents.replace(old_name, new_name)
                            for old_line, new_line in replacements.items():
                                contents = contents.replace(old_line, new_line)

                            match = java_package_re.search(contents)
                            if not match:
                                mx.abort(
                                    'Could not find package declaration in {}'.
                                    format(src_file))
                            java_package = match.group('package')
                            if any(ex in java_package for ex in info.excludes):
                                mx.log('  excluding ' + filename)
                                continue

                            new_line_count = len(contents.split('\n'))
                            if new_line_count > old_line_count:
                                mx.abort(
                                    'Pattern replacement caused line count to grow from {} to {} in {}'
                                    .format(old_line_count, new_line_count,
                                            src_file))
                            else:
                                if new_line_count < old_line_count:
                                    contents = contents.replace(
                                        '\npackage ', '\n' *
                                        (old_line_count - new_line_count) +
                                        '\npackage ')
                            new_line_count = len(contents.split('\n'))
                            if new_line_count != old_line_count:
                                mx.abort('Unable to correct line count for {}'.
                                         format(src_file))
                            for forbidden in blacklist:
                                if forbidden in contents:
                                    mx.abort(
                                        'Found blacklisted pattern \'{}\' in {}'
                                        .format(forbidden, src_file))
                        dst_dir = os.path.dirname(dst_file)
                        if not exists(dst_dir):
                            os.makedirs(dst_dir)
                        if first_file:
                            mx.log('  copying: ' + source_dir)
                            mx.log('       to: ' + target_dir)
                            if p.testProject or p.definedAnnotationProcessors:
                                to_exclude = p.name
                                for old_name, new_name in package_renamings.items(
                                ):
                                    if to_exclude.startswith(old_name):
                                        sfx = '' if to_exclude == old_name else to_exclude[
                                            len(old_name):]
                                        to_exclude = new_name + sfx
                                        break
                                jdk_internal_vm_compiler_EXCLUDES.add(
                                    to_exclude)
                                if p.testProject:
                                    jdk_internal_vm_compiler_test_SRC.add(
                                        to_exclude)
                            first_file = False
                        with open(dst_file, 'w') as fp:
                            fp.write(contents)

    def replace_lines(filename,
                      begin_lines,
                      end_line,
                      replace_lines,
                      old_line_check,
                      preserve_indent=False,
                      append_mode=False):
        mx.log('Updating ' + filename + '...')
        old_lines = []
        new_lines = []
        with open(filename) as fp:
            for begin_line in begin_lines:
                line = fp.readline()
                while line:
                    stripped_line = line.strip()
                    if stripped_line == begin_line:
                        new_lines.append(line)
                        break
                    new_lines.append(line)
                    line = fp.readline()
                assert line, begin_line + ' not found'

            lines = fp.readlines()
            line_in_def = True

            indent = 0
            if preserve_indent:
                line = lines[0]
                lstripped_line = line.lstrip()
                indent = len(line) - len(lstripped_line)

            if not append_mode:
                for replace in replace_lines:
                    new_lines.append(' ' * indent + replace)

            for line in lines:
                stripped_line = line.strip()
                if line_in_def:
                    if stripped_line == end_line:
                        line_in_def = False
                        new_lines.append(line)
                    else:
                        old_line_check(line)
                        if append_mode:
                            new_lines.append(line)
                    if append_mode and not line_in_def:
                        # reach end line and append new lines
                        for replace in replace_lines:
                            new_lines.append(replace)
                else:
                    new_lines.append(line)
        with open(filename, 'w') as fp:
            for line in new_lines:
                fp.write(line)
        return old_lines

    def single_column_with_continuation(line):
        parts = line.split()
        assert len(parts) == 2 and parts[1] == '\\', line

    # Update jdk.internal.vm.compiler.EXCLUDES in make/CompileJavaModules.gmk
    # to exclude all test, benchmark and annotation processor packages.
    CompileJavaModules_gmk = join(jdkrepo, 'make', 'CompileJavaModules.gmk')  # pylint: disable=invalid-name
    new_lines = []
    for pkg in sorted(jdk_internal_vm_compiler_EXCLUDES):
        new_lines.append(pkg + ' \\\n')
    begin_lines = ['jdk.internal.vm.compiler_EXCLUDES += \\']
    end_line = '#'
    old_line_check = single_column_with_continuation
    replace_lines(CompileJavaModules_gmk,
                  begin_lines,
                  end_line,
                  new_lines,
                  old_line_check,
                  preserve_indent=True)

    if args.version == 11:
        # add aot exclude
        out = run_output(['grep', 'jdk.aot_EXCLUDES', CompileJavaModules_gmk],
                         cwd=jdkrepo)
        if out:
            # replace existing exclude setting
            begin_lines = ['jdk.aot_EXCLUDES += \\']
            end_line = '#'
            new_lines = ['jdk.tools.jaotc.test \\\n']
            replace_lines(CompileJavaModules_gmk,
                          begin_lines,
                          end_line,
                          new_lines,
                          old_line_check,
                          preserve_indent=True)
        else:
            # append exclude setting after jdk.internal.vm.compiler_EXCLUDES
            new_lines = [
                '\n', 'jdk.aot_EXCLUDES += \\\n',
                '    jdk.tools.jaotc.test \\\n', '    #\n', '\n'
            ]  # indent is inlined
            replace_lines(CompileJavaModules_gmk,
                          begin_lines,
                          end_line,
                          new_lines,
                          old_line_check,
                          preserve_indent=True,
                          append_mode=True)

    # Update 'SRC' in the 'Compile graalunit tests' section of make/test/JtregGraalUnit.gmk
    # to include all test packages.
    JtregGraalUnit_gmk = join(jdkrepo, 'make', 'test', 'JtregGraalUnit.gmk')  # pylint: disable=invalid-name
    new_lines = []
    jdk_internal_vm_compiler_test_SRC.discard('jdk.tools.jaotc.test')
    jdk_internal_vm_compiler_test_SRC.discard(
        'org.graalvm.compiler.microbenchmarks')
    jdk_internal_vm_compiler_test_SRC.discard(
        'org.graalvm.compiler.virtual.bench')
    jdk_internal_vm_compiler_test_SRC.discard('org.graalvm.micro.benchmarks')
    for pkg in sorted(jdk_internal_vm_compiler_test_SRC):
        new_lines.append('$(SRC_DIR)/' + pkg + '/src \\\n')
    if args.version == 11:
        begin_lines = ['### Compile and build graalunit tests', 'SRC := \\']
    else:
        begin_lines = ['### Compile graalunit tests', 'SRC := \\']
    end_line = ', \\'
    old_line_check = single_column_with_continuation
    replace_lines(JtregGraalUnit_gmk,
                  begin_lines,
                  end_line,
                  new_lines,
                  old_line_check,
                  preserve_indent=True)

    overwritten = ''
    if not git_repo:
        mx.log('Adding new files to HG...')
        m_src_dirs = []
        for m in graal_modules:
            m_src_dirs.append(join('src', m.name))
        out = run_output([
            'hg', 'log', '-r', 'last(keyword("Update Graal"))', '--template',
            '{rev}'
        ] + m_src_dirs,
                         cwd=jdkrepo)
        last_graal_update = out.strip()

        for m in graal_modules:
            m_src_dir = join('src', m.name)
            if last_graal_update:
                overwritten += run_output([
                    'hg', 'diff', '-r', last_graal_update, '-r', 'tip',
                    m_src_dir
                ],
                                          cwd=jdkrepo)
            mx.run(['hg', 'add', m_src_dir], cwd=jdkrepo)
        mx.log('Removing old files from HG...')
        for m in graal_modules:
            m_src_dir = join('src', m.name)
            out = run_output(['hg', 'status', '-dn', m_src_dir], cwd=jdkrepo)
            if out:
                mx.run(['hg', 'rm'] + out.split(), cwd=jdkrepo)

    out = run_output(['git', 'tag', '-l', 'JDK-*'],
                     cwd=mx_compiler._suite.vc_dir)
    last_jdk_tag = sorted(out.split(), reverse=True)[0]

    pretty = args.pretty or 'format:%h %ad %>(20) %an %s'
    out = run_output([
        'git', '--no-pager', 'log', '--merges', '--abbrev-commit',
        '--pretty=' + pretty, '--first-parent', '-r', last_jdk_tag + '..HEAD'
    ] + copied_source_dirs,
                     cwd=mx_compiler._suite.vc_dir)
    changes_file = 'changes-since-{}.txt'.format(last_jdk_tag)
    with open(changes_file, 'w') as fp:
        fp.write(out)
    mx.log('Saved changes since {} to {}'.format(
        last_jdk_tag, os.path.abspath(changes_file)))
    if overwritten:
        overwritten_file = 'overwritten-diffs.txt'
        with open(overwritten_file, 'w') as fp:
            fp.write(overwritten)
        mx.warn(
            'Overwritten changes detected in OpenJDK Graal! See diffs in ' +
            os.path.abspath(overwritten_file))
Exemplo n.º 46
0
def run_shared_lib_test(args=None):
    mx.run_mx([
        "--dynamicimports", "/substratevm,/vm", "build",
        "--force-deprecation-as-warning", "--dependencies",
        "GRAAL_MANAGEMENT,POLYGLOT_NATIVE_API_HEADERS,libpolyglot.so.image"
    ],
              nonZeroIsFatal=True)
    vmdir = os.path.join(mx.suite("truffle").dir, "..", "vm")
    svm_lib_path = os.path.join(vmdir, "mxbuild",
                                "-".join([mx.get_os(),
                                          mx.get_arch()]),
                                "libpolyglot.so.image")
    fd = name = progname = None
    try:
        fd, name = tempfile.mkstemp(suffix='.c')
        os.write(
            fd, """
        #include "stdio.h"
        #include "polyglot_api.h"

        #define assert_ok(msg, f) { if (!(f)) { \\
             const poly_extended_error_info* error_info; \\
             poly_get_last_error_info(isolate_thread, &error_info); \\
             fprintf(stderr, "%s\\n", error_info->error_message); \\
             return fprintf(stderr, "%s\\n", msg); } } while (0)

        poly_isolate global_isolate;
        poly_thread isolate_thread;
        poly_engine engine;
        poly_context context;

        static poly_status create_context() {
            poly_status status;

            if (poly_attach_thread(global_isolate, &isolate_thread)) {
                return poly_generic_failure;
            }

            poly_engine_builder engine_builder;
            status = poly_create_engine_builder(isolate_thread, &engine_builder);
            if (status != poly_ok) {
                return status;
            }
            status = poly_engine_builder_build(isolate_thread, engine_builder, &engine);
            if (status != poly_ok) {
                return status;
            }
            poly_context_builder builder;
            status = poly_create_context_builder(isolate_thread, NULL, 0, &builder);
            if (status != poly_ok) {
                return status;
            }
            status = poly_context_builder_engine(isolate_thread, builder, engine);
            if (status != poly_ok) {
                return status;
            }
            status = poly_context_builder_option(isolate_thread, builder, "python.VerboseFlag", "true");
            if (status != poly_ok) {
            return status;
            }
            status = poly_context_builder_allow_io(isolate_thread, builder, true);
            if (status != poly_ok) {
            return status;
            }
            status = poly_context_builder_build(isolate_thread, builder, &context);
            if (status != poly_ok) {
                return status;
            }

            return poly_ok;
        }

        static poly_status tear_down_context() {
            poly_status status = poly_context_close(isolate_thread, context, true);
            if (status != poly_ok) {
                return status;
            }

            status = poly_engine_close(isolate_thread, engine, true);
            if (status != poly_ok) {
                return status;
            }

            if (poly_detach_thread(isolate_thread)) {
                return poly_ok;
            }

            return poly_ok;
        }

        static int test_basic_python_function() {
            assert_ok("Context creation failed.", create_context() == poly_ok);

            poly_value func;
            assert_ok("function eval failed", poly_context_eval(isolate_thread, context, "python", "test_func", "def test_func(x):\\n  return x * x\\ntest_func", &func) == poly_ok);
            int32_t arg_value = 42;
            poly_value primitive_object;
            assert_ok("create argument failed", poly_create_int32(isolate_thread, context, arg_value, &primitive_object) == poly_ok);
            poly_value arg[1] = {primitive_object};
            poly_value value;
            assert_ok("invocation was unsuccessful", poly_value_execute(isolate_thread, func, arg, 1, &value) == poly_ok);

            int32_t result_value;
            poly_value_as_int32(isolate_thread, value, &result_value);

            assert_ok("value computation was incorrect", result_value == 42 * 42);
            assert_ok("Context tear down failed.", tear_down_context() == poly_ok);
            return 0;
        }

        int32_t main(int32_t argc, char **argv) {
            poly_isolate_params isolate_params = {};
            if (poly_create_isolate(&isolate_params, &global_isolate, &isolate_thread)) {
                return 1;
            }
            return test_basic_python_function();
        }
        """)
        os.close(fd)
        progname = os.path.join(SUITE.dir, "graalpython-embedded-tool")
        mx.log("".join([
            "Running ", "'clang",
            "-I%s" % svm_lib_path,
            "-L%s" % svm_lib_path, name, "-o", progname, "-lpolyglot"
        ]))
        mx.run([
            "clang",
            "-I%s" % svm_lib_path,
            "-L%s" % svm_lib_path, name,
            "-o%s" % progname, "-lpolyglot"
        ],
               nonZeroIsFatal=True)
        mx.log("Running " + progname + " with LD_LIBRARY_PATH " + svm_lib_path)
        mx.run(["ls", "-l", progname])
        mx.run(["ls", "-l", svm_lib_path])
        run_env = {
            "LD_LIBRARY_PATH": svm_lib_path,
            "GRAAL_PYTHONHOME": os.environ["GRAAL_PYTHONHOME"]
        }
        print(run_env)
        mx.run([progname], env=run_env)
    finally:
        try:
            os.unlink(progname)
        except:
            pass
        try:
            os.close(fd)
        except:
            pass
        try:
            os.unlink(name)
        except:
            pass
Exemplo n.º 47
0
def run_shared_lib_test(args=None):
    if args is None:
        args = []
    launcher = python_so(args)
    svm_lib_path = os.path.abspath(
        os.path.join(launcher, "..", "..", "jre", "lib", "polyglot"))
    fd = name = progname = None
    try:
        fd, name = tempfile.mkstemp(suffix='.c')
        os.write(
            fd, b"""
        #include "stdio.h"
        #include "polyglot_api.h"

        #define assert_ok(msg, f) { if (!(f)) { \\
             const poly_extended_error_info* error_info; \\
             poly_get_last_error_info(isolate_thread, &error_info); \\
             fprintf(stderr, "%%s\\n", error_info->error_message); \\
             return fprintf(stderr, "%%s\\n", msg); } } while (0)

        poly_isolate global_isolate;
        poly_thread isolate_thread;
        poly_engine engine;
        poly_context context;

        static poly_status create_context() {
            poly_status status;

            if (poly_attach_thread(global_isolate, &isolate_thread)) {
                return poly_generic_failure;
            }

            poly_engine_builder engine_builder;
            status = poly_create_engine_builder(isolate_thread, &engine_builder);
            if (status != poly_ok) {
                return status;
            }
            status = poly_engine_builder_build(isolate_thread, engine_builder, &engine);
            if (status != poly_ok) {
                return status;
            }
            poly_context_builder builder;
            status = poly_create_context_builder(isolate_thread, NULL, 0, &builder);
            if (status != poly_ok) {
                return status;
            }
            status = poly_context_builder_engine(isolate_thread, builder, engine);
            if (status != poly_ok) {
                return status;
            }
            status = poly_context_builder_option(isolate_thread, builder, "python.VerboseFlag", "true");
            if (status != poly_ok) {
                return status;
            }
        #if %s
            status = poly_context_builder_option(isolate_thread, builder, "llvm.managed", "true");
            if (status != poly_ok) {
                return status;
            }
        #endif
            status = poly_context_builder_allow_io(isolate_thread, builder, true);
            if (status != poly_ok) {
                return status;
            }
            status = poly_context_builder_build(isolate_thread, builder, &context);
            if (status != poly_ok) {
                return status;
            }

            return poly_ok;
        }

        static poly_status tear_down_context() {
            poly_status status = poly_context_close(isolate_thread, context, true);
            if (status != poly_ok) {
                return status;
            }

            status = poly_engine_close(isolate_thread, engine, true);
            if (status != poly_ok) {
                return status;
            }

            if (poly_detach_thread(isolate_thread)) {
                return poly_ok;
            }

            return poly_ok;
        }

        static int test_basic_python_function() {
            assert_ok("Context creation failed.", create_context() == poly_ok);

            poly_value func;
            assert_ok("function eval failed", poly_context_eval(isolate_thread, context, "python", "test_func", "def test_func(x):\\n  return x * x\\ntest_func", &func) == poly_ok);
            int32_t arg_value = 42;
            poly_value primitive_object;
            assert_ok("create argument failed", poly_create_int32(isolate_thread, context, arg_value, &primitive_object) == poly_ok);
            poly_value arg[1] = {primitive_object};
            poly_value value;
            assert_ok("invocation was unsuccessful", poly_value_execute(isolate_thread, func, arg, 1, &value) == poly_ok);

            int32_t result_value;
            poly_value_as_int32(isolate_thread, value, &result_value);

            assert_ok("value computation was incorrect", result_value == 42 * 42);
            assert_ok("Context tear down failed.", tear_down_context() == poly_ok);
            return 0;
        }

        int32_t main(int32_t argc, char **argv) {
            poly_isolate_params isolate_params = {};
            if (poly_create_isolate(&isolate_params, &global_isolate, &isolate_thread)) {
                return 1;
            }
            return test_basic_python_function();
        }
        """ % (b"1" if "sandboxed" in args else b"0"))
        os.close(fd)
        progname = os.path.join(SUITE.dir, "graalpython-embedded-tool")
        mx.log("".join([
            "Running ", "'clang",
            "-I%s" % svm_lib_path,
            "-L%s" % svm_lib_path, name, "-o", progname, "-lpolyglot"
        ]))
        mx.run([
            "clang",
            "-I%s" % svm_lib_path,
            "-L%s" % svm_lib_path, name,
            "-o%s" % progname, "-lpolyglot"
        ],
               nonZeroIsFatal=True)
        mx.log("Running " + progname + " with LD_LIBRARY_PATH " + svm_lib_path)
        mx.run(["ls", "-l", progname])
        mx.run(["ls", "-l", svm_lib_path])
        run_env = {
            "LD_LIBRARY_PATH": svm_lib_path,
            "GRAAL_PYTHONHOME": os.environ["GRAAL_PYTHONHOME"]
        }
        mx.log(repr(run_env))
        mx.run([progname], env=run_env)
    finally:
        try:
            os.unlink(progname)
        except:
            pass
        try:
            os.close(fd)
        except:
            pass
        try:
            os.unlink(name)
        except:
            pass
Exemplo n.º 48
0
def python3_unittests(args):
    mx.run([
        "python3",
        "graalpython/com.oracle.graal.python.test/src/python_unittests.py",
        "-v"
    ] + args)
Exemplo n.º 49
0
def python3_unittests(args):
    """run the cPython stdlib unittests"""
    mx.run(["python3", "graalpython/com.oracle.graal.python.test/src/python_unittests.py", "-v"] + args)
Exemplo n.º 50
0
def jdkartifactstats(args):
    """show stats about JDK deployed Graal artifacts"""
    artifacts = {}
    jdkDir = get_jvmci_jdk().home

    def _getDeployedJars():
        if JVMCI_VERSION < 9:
            for root, _, filenames in os.walk(join(jdkDir, 'jre', 'lib')):
                for f in filenames:
                    if f.endswith('.jar') and not f.endswith('.stripped.jar'):
                        yield join(root, f)
        else:
            for jdkDist in jdkDeployedDists:
                dist = jdkDist.dist()
                if isinstance(jdkDist, JvmciJDKDeployedDist):
                    yield dist.path

    for jar in _getDeployedJars():
        f = basename(jar)
        if 'truffle' in f:
            if 'enterprise' in f:
                artifacts.setdefault('GraalEnterpriseTruffle', []).append(jar)
            else:
                artifacts.setdefault('GraalTruffle', []).append(jar)
        elif 'enterprise' in f:
            artifacts.setdefault('GraalEnterprise', []).append(jar)
        elif 'jvmci' in f:
            artifacts.setdefault('JVMCI', []).append(jar)
        elif 'graal' in f:
            artifacts.setdefault('Graal', []).append(jar)
        else:
            mx.logv('ignored: ' + jar)

    print '{:>10}  {:>10}  {:>10}  {}'.format('All', 'NoVars', 'None', 'Jar')
    for category in sorted(artifacts.viewkeys()):
        jars = artifacts[category]
        if jars:
            totals = (0, 0, 0)
            print
            for j in jars:
                gSize = os.path.getsize(j)
                stripped = j[:-len('.jar')] + '.stripped.jar'
                mx.run([
                    mx.get_jdk().pack200, '--repack', '--quiet',
                    '-J-Djava.util.logging.config.file=',
                    '-DLocalVariableTypeTable=strip',
                    '-DLocalVariableTable=strip', stripped, j
                ])
                gLinesSourceSize = os.path.getsize(stripped)
                mx.run([
                    mx.get_jdk().pack200, '--repack', '--quiet',
                    '-J-Djava.util.logging.config.file=', '-G', stripped, j
                ])
                gNoneSize = os.path.getsize(stripped)
                os.remove(stripped)
                print '{:10,}  {:10,}  {:10,}  {}:{}'.format(
                    gSize, gLinesSourceSize, gNoneSize, category, basename(j))
                t1, t2, t3 = totals
                totals = (t1 + gSize, t2 + gLinesSourceSize, t3 + gNoneSize)
            t1, t2, t3 = totals
            print '{:10,}  {:10,}  {:10,}  {}'.format(t1, t2, t3, category)

    jvmLib = join(jdkDir, relativeVmLibDirInJdk(), get_vm(),
                  mx.add_lib_suffix(mx.add_lib_prefix('jvm')))
    print
    if exists(jvmLib):
        print '{:10,}  {}'.format(os.path.getsize(jvmLib), jvmLib)
    else:
        print '{:>10}  {}'.format('<missing>', jvmLib)
Exemplo n.º 51
0
def python_build_watch(args):
    """
    Watch the suite and on any changes to .class, .jar, .h, or .c files rebuild.
    By default, rebuilds only the archives and non-Java projects.
    """
    parser = ArgumentParser(prog='mx python-build-watch')
    parser.add_argument('--full',
                        action='store_true',
                        help='Run a full mx build',
                        required=False)
    parser.add_argument('--graalvm',
                        action='store_true',
                        help='Build a graalvm',
                        required=False)
    parser.add_argument(
        '--no-java',
        action='store_true',
        help='Build only archives and native projects [default]',
        required=False)
    args = parser.parse_args(args)
    if sum([args.full, args.graalvm, args.no_java]) > 1:
        mx.abort("Only one of --full, --graalvm, --no-java can be specified")
    if args.full:
        # suffixes = [".c", ".h", ".class", ".jar", ".java"]
        excludes = [".*\\.py$"]
    elif args.graalvm:
        # suffixes = [".c", ".h", ".class", ".jar", ".java", ".py"]
        excludes = ["mx_.*\\.py$"]
    else:
        # suffixes = [".c", ".h", ".class", ".jar"]
        excludes = [".*\\.py$", ".*\\.java$"]

    cmd = [
        "inotifywait", "-q", "-e", "close_write,moved_to", "-r", "--format=%f"
    ]
    for e in excludes:
        cmd += ["--exclude", e]
    cmd += ["@%s" % os.path.join(SUITE.dir, ".git"), SUITE.dir]
    cmd_qq = cmd[:]
    cmd_qq[1] = "-qq"
    was_quiet = mx.get_opts().quiet

    while True:
        out = mx.OutputCapture()
        if mx.run(cmd, out=out, nonZeroIsFatal=False) != 0:
            continue
        changed_file = out.data.strip()
        mx.logv(changed_file)
        if any(
                changed_file.endswith(ext)
                for ext in [".c", ".h", ".class", ".jar"]):
            if not mx.get_opts().quiet:
                sys.stdout.write("Build needed ")
                sys.stdout.flush()
            while True:
                # re-run this until it times out, which we'll interpret as quiet
                # time
                if not mx.get_opts().quiet:
                    sys.stdout.write(".")
                    sys.stdout.flush()
                mx.get_opts().quiet = True
                try:
                    retcode = mx.run(cmd_qq, timeout=3, nonZeroIsFatal=False)
                finally:
                    mx.get_opts().quiet = was_quiet
                if retcode == mx.ERROR_TIMEOUT:
                    if not mx.get_opts().quiet:
                        sys.stdout.write("\n")
                    break
            mx.log("Building.")
            if args.full:
                mx.command_function("build")()
            elif args.graalvm:
                mx.log(python_gvm())
            else:
                nativebuild([])
            mx.log("Build done.")
Exemplo n.º 52
0
def jlink_new_jdk(jdk,
                  dst_jdk_dir,
                  module_dists,
                  root_module_names=None,
                  missing_export_target_action='create',
                  with_source=lambda x: True,
                  vendor_info=None,
                  dedup_legal_notices=True):
    """
    Uses jlink from `jdk` to create a new JDK image in `dst_jdk_dir` with `module_dists` and
    their dependencies added to the JDK image, replacing any existing modules of the same name.

    :param JDKConfig jdk: source JDK
    :param str dst_jdk_dir: path to use for the jlink --output option
    :param list module_dists: list of distributions defining modules
    :param list root_module_names: list of strings naming the module root set for the new JDK image.
                     The named modules must either be in `module_dists` or in `jdk`. If None, then
                     the root set will be all the modules in ``module_dists` and `jdk`.
    :param str missing_export_target_action: the action to perform for a qualifed export target that
                     is not present in `module_dists` and does not have a hash stored in java.base.
                     The choices are:
                       "create" - an empty module is created
                        "error" - raise an error
                           None - do nothing
    :param lambda with_source: returns True if the sources of a module distribution must be included in the new JDK
    :param dict vendor_info: values for the jlink vendor options added by JDK-8232080
    """
    assert callable(with_source)

    if jdk.javaCompliance < '9':
        mx.abort('Cannot derive a new JDK from ' + jdk.home +
                 ' with jlink since it is not JDK 9 or later')

    exploded_java_base_module = join(jdk.home, 'modules', 'java.base')
    if exists(exploded_java_base_module):
        mx.abort(
            'Cannot derive a new JDK from ' + jdk.home +
            ' since it appears to be a developer build with exploded modules')

    jimage = join(jdk.home, 'lib', 'modules')
    jmods_dir = join(jdk.home, 'jmods')
    if not isfile(jimage):
        mx.abort('Cannot derive a new JDK from ' + jdk.home + ' since ' +
                 jimage + ' is missing or is not an ordinary file')
    if not isdir(jmods_dir):
        mx.abort('Cannot derive a new JDK from ' + jdk.home + ' since ' +
                 jmods_dir + ' is missing or is not a directory')

    jdk_modules = {jmd.name: jmd for jmd in jdk.get_modules()}
    modules = [as_java_module(dist, jdk) for dist in module_dists]
    all_module_names = frozenset(
        list(jdk_modules.keys()) + [m.name for m in modules])

    # Read hashes stored in java.base (the only module in the JDK where hashes are stored)
    out = mx.LinesOutputCapture()
    mx.run([
        jdk.exe_path('jmod'), 'describe',
        jdk_modules['java.base'].get_jmod_path()
    ],
           out=out)
    lines = out.lines
    hashes = {}
    for line in lines:
        if line.startswith('hashes'):
            parts = line.split()
            assert len(
                parts
            ) == 4, 'expected hashes line to have 4 fields, got {} fields: {}'.format(
                len(parts), line)
            _, module_name, algorithm, hash_value = parts
            hashes[module_name] = (algorithm, hash_value)

    build_dir = mx.ensure_dir_exists(join(dst_jdk_dir + ".build"))
    try:
        # Handle targets of qualified exports that are not present in `modules`
        target_requires = {}
        for jmd in modules:
            for targets in jmd.exports.values():
                for target in targets:
                    if target not in all_module_names and target not in hashes:
                        target_requires.setdefault(target, set()).add(jmd.name)
        if target_requires and missing_export_target_action is not None:
            if missing_export_target_action == 'error':
                mx.abort(
                    'Target(s) of qualified exports cannot be resolved: ' +
                    '.'.join(target_requires.keys()))
            assert missing_export_target_action == 'create', 'invalid value for missing_export_target_action: ' + str(
                missing_export_target_action)

            extra_modules = []
            for name, requires in target_requires.items():
                module_jar = join(build_dir, name + '.jar')
                jmd = JavaModuleDescriptor(
                    name, {},
                    requires={module: []
                              for module in requires},
                    uses=set(),
                    provides={},
                    jarpath=module_jar)
                extra_modules.append(jmd)
                module_build_dir = mx.ensure_dir_exists(join(build_dir, name))
                module_info_java = join(module_build_dir, 'module-info.java')
                module_info_class = join(module_build_dir, 'module-info.class')
                with open(module_info_java, 'w') as fp:
                    print(jmd.as_module_info(), file=fp)
                mx.run([
                    jdk.javac, '-d', module_build_dir,
                    '--limit-modules=java.base,' +
                    ','.join(jmd.requires.keys()),
                    '--module-path=' + os.pathsep.join(
                        (m.jarpath for m in modules)), module_info_java
                ])
                with ZipFile(module_jar, 'w') as zf:
                    zf.write(module_info_class, basename(module_info_class))
                if exists(jmd.get_jmod_path()):
                    os.remove(jmd.get_jmod_path())
                mx.run([
                    jdk.javac.replace('javac', 'jmod'), 'create',
                    '--class-path=' + module_build_dir,
                    jmd.get_jmod_path()
                ])

            modules.extend(extra_modules)
            all_module_names = frozenset(
                list(jdk_modules.keys()) + [m.name for m in modules])

        # Extract src.zip from source JDK
        jdk_src_zip = join(jdk.home, 'lib', 'src.zip')
        dst_src_zip_contents = {}
        if isfile(jdk_src_zip):
            mx.logv('[Extracting ' + jdk_src_zip + ']')
            with ZipFile(jdk_src_zip, 'r') as zf:
                for name in zf.namelist():
                    if not name.endswith('/'):
                        dst_src_zip_contents[name] = zf.read(name)
        else:
            mx.warn("'{}' does not exist or is not a file".format(jdk_src_zip))

        # Edit lib/security/default.policy in java.base
        patched_java_base = join(build_dir, 'java.base.jmod')
        with open(join(jmods_dir, 'java.base.jmod'),
                  'rb') as src_f, open(patched_java_base, 'wb') as dst_f:
            jmod_header = src_f.read(4)
            if len(jmod_header) != 4 or jmod_header != b'JM\x01\x00':
                raise mx.abort("Unexpected jmod header: " +
                               b2a_hex(jmod_header).decode('ascii'))
            dst_f.write(jmod_header)
            policy_result = 'not found'
            with ZipFile(src_f, 'r') as src_zip, ZipFile(
                    dst_f, 'w', src_zip.compression) as dst_zip:
                for i in src_zip.infolist():
                    if i.filename[-1] == '/':
                        continue
                    src_member = src_zip.read(i)
                    if i.filename == 'lib/security/default.policy':
                        if 'grant codeBase "jrt:/com.oracle.graal.graal_enterprise"'.encode(
                                'utf-8') in src_member:
                            policy_result = 'unmodified'
                        else:
                            policy_result = 'modified'
                            src_member += """
grant codeBase "jrt:/com.oracle.graal.graal_enterprise" {
    permission java.security.AllPermission;
};
""".encode('utf-8')
                    dst_zip.writestr(i, src_member)
            if policy_result == 'not found':
                raise mx.abort(
                    "Couldn't find `lib/security/default.policy` in " +
                    join(jmods_dir, 'java.base.jmod'))

        for jmd in modules:
            # Remove existing sources for all the modules that we include
            dst_src_zip_contents = {
                key: dst_src_zip_contents[key]
                for key in dst_src_zip_contents if not key.startswith(jmd.name)
            }

            if with_source(jmd.dist):
                # Add the sources that we can share.
                # Extract module sources
                jmd_src_zip = jmd.jarpath[0:-len('.jar')] + '.src.zip'
                if isfile(jmd_src_zip):
                    mx.logv('[Extracting ' + jmd_src_zip + ']')
                    with ZipFile(jmd_src_zip, 'r') as zf:
                        for name in zf.namelist():
                            if not name.endswith('/'):
                                dst_src_zip_contents[jmd.name + '/' +
                                                     name] = zf.read(name)

                # Add module-info.java to sources
                dst_src_zip_contents[jmd.name +
                                     '/module-info.java'] = jmd.as_module_info(
                                         extras_as_comments=False)

        # Now build the new JDK image with jlink
        jlink = [jdk.javac.replace('javac', 'jlink')]

        if jdk_enables_jvmci_by_default(jdk):
            # On JDK 9+, +EnableJVMCI forces jdk.internal.vm.ci to be in the root set
            jlink += ['-J-XX:-EnableJVMCI', '-J-XX:-UseJVMCICompiler']
        if root_module_names is not None:
            missing = frozenset(root_module_names) - all_module_names
            if missing:
                mx.abort(
                    'Invalid module(s): {}.\nAvailable modules: {}'.format(
                        ','.join(missing), ','.join(sorted(all_module_names))))
            jlink.append('--add-modules=' + ','.join(root_module_names))
        else:
            jlink.append('--add-modules=' + ','.join(sorted(all_module_names)))

        module_path = patched_java_base + os.pathsep + jmods_dir
        if modules:
            module_path = os.pathsep.join(
                (m.get_jmod_path(respect_stripping=True)
                 for m in modules)) + os.pathsep + module_path
        jlink.append('--module-path=' + module_path)
        jlink.append('--output=' + dst_jdk_dir)

        # These options are derived from how OpenJDK runs jlink to produce the final runtime image.
        jlink.extend([
            '-J-XX:+UseSerialGC', '-J-Xms32M', '-J-Xmx512M',
            '-J-XX:TieredStopAtLevel=1'
        ])
        jlink.append('-J-Dlink.debug=true')
        if dedup_legal_notices:
            jlink.append('--dedup-legal-notices=error-if-not-same-content')
        jlink.append('--keep-packaged-modules=' + join(dst_jdk_dir, 'jmods'))

        if jdk_has_new_jlink_options(jdk):
            if jdk_omits_warning_for_jlink_set_ThreadPriorityPolicy(jdk):
                thread_priority_policy_option = ' -XX:ThreadPriorityPolicy=1'
            else:
                mx.logv('[Creating JDK without -XX:ThreadPriorityPolicy=1]')
                thread_priority_policy_option = ''

            if jdk_supports_enablejvmciproduct(jdk):
                if any(
                    (m.name == 'jdk.internal.vm.compiler' for m in modules)):
                    jlink.append(
                        '--add-options=-XX:+UnlockExperimentalVMOptions -XX:+EnableJVMCIProduct -XX:-UnlockExperimentalVMOptions'
                        + thread_priority_policy_option)
                else:
                    # Don't default to using JVMCI as JIT unless Graal is being updated in the image.
                    # This avoids unexpected issues with using the out-of-date Graal compiler in
                    # the JDK itself.
                    jlink.append(
                        '--add-options=-XX:+UnlockExperimentalVMOptions -XX:+EnableJVMCIProduct -XX:-UseJVMCICompiler -XX:-UnlockExperimentalVMOptions'
                        + thread_priority_policy_option)
            else:
                mx.logv('[Creating JDK without -XX:+EnableJVMCIProduct]')
                if thread_priority_policy_option:
                    jlink.append('--add-options=' +
                                 thread_priority_policy_option.strip())
            if vendor_info is not None:
                for name, value in vendor_info.items():
                    jlink.append('--' + name + '=' + value)

        # TODO: investigate the options below used by OpenJDK to see if they should be used:
        # --release-info: this allow extra properties to be written to the <jdk>/release file
        # --order-resources: specifies order of resources in generated lib/modules file.
        #       This is apparently not so important if a CDS archive is available.
        # --generate-jli-classes: pre-generates a set of java.lang.invoke classes.
        #       See https://github.com/openjdk/jdk/blob/master/make/GenerateLinkOptData.gmk
        mx.logv('[Creating JDK image in {}]'.format(dst_jdk_dir))
        mx.run(jlink)

        dst_src_zip = join(dst_jdk_dir, 'lib', 'src.zip')
        mx.logv('[Creating ' + dst_src_zip + ']')
        with ZipFile(dst_src_zip,
                     'w',
                     compression=ZIP_DEFLATED,
                     allowZip64=True) as zf:
            for name, contents in sorted(dst_src_zip_contents.items()):
                zf.writestr(name, contents)

        mx.logv('[Copying static libraries]')
        lib_prefix = mx.add_lib_prefix('')
        lib_suffix = '.lib' if mx.is_windows() else '.a'
        lib_directory = join(jdk.home, 'lib')
        dst_lib_directory = join(dst_jdk_dir, 'lib')
        for f in os.listdir(lib_directory):
            if f.startswith(lib_prefix) and f.endswith(lib_suffix):
                lib_path = join(lib_directory, f)
                if isfile(lib_path):
                    shutil.copy2(lib_path, dst_lib_directory)

        # Build the list of modules whose classes might have annotations
        # to be processed by native-image (GR-15192).
        with open(join(dst_jdk_dir, 'lib', 'native-image-modules.list'),
                  'w') as fp:
            print(
                '# Modules whose classes might have annotations processed by native-image',
                file=fp)
            for m in modules:
                print(m.name, file=fp)

    finally:
        if not mx.get_opts().verbose:
            # Preserve build directory so that javac command can be re-executed
            # by cutting and pasting verbose output.
            shutil.rmtree(build_dir)

    # Create CDS archive (https://openjdk.java.net/jeps/341).
    out = mx.OutputCapture()
    mx.logv('[Creating CDS shared archive]')
    if mx.run([
            mx.exe_suffix(join(dst_jdk_dir, 'bin', 'java')), '-Xshare:dump',
            '-Xmx128M', '-Xms128M'
    ],
              out=out,
              err=out,
              nonZeroIsFatal=False) != 0:
        mx.log(out.data)
        mx.abort('Error generating CDS shared archive')
Exemplo n.º 53
0
 def run(self, inputFile, outputFile, flags):
     return mx.run([mx_sulong.findLLVMProgram('opt'), '-o', outputFile] + self.passes + [inputFile])
Exemplo n.º 54
0
def _mxrun(args, cwd=_suite.dir, verbose=False, out=None, env=None):
    if verbose:
        mx.log('Running \'{}\''.format(' '.join(args)))
    status = mx.run(args, nonZeroIsFatal=False, cwd=cwd, out=out, env=env)
    if status:
        mx.abort(status)
Exemplo n.º 55
0
    def run_java(self,
                 args,
                 out=None,
                 err=None,
                 cwd=None,
                 nonZeroIsFatal=False):
        if '-version' in args:
            return super(NativeImageVM,
                         self).run_java(args,
                                        out=out,
                                        err=err,
                                        cwd=cwd,
                                        nonZeroIsFatal=nonZeroIsFatal)
        else:
            config = NativeImageVM.BenchmarkConfig()
            original_java_run_args = config.parse(args)
            executable, classpath_arguments, system_properties, image_run_args = NativeImageVM.extract_benchmark_arguments(
                original_java_run_args)

            # Agent configuration and/or HotSpot profiling
            needs_config = (config.config_dir is None) and config.needs_config
            if needs_config or self.hotspot_pgo:
                config_vm_args = []
                profiling_vm_args = []
                config_run_args = []
                profiling_run_args = []

                if needs_config:
                    config.config_dir = mx.mkdtemp(suffix='config',
                                                   prefix='native-image')
                    config_vm_args += [
                        '-agentlib:native-image-agent=config-output-dir=' +
                        str(config.config_dir)
                    ]
                    config_run_args += config.extra_agent_run_args

                if self.hotspot_pgo:
                    profiling_vm_args += [
                        '-Dgraal.ProfilesCollectExperimental=true'
                    ]
                    config_run_args = []
                    profiling_run_args = config.extra_profile_run_args

                agent_run_args = config_vm_args + profiling_vm_args + original_java_run_args + config.extra_run_args + config_run_args + profiling_run_args
                mx.log(
                    'Running with HotSpot to get the configuration files and profiles. This could take a while:'
                )
                super(NativeImageVM,
                      self).run_java(agent_run_args,
                                     out=None,
                                     err=None,
                                     cwd=cwd,
                                     nonZeroIsFatal=nonZeroIsFatal)

            base_image_build_args = [
                os.path.join(mx_vm.graalvm_home(fatalIfMissing=True), 'bin',
                             'native-image')
            ]
            base_image_build_args += ['--no-fallback']
            base_image_build_args += ['-J-ea', '-J-esa']
            base_image_build_args += system_properties
            base_image_build_args += classpath_arguments
            executable_name = (executable[1] if executable[0] is '-jar' else
                               executable[0]).lower()
            base_image_build_args += executable
            base_image_build_args += ['-H:Name=' + executable_name]
            if needs_config:
                base_image_build_args += [
                    '-H:ConfigurationFileDirectories=' + config.config_dir
                ]
            base_image_build_args += config.extra_image_build_arguments

            # PGO instrumentation
            i = 0
            while i < self.pgo_instrumented_iterations:
                instrument_args = [
                    '--pgo-instrument'
                ] + [] if i is 0 and not self.hotspot_pgo else ['--pgo']
                instrument_image_build_args = base_image_build_args + instrument_args
                mx.log('Building the instrumentation image with: ')
                mx.log(' ' + ' '.join([
                    pipes.quote(str(arg))
                    for arg in instrument_image_build_args
                ]))
                mx.run(instrument_image_build_args,
                       out=None,
                       err=None,
                       cwd=cwd,
                       nonZeroIsFatal=nonZeroIsFatal)

                image_run_cmd = [
                    os.path.abspath(executable_name)
                ] + image_run_args + config.extra_run_args + config.extra_profile_run_args
                mx.log('Running the instrumented image with: ')
                mx.log(
                    ' ' +
                    ' '.join([pipes.quote(str(arg)) for arg in image_run_cmd]))
                mx.run(image_run_cmd,
                       out=out,
                       err=err,
                       cwd=cwd,
                       nonZeroIsFatal=nonZeroIsFatal)

                i += 1

            # Build the final image
            pgo_args = ['--pgo'] if self.pgo_instrumented_iterations > 0 or self.hotspot_pgo else []
            final_image_args = base_image_build_args + pgo_args
            mx.log('Building the final image with: ')
            mx.log(
                ' ' +
                ' '.join([pipes.quote(str(arg)) for arg in final_image_args]))
            mx.run(final_image_args,
                   out=None,
                   err=None,
                   cwd=cwd,
                   nonZeroIsFatal=nonZeroIsFatal)

            # Execute the benchmark
            image_run_cmd = [os.path.abspath(executable_name)
                             ] + image_run_args + config.extra_run_args
            mx.log('Running the produced native executable with: ')
            mx.log(' ' +
                   ' '.join([pipes.quote(str(arg)) for arg in image_run_cmd]))
            mx.run(image_run_cmd,
                   out=out,
                   err=err,
                   cwd=cwd,
                   nonZeroIsFatal=nonZeroIsFatal)
Exemplo n.º 56
0
def _fastr_gate_runner(args, tasks):
    with mx_gate.Task('Setup no specials',
                      tasks,
                      tags=[FastRGateTags.no_specials]) as t:
        if t:
            os.environ['FASTR_OPTION_UseSpecials'] = 'false'

    with mx_gate.Task('Setup no dsl cache',
                      tasks,
                      tags=[FastRGateTags.no_dsl_cache]) as t:
        if t:
            os.environ['FASTR_OPTION_DSLCacheSizeFactor'] = '0'

    with mx_gate.Task('SetupLLVM', tasks, tags=[FastRGateTags.llvm]) as t:
        if t:
            os.environ['FASTR_RFFI'] = 'llvm'

    with mx_gate.Task('GCTorture1', tasks,
                      tags=[FastRGateTags.gc_torture1]) as t:
        if t:
            os.environ['FASTR_GCTORTURE'] = '1'

    with mx_gate.Task('GCTorture3', tasks,
                      tags=[FastRGateTags.gc_torture3]) as t:
        if t:
            os.environ['FASTR_GCTORTURE'] = '3'

    with mx_gate.Task('VerySlowAsserts',
                      tasks,
                      tags=[FastRGateTags.very_slow_asserts]) as t:
        if t:
            os.environ['FASTR_TEST_VERY_SLOW_ASSERTS'] = 'true'
    '''
    The specific additional gates tasks provided by FastR.
    '''
    with mx_gate.Task('ExtSoftVersions', tasks,
                      tags=[mx_gate.Tags.always]) as t:
        if t:
            new_env = os.environ.copy()
            new_env['R_DEFAULT_PACKAGES'] = 'base'
            run_r(['-q', '-e', 'extSoftVersion()'], 'R', env=new_env)

    with mx_gate.Task('LibsInfo', tasks, tags=[mx_gate.Tags.always]) as t:
        if t:
            mx.log("Libraries captured in FASTR_HOME/lib:")
            lib_dir = os.path.join(_fastr_suite.dir, 'lib')
            for f in os.listdir(lib_dir):
                full_path = os.path.join(lib_dir, f)
                mx.run(['file', full_path], nonZeroIsFatal=False)
                mx.run(['objdump', '-s', '--section', '.comment', full_path],
                       nonZeroIsFatal=False)
                mx.log('------')

    # ---------------------------------
    # Style checks:

    # FastR has custom copyright check
    with mx_gate.Task('Copyright check', tasks,
                      tags=[mx_gate.Tags.style]) as t:
        if t:
            if mx.checkcopyrights(['--primary']) != 0:
                t.abort('copyright errors')

    # check that the expected test output file is up to date
    with mx_gate.Task('UnitTests: ExpectedTestOutput file check',
                      tasks,
                      tags=[mx_gate.Tags.style]) as t:
        if t:
            mx_unittest.unittest([
                '-Dfastr.test.gen.expected=' +
                _test_srcdir(), '-Dfastr.test.check.expected=true'
            ] + _gate_unit_tests())

    # ----------------------------------
    # Basic tests:

    with mx_gate.Task(
            'UnitTests',
            tasks,
            tags=[FastRGateTags.basic_tests, FastRGateTags.unit_tests]) as t:
        if t:
            mx_unittest.unittest(_gate_noapps_unit_tests())

    with mx_gate.Task('Rembedded', tasks,
                      tags=[FastRGateTags.basic_tests]) as t:
        if t:
            if rembedtest([]) != 0:
                t.abort("Rembedded tests failed")

    # ----------------------------------
    # Package tests:

    with mx_gate.Task('Recommended load test',
                      tasks,
                      tags=[FastRGateTags.recommended_load]) as t:
        if t:
            # Note: this is a convenience mx gate job for testing the loading of recommended packages
            # We also test the loading of recommended pkgs in the "graalvm-tests"
            if not os.path.exists(
                    os.path.join(_fastr_suite.dir, 'library', 'spatial')):
                mx.abort(
                    'Recommended packages seem to be not installed in FastR. Did you forget to build with FASTR_RELEASE=true?'
                )
            pkgs = [
                'codetools', 'MASS', 'boot', 'class', 'cluster', 'lattice',
                'nnet', 'spatial', 'Matrix', 'KernSmooth', 'foreign', 'nlme',
                'rpart', 'survival'
            ]
            # Creates code that looks like: require(codetools) && require(MASS) && ...
            require_stmts = ' && '.join(
                ['require(' + pkg + ')' for pkg in pkgs])
            test_load = 'if (!(' + require_stmts + ')) q(status=1) else q(status=42)'
            if run_r(['--vanilla', '-e', test_load], 'R',
                     nonZeroIsFatal=False) != 42:
                mx.abort("Loading of recommended packages failed")

    with mx_gate.Task('Internal pkg test',
                      tasks,
                      tags=[FastRGateTags.internal_pkgs_test]) as t:
        if t:
            internal_pkg_tests()

    # CRAN packages are listed in files com.oracle.truffle.r.test.packages/gated0, gated1, ...
    # We loop over all such files and crete gate task for each of them
    # See also documentation in FastRGateTags.cran_pkgs_tests
    for i in range(1, 1000):
        list_file = os.path.join(
            _fastr_suite.dir,
            'com.oracle.truffle.r.test.packages/gated' + str(i))
        if not os.path.exists(list_file):
            break
        with mx_gate.Task('CRAN pkg test: ' + str(i),
                          tasks,
                          tags=[FastRGateTags.cran_pkgs_test + str(i)]) as t:
            if t:
                check_last = False if mx_gate.Task.tags is None else FastRGateTags.cran_pkgs_test_check_last in mx_gate.Task.tags  # pylint: disable=unsupported-membership-test
                if check_last:
                    next_file = os.path.join(
                        _fastr_suite.dir,
                        'com.oracle.truffle.r.test.packages/gated' +
                        str(i + 1))
                    if os.path.exists(next_file):
                        mx.abort(
                            "File %s exists, but the gate thinks that %s is the last file. Did you forget to update the gate configuration?"
                            % (next_file, list_file))
                cran_pkg_tests(list_file)
Exemplo n.º 57
0
def graalpython_gate_runner(args, tasks):
    with Task('GraalPython JUnit', tasks,
              tags=[GraalPythonTags.junit]) as task:
        if task:
            punittest(['--verbose'])

    with Task('GraalPython Python tests',
              tasks,
              tags=[GraalPythonTags.unittest]) as task:
        if task:
            gate_unittests()

    with Task('GraalPython C extension tests',
              tasks,
              tags=[GraalPythonTags.cpyext]) as task:
        if task:
            # we deliberately added this to test the combination of Sulong and 'mx_unittest'
            unittest([
                '--regex',
                re.escape(
                    'com.oracle.graal.python.test.module.MemoryviewTest'),
                "-Dgraal.TraceTruffleCompilation=true"
            ])

            gate_unittests(subdir="cpyext/")

    with Task('GraalPython C extension managed tests',
              tasks,
              tags=[GraalPythonTags.cpyext_managed]) as task:
        if task:
            mx.run_mx([
                "--dynamicimports", "sulong-managed", "python-gate-unittests",
                "--llvm.configuration=managed", "--subdir=cpyext", "--"
            ])

    with Task('GraalPython C extension sandboxed tests',
              tasks,
              tags=[GraalPythonTags.cpyext_sandboxed]) as task:
        if task:
            mx.run_mx([
                "--dynamicimports", "sulong-managed", "python-gate-unittests",
                "--llvm.configuration=sandboxed", "--subdir=cpyext", "--"
            ])

    with Task('GraalPython Python tests on SVM',
              tasks,
              tags=[GraalPythonTags.svmunit]) as task:
        if task:
            svm_image_name = python_svm(["-h"])
            run_python_unittests(svm_image_name)

    with Task('GraalPython license header update',
              tasks,
              tags=[GraalPythonTags.license]) as task:
        if task:
            python_checkcopyrights([])

    with Task('GraalPython GraalVM shared-library build',
              tasks,
              tags=[GraalPythonTags.downstream,
                    GraalPythonTags.graalvm]) as task:
        if task:
            run_shared_lib_test()

    with Task('GraalPython GraalVM build',
              tasks,
              tags=[GraalPythonTags.downstream,
                    GraalPythonTags.graalvm]) as task:
        if task:
            svm_image = python_svm(["--version"])
            benchmark = os.path.join(PATH_MESO, "image-magix.py")
            out = mx.OutputCapture()
            mx.run([svm_image, benchmark],
                   nonZeroIsFatal=True,
                   out=mx.TeeOutputCapture(out))
            success = "\n".join([
                "[0, 0, 0, 0, 0, 0, 10, 10, 10, 0, 0, 10, 3, 10, 0, 0, 10, 10, 10, 0, 0, 0, 0, 0, 0]",
            ])
            if success not in out.data:
                mx.abort('Output from generated SVM image "' + svm_image +
                         '" did not match success pattern:\n' + success)
Exemplo n.º 58
0
def build_binary_pkgs(args_in, **kwargs):
    '''
    Builds binary packages of components that we cache for build speed-up.
    See the CI scripts for details.
    '''
    parser = ArgumentParser()
    parser.add_argument(
        '--f2c-version',
        type=int,
        dest='f2c_version',
        required=True,
        help='Current version of f2c, the tarball will use this + 1')
    parser.add_argument(
        '--recommended-pkgs-version',
        default=0,
        type=int,
        dest='recommended_version',
        help=
        'Current version of recommended packages binary, the tarball will use this + 1'
    )
    parser.add_argument('--recommended-pkgs-list',
                        dest='recommended_list',
                        required=True,
                        help='Comma separated list of recommended packages')
    args = parser.parse_args(args_in)

    os_name = platform.system().lower()
    dest_dir = os.path.join(_fastr_suite.dir, 'binary-packages')
    shutil.rmtree(dest_dir, ignore_errors=True)
    mx.ensure_dir_exists(dest_dir)

    # F2C
    # creates binary-packages/f2c-binary-{version}-{osname}-amd64/f2c with contents of FASTR_HOME/f2c
    f2c_name = 'f2c-binary-' + str(args.f2c_version +
                                   1) + '-' + os_name + '-amd64'
    f2c_path = os.path.join(dest_dir, f2c_name)
    shutil.copytree(os.path.join(_fastr_suite.dir, 'f2c'),
                    os.path.join(f2c_path, 'f2c'))
    # creates the tarball
    result_tarball = os.path.join(dest_dir, f2c_name + '.tar.gz')
    with tarfile.open(result_tarball, "w:gz") as tar:
        tar.add(f2c_path, arcname=os.path.basename(f2c_path))
    mx.log("Binary package created at: " + result_tarball)

    # Recommended packages
    # creates binary-packages/fastr-recommended-pkgs-{version}-{osname}-amd64/fastr-recommended-pkgs
    pkgs_name = 'fastr-recommended-pkgs-' + str(args.recommended_version +
                                                1) + '-' + os_name + '-amd64'
    pkgs_path = os.path.join(dest_dir, pkgs_name)
    pkgs_pkgs_path = os.path.join(pkgs_path, 'pkgs')
    mx.ensure_dir_exists(pkgs_pkgs_path)
    for pkg_name in args.recommended_list.split(','):
        shutil.copytree(os.path.join(_fastr_suite.dir, 'library', pkg_name),
                        os.path.join(pkgs_pkgs_path, pkg_name))
    # add file with API digest
    try:
        with open(os.path.join(pkgs_path, 'api-checksum.txt'), 'w') as f:
            sys.stdout = f
            pkgcache(['--print-api-checksum', '--vm', 'fastr'])
    finally:
        sys.stdout = sys.__stdout__
    # creates the tarball
    result_tarball = os.path.join(dest_dir, pkgs_name + '.tar.gz')
    with tarfile.open(result_tarball, "w:gz") as tar:
        tar.add(pkgs_path, arcname=os.path.basename(pkgs_path))
    mx.log("Binary package created at: " + result_tarball)

    mx.log("Contents of the " + dest_dir + "directory: ")
    mx.run(['ls', '-R', dest_dir])
    return 0
Exemplo n.º 59
0
 def execute_command(self, final_command=False):
     write_output = final_command or self.is_gate
     self.exit_code = mx.run(self.command, out=self.stdout(write_output), err=self.stderr(write_output), cwd=self.cwd, nonZeroIsFatal=False)
Exemplo n.º 60
0
def llvm_tool(args=None, out=None, **kwargs):
    if len(args) < 1:
        mx.abort("usage: mx llvm-tool <llvm-tool> [args...]")
    llvm_program = findBundledLLVMProgram(args[0])
    mx.run([llvm_program] + args[1:], out=out, **kwargs)