Beispiel #1
0
def igv(args):
    """run the Ideal Graph Visualizer"""
    logFile = '.ideal_graph_visualizer.log'
    with open(join(_suite.dir, logFile), 'w') as fp:
        mx.logv('[Ideal Graph Visualizer log is in ' + fp.name + ']')
        nbplatform = join(_suite.dir, 'src', 'share', 'tools', 'IdealGraphVisualizer', 'nbplatform')

        # Remove NetBeans platform if it is earlier than the current supported version
        if exists(nbplatform):
            updateTrackingFile = join(nbplatform, 'platform', 'update_tracking', 'org-netbeans-core.xml')
            if not exists(updateTrackingFile):
                mx.log('Could not find \'' + updateTrackingFile + '\', removing NetBeans platform')
                shutil.rmtree(nbplatform)
            else:
                dom = xml.dom.minidom.parse(updateTrackingFile)
                currentVersion = mx.VersionSpec(dom.getElementsByTagName('module_version')[0].getAttribute('specification_version'))
                supportedVersion = mx.VersionSpec('3.43.1')
                if currentVersion < supportedVersion:
                    mx.log('Replacing NetBeans platform version ' + str(currentVersion) + ' with version ' + str(supportedVersion))
                    shutil.rmtree(nbplatform)
                elif supportedVersion < currentVersion:
                    mx.log('Supported NetBeans version in igv command should be updated to ' + str(currentVersion))

        if not exists(nbplatform):
            mx.logv('[This execution may take a while as the NetBeans platform needs to be downloaded]')

        env = _igvBuildEnv()
        # make the jar for Batik 1.7 available.
        env['IGV_BATIK_JAR'] = mx.library('BATIK').get_path(True)
        if mx.run(['ant', '-f', mx._cygpathU2W(join(_suite.dir, 'src', 'share', 'tools', 'IdealGraphVisualizer', 'build.xml')), '-l', mx._cygpathU2W(fp.name), 'run'], env=env, nonZeroIsFatal=False):
            mx.abort("IGV ant build & launch failed. Check '" + logFile + "'. You can also try to delete 'src/share/tools/IdealGraphVisualizer/nbplatform'.")
Beispiel #2
0
def native_image_layout(dists, subdir, native_image_root, debug_gr_8964=False):
    if not dists:
        return
    dest_path = join(native_image_root, subdir)
    # Cleanup leftovers from previous call
    if exists(dest_path):
        if debug_gr_8964:
            mx.log('[mx_substratevm.native_image_layout: remove_tree: ' + dest_path + ']')
        remove_tree(dest_path)
    mkpath(dest_path)
    # Create symlinks to conform with native-image directory layout scheme
    def symlink_jar(jar_path):
        if debug_gr_8964:
            def log_stat(prefix, file_name):
                file_stat = os.stat(file_name)
                mx.log('    ' + prefix + '.st_mode: ' + oct(file_stat.st_mode))
                mx.log('    ' + prefix + '.st_mtime: ' + time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime(file_stat.st_mtime)))

            dest_jar = join(dest_path, basename(jar_path))
            mx.log('[mx_substratevm.native_image_layout.symlink_jar: symlink_or_copy')
            mx.log('  src: ' + jar_path)
            log_stat('src', jar_path)
            mx.log('  dst: ' + dest_jar)
            symlink_or_copy(jar_path, dest_jar, debug_gr_8964)
            log_stat('dst', dest_jar)
            mx.log(']')
        else:
            symlink_or_copy(jar_path, join(dest_path, basename(jar_path)), debug_gr_8964)

    for dist in dists:
        mx.logv('Add ' + type(dist).__name__ + ' ' + str(dist) + ' to ' + dest_path)
        symlink_jar(dist.path)
        if not dist.isBaseLibrary() and dist.sourcesPath:
            symlink_jar(dist.sourcesPath)
Beispiel #3
0
def hsdis(args, copyToDir=None):
    """download the hsdis library

    This is needed to support HotSpot's assembly dumping features.
    By default it downloads the Intel syntax version, use the 'att' argument to install AT&T syntax."""
    flavor = 'intel'
    if 'att' in args:
        flavor = 'att'
    if mx.get_arch() == "sparcv9":
        flavor = "sparcv9"
    lib = mx.add_lib_suffix('hsdis-' + mx.get_arch())
    path = join(_suite.dir, 'lib', lib)

    sha1s = {
        'att/hsdis-amd64.dll' : 'bcbd535a9568b5075ab41e96205e26a2bac64f72',
        'att/hsdis-amd64.so' : '58919ba085d4ef7a513f25bae75e7e54ee73c049',
        'intel/hsdis-amd64.dll' : '6a388372cdd5fe905c1a26ced614334e405d1f30',
        'intel/hsdis-amd64.so' : '844ed9ffed64fe9599638f29a8450c50140e3192',
        'intel/hsdis-amd64.dylib' : 'fdb13ef0d7d23d93dacaae9c98837bea0d4fc5a2',
        'sparcv9/hsdis-sparcv9.so': '970640a9af0bd63641f9063c11275b371a59ee60',
    }

    flavoredLib = flavor + "/" + lib
    if flavoredLib not in sha1s:
        mx.logv("hsdis not supported on this plattform or architecture")
        return

    if not exists(path):
        sha1 = sha1s[flavoredLib]
        sha1path = path + '.sha1'
        mx.download_file_with_sha1('hsdis', path, ['https://lafo.ssw.uni-linz.ac.at/pub/hsdis/' + flavoredLib], sha1, sha1path, True, True, sources=False)
    if copyToDir is not None and exists(copyToDir):
        shutil.copy(path, copyToDir)
Beispiel #4
0
 def _log_ignored_component(kept, ignored):
     """
     :type kept: GraalVmComponent
     :type ignored: GraalVmComponent
     """
     mx.logv('Suites \'{}\' and \'{}\' are registering a component with the same short name (\'{}\'), with priority \'{}\' and \'{}\' respectively.'.format(kept.suite.name, ignored.suite.name, kept.short_name, kept.priority, ignored.priority))
     mx.logv('Ignoring the one from suite \'{}\'.'.format(ignored.suite.name))
Beispiel #5
0
def jdkartifactstats(args):
    """show stats about JDK deployed Graal artifacts"""
    artifacts = {}
    jdkDir = get_jvmci_jdk().home
    def _getDeployedJars():
        if JVMCI_VERSION < 9:
            for root, _, filenames in os.walk(join(jdkDir, 'jre', 'lib')):
                for f in filenames:
                    if f.endswith('.jar') and not f.endswith('.stripped.jar'):
                        yield join(root, f)
        else:
            for jdkDist in jdkDeployedDists:
                dist = jdkDist.dist()
                if isinstance(jdkDist, JvmciJDKDeployedDist):
                    yield dist.path

    for jar in _getDeployedJars():
        f = basename(jar)
        if 'truffle' in f:
            if 'enterprise' in f:
                artifacts.setdefault('GraalEnterpriseTruffle', []).append(jar)
            else:
                artifacts.setdefault('GraalTruffle', []).append(jar)
        elif 'enterprise' in f:
            artifacts.setdefault('GraalEnterprise', []).append(jar)
        elif 'jvmci' in f:
            artifacts.setdefault('JVMCI', []).append(jar)
        elif 'graal' in f:
            artifacts.setdefault('Graal', []).append(jar)
        else:
            mx.logv('ignored: ' + jar)

    print '{:>10}  {:>10}  {:>10}  {}'.format('All', 'NoVars', 'None', 'Jar')
    for category in sorted(artifacts.viewkeys()):
        jars = artifacts[category]
        if jars:
            totals = (0, 0, 0)
            print
            for j in jars:
                gSize = os.path.getsize(j)
                stripped = j[:-len('.jar')] + '.stripped.jar'
                mx.run([mx.get_jdk().pack200, '--repack', '--quiet', '-J-Djava.util.logging.config.file=', '-DLocalVariableTypeTable=strip', '-DLocalVariableTable=strip', stripped, j])
                gLinesSourceSize = os.path.getsize(stripped)
                mx.run([mx.get_jdk().pack200, '--repack', '--quiet', '-J-Djava.util.logging.config.file=', '-G', stripped, j])
                gNoneSize = os.path.getsize(stripped)
                os.remove(stripped)
                print '{:10,}  {:10,}  {:10,}  {}:{}'.format(gSize, gLinesSourceSize, gNoneSize, category, basename(j))
                t1, t2, t3 = totals
                totals = (t1 + gSize, t2 + gLinesSourceSize, t3 + gNoneSize)
            t1, t2, t3 = totals
            print '{:10,}  {:10,}  {:10,}  {}'.format(t1, t2, t3, category)

    jvmLib = join(jdkDir, relativeVmLibDirInJdk(), get_vm(), mx.add_lib_suffix(mx.add_lib_prefix('jvm')))
    print
    if exists(jvmLib):
        print '{:10,}  {}'.format(os.path.getsize(jvmLib), jvmLib)
    else:
        print '{:>10}  {}'.format('<missing>', jvmLib)
Beispiel #6
0
def native_image_extract(dists, subdir, native_image_root):
    target_dir = join(native_image_root, subdir)
    for distribution in dists:
        mx.logv('Add dist {} to {}'.format(distribution, target_dir))
        if distribution.path.endswith('tar'):
            compressedFile = tarfile.open(distribution.path, 'r:')
        elif distribution.path.endswith('jar') or distribution.path.endswith('zip'):
            compressedFile = zipfile.ZipFile(distribution.path)
        else:
            raise mx.abort('Unsupported compressed file ' + distribution.path)
        compressedFile.extractall(target_dir)
Beispiel #7
0
def mavenSetup():
    mavenDir = join(_suite.dir, 'mxbuild', 'mvn')
    maven_repo_arg = '-Dmaven.repo.local=' + mavenDir
    env = os.environ.copy()
    env['JRUBY_BUILD_MORE_QUIET'] = 'true'
    # HACK: since the maven executable plugin does not configure the
    # java executable that is used we unfortunately need to append it to the PATH
    javaHome = os.getenv('JAVA_HOME')
    if javaHome:
        env["PATH"] = javaHome + '/bin' + os.pathsep + env["PATH"]
        mx.logv('Setting PATH to {}'.format(os.environ["PATH"]))
    mx.run(['java', '-version'])
    return maven_repo_arg, env
Beispiel #8
0
def native_image_option_properties(option_kind, option_flag, native_image_root):
    target_dir = join(native_image_root, option_kind, option_flag)
    target_path = remove_existing_symlink(join(target_dir, 'native-image.properties'))

    option_properties = None
    for svm_suite in svmSuites:
        candidate = join(svm_suite.mxDir, option_kind + '-' + option_flag + '.properties')
        if exists(candidate):
            option_properties = candidate

    if option_properties:
        mx.logv('Add symlink to ' + str(option_properties))
        mkpath(target_dir)
        symlink_or_copy(option_properties, target_path)
Beispiel #9
0
def build_number():
    """
    Get the current build number from the BUILD_NUMBER environment variable. If BUILD_NUMBER is not set or not a number,
    a default of -1 is returned.

    :return: the build number
    :rtype: int
    """
    build_num = mx.get_env("BUILD_NUMBER", default="-1")
    try:
        return int(build_num)
    except ValueError:
        mx.logv("Could not parse the build number from BUILD_NUMBER. Expected int, instead got: {0}".format(build_num))
        return -1
Beispiel #10
0
def mavenSetup():
    buildPack = join(_suite.dir, 'jruby-build-pack/maven')
    mavenDir = buildPack if isdir(buildPack) else join(_suite.dir, 'mxbuild/mvn')
    maven_repo_arg = '-Dmaven.repo.local=' + mavenDir
    env = os.environ.copy()
    if not mx.get_opts().verbose:
        env['JRUBY_BUILD_MORE_QUIET'] = 'true'
    # HACK: since the maven executable plugin does not configure the
    # java executable that is used we unfortunately need to prepend it to the PATH
    javaHome = os.getenv('JAVA_HOME')
    if javaHome:
        env["PATH"] = javaHome + '/bin' + os.pathsep + env["PATH"]
        mx.logv('Setting PATH to {}'.format(os.environ["PATH"]))
    mx.run(['java', '-version'], env=env)
    return maven_repo_arg, env
Beispiel #11
0
    def build(self):
        if not self.subject.build:
            mx.log("...skip build of {}".format(self.subject))
            return
        mx.log('...perform build of {}'.format(self.subject))

        rubyDir = _suite.dir
        mavenDir = os.path.join(rubyDir, 'mxbuild', 'mvn')

        # HACK: since the maven executable plugin does not configure the
        # java executable that is used we unfortunately need to append it to the PATH
        javaHome = os.getenv('JAVA_HOME')
        if javaHome:
            os.environ["PATH"] = os.environ["JAVA_HOME"] + '/bin' + os.pathsep + os.environ["PATH"]

        mx.logv('Setting PATH to {}'.format(os.environ["PATH"]))
        mx.logv('Calling java -version')
        mx.run(['java', '-version'])

        # Truffle version

        truffle = mx.suite('truffle')
        truffle_commit = truffle.vc.parent(truffle.dir)
        maven_version_arg = '-Dtruffle.version=' + truffle_commit
        maven_repo_arg = '-Dmaven.repo.local=' + mavenDir
        
        mx.run_mx(['maven-install', '--repo', mavenDir, '--only', 'TRUFFLE_API,TRUFFLE_DEBUG,TRUFFLE_DSL_PROCESSOR,TRUFFLE_TCK'], suite=truffle)

        open(os.path.join(rubyDir, 'VERSION'), 'w').write('graal-vm\n')

        # Build jruby-truffle
        
        env = os.environ.copy()
        env['JRUBY_BUILD_MORE_QUIET'] = 'true'

        mx.run_maven(['-q', '--version', maven_repo_arg], nonZeroIsFatal=False, cwd=rubyDir, env=env)

        mx.log('Building without tests')

        mx.run_maven(['-q', '-DskipTests', maven_version_arg, maven_repo_arg], cwd=rubyDir, env=env)

        mx.log('Building complete version')

        mx.run_maven(['-q', '-Pcomplete', '-DskipTests', maven_version_arg, maven_repo_arg], cwd=rubyDir, env=env)
        mx.run(['zip', '-d', 'maven/jruby-complete/target/jruby-complete-graal-vm.jar', 'META-INF/jruby.home/lib/*'], cwd=rubyDir)
        mx.run(['bin/jruby', 'bin/gem', 'install', 'bundler', '-v', '1.10.6'], cwd=rubyDir)
    
        mx.log('...finished build of {}'.format(self.subject))
def testdownstream(args):
    """test downstream users of GraalCore"""
    parser = ArgumentParser(prog='mx testdownstream')
    parser.add_argument('--target', action='store', help='URL of client repo to clone', required=True, metavar='<url>')
    parser.add_argument('--suitedir', action='store', help='directory of target suite in client repo', default='.', metavar='<path>')
    parser.add_argument('-C', dest='clientMxCmd', action='append', help='arg to mx command run on client (e.g., -C-v -C--strict-compliance -Cgate)', default=[], metavar='<arg>')

    args = parser.parse_args(args)

    workDir = join(_suite.get_output_root(), 'testdownstream')
    mirror = join(workDir, _suite.name)
    if exists(mirror):
        shutil.rmtree(mirror)
    mx.ensure_dir_exists(mirror)
    for f in os.listdir(_suite.dir):
        subDir = join(_suite.dir, f)
        if subDir == _suite.get_output_root():
            continue
        src = join(_suite.dir, f)
        dst = join(mirror, f)
        mx.logv('[Creating symlink from {} to {}]'.format(dst, src))
        os.symlink(src, dst)

    # Deduce a target name from the target URL
    url = urlparse(args.target)
    targetName = url.path
    if targetName.rfind('/') != -1:
        targetName = targetName[targetName.rfind('/') + 1:]
    if targetName.endswith('.git'):
        targetName = targetName[0:-len('.git')]

    targetDir = join(workDir, targetName)
    git = mx.GitConfig()
    if exists(targetDir):
        git.pull(targetDir)
    else:
        git.clone(args.target, targetDir)

    # See if there's a matching (non-master) branch downstream and use it if there is
    branch = git.git_command(_suite.dir, ['rev-parse', '--abbrev-ref', 'HEAD']).strip()
    if branch != 'master':
        git.git_command(targetDir, ['checkout', branch], abortOnError=False)

    targetSuiteDir = join(targetDir, args.suitedir)
    cmd = ['--java-home=' + mx.get_jdk().home] + args.clientMxCmd
    mx.logv('[running "mx ' + ' '.join(cmd) + '" in ' + targetSuiteDir + ']')
    return mx.run_mx(cmd, targetSuiteDir)
def hsdis(args, copyToDir=None):
    """download the hsdis library

    This is needed to support HotSpot's assembly dumping features.
    By default it downloads the Intel syntax version, use the 'att' argument to install AT&T syntax."""
    flavor = None
    if mx.get_arch() == "amd64":
        flavor = mx.get_env('HSDIS_SYNTAX')
        if flavor is None:
            flavor = 'intel'
        if 'att' in args:
            flavor = 'att'

    libpattern = mx.add_lib_suffix('hsdis-' + mx.get_arch() + '-' + mx.get_os() + '-%s')

    sha1s = {
        'att/hsdis-amd64-windows-%s.dll' : 'bcbd535a9568b5075ab41e96205e26a2bac64f72',
        'att/hsdis-amd64-linux-%s.so' : '36a0b8e30fc370727920cc089f104bfb9cd508a0',
        'att/hsdis-amd64-darwin-%s.dylib' : 'c1865e9a58ca773fdc1c5eea0a4dfda213420ffb',
        'intel/hsdis-amd64-windows-%s.dll' : '6a388372cdd5fe905c1a26ced614334e405d1f30',
        'intel/hsdis-amd64-linux-%s.so' : '0d031013db9a80d6c88330c42c983fbfa7053193',
        'intel/hsdis-amd64-darwin-%s.dylib' : '67f6d23cbebd8998450a88b5bef362171f66f11a',
        'hsdis-sparcv9-solaris-%s.so': '970640a9af0bd63641f9063c11275b371a59ee60',
        'hsdis-sparcv9-linux-%s.so': '0c375986d727651dee1819308fbbc0de4927d5d9',
    }

    if flavor:
        flavoredLib = flavor + "/" + libpattern
    else:
        flavoredLib = libpattern
    if flavoredLib not in sha1s:
        mx.warn("hsdis with flavor '{}' not supported on this plattform or architecture".format(flavor))
        return

    sha1 = sha1s[flavoredLib]
    lib = flavoredLib % (sha1)
    path = join(_suite.get_output_root(), lib)
    if not exists(path):
        sha1path = path + '.sha1'
        mx.download_file_with_sha1('hsdis', path, ['https://lafo.ssw.uni-linz.ac.at/pub/hsdis/' + lib], sha1, sha1path, True, True, sources=False)
    if copyToDir is not None and exists(copyToDir):
        destFileName = mx.add_lib_suffix('hsdis-' + mx.get_arch())
        mx.logv('Copying {} to {}'.format(path, copyToDir + os.sep + destFileName))
        shutil.copy(path, copyToDir + os.sep + destFileName)
Beispiel #14
0
def _netbeansinit_suite(args,
                        suite,
                        refreshOnly=False,
                        buildProcessorJars=True):
    mxOutputDir = mx.ensure_dir_exists(suite.get_mx_output_dir())
    configZip = mx.TimeStampFile(join(mxOutputDir, 'netbeans-config.zip'))
    configLibsZip = join(mxOutputDir, 'eclipse-config-libs.zip')
    if refreshOnly and not configZip.exists():
        return

    if mx_ideconfig._check_ide_timestamp(suite, configZip, 'netbeans'):
        mx.logv('[NetBeans configurations are up to date - skipping]')
        return

    files = []
    libFiles = []
    jdks = set()
    for p in suite.projects:
        if not p.isJavaProject():
            continue

        if exists(join(p.dir, 'plugin.xml')):  # eclipse plugin project
            continue

        includedInDists = [d for d in suite.dists if p in d.archived_deps()]
        _netbeansinit_project(p, jdks, files, libFiles, includedInDists)
    mx.log('If using NetBeans:')
    # http://stackoverflow.com/questions/24720665/cant-resolve-jdk-internal-package
    mx.log(
        '  1. Edit etc/netbeans.conf in your NetBeans installation and modify netbeans_default_options variable to include "-J-DCachingArchiveProvider.disableCtSym=true"'
    )
    mx.log(
        '  2. Ensure that the following platform(s) are defined (Tools -> Java Platforms):'
    )
    for jdk in jdks:
        mx.log('        JDK_' + str(jdk.version))
    mx.log(
        '  3. Open/create a Project Group for the directory containing the projects (File -> Project Group -> New Group... -> Folder of Projects)'
    )

    mx_ideconfig._zip_files(files, suite.dir, configZip.path)
    mx_ideconfig._zip_files(libFiles, suite.dir, configLibsZip)
Beispiel #15
0
def setupNodeEnvironment(args, add_graal_vm_args=True):
    args = args if args else []
    mode, vmArgs, progArgs = _parseArgs(args)
    setLibraryPath()

    if _is_windows:
        processDevkitRoot()

    if mx.suite('vm', fatalIfMissing=False) is not None and mx.suite('substratevm', fatalIfMissing=False) is not None:
        _prepare_svm_env()
        return mode, vmArgs, progArgs

    if mx.suite('vm', fatalIfMissing=False) is not None or mx.suite('substratevm', fatalIfMissing=False) is not None:
        mx.warn("Running on the JVM.\nIf you want to run on SubstrateVM, you need to dynamically import both '/substratevm' and '/vm'.\nExample: 'mx --env svm node'")

    _setEnvVar('JAVA_HOME', _java_home())
    if mx.suite('compiler', fatalIfMissing=False) is None:
        _setEnvVar('GRAAL_SDK_JAR_PATH', mx.distribution('sdk:GRAAL_SDK').path)
    _setEnvVar('LAUNCHER_COMMON_JAR_PATH', mx.distribution('sdk:LAUNCHER_COMMON').path)
    _setEnvVar('TRUFFLENODE_JAR_PATH', mx.distribution('TRUFFLENODE').path)
    node_jvm_cp = (os.environ['NODE_JVM_CLASSPATH'] + pathsep) if 'NODE_JVM_CLASSPATH' in os.environ else ''
    node_cp = node_jvm_cp + mx.classpath(['TRUFFLENODE'] + (['tools:CHROMEINSPECTOR', 'tools:TRUFFLE_PROFILER', 'tools:AGENTSCRIPT'] if mx.suite('tools', fatalIfMissing=False) is not None else []))
    _setEnvVar('NODE_JVM_CLASSPATH', node_cp)

    prevPATH = os.environ['PATH']
    _setEnvVar('PATH', "%s%s%s" % (join(_suite.mxDir, 'fake_launchers'), pathsep, prevPATH))

    if _has_jvmci() and add_graal_vm_args:
        if mx.suite('graal-enterprise', fatalIfMissing=False):
            # explicitly require the enterprise compiler configuration
            vmArgs += ['-Dgraal.CompilerConfiguration=enterprise']
        if mx.suite('compiler', fatalIfMissing=False):
            vmArgs += ['-Djvmci.Compiler=graal', '-XX:+UnlockExperimentalVMOptions', '-XX:+EnableJVMCI']

    if isinstance(_suite, BinarySuite):
        mx.logv('%s is a binary suite' % _suite.name)
        tarfilepath = mx.distribution('TRUFFLENODE_GRAALVM_SUPPORT').path
        with tarfile.open(tarfilepath, 'r:') as tar:
            mx.logv('Extracting {} to {}'.format(tarfilepath, _suite.dir))
            tar.extractall(_suite.dir)

    return mode, vmArgs, progArgs
Beispiel #16
0
    def commits_in_range(self):
        grep_filter = str(self.config.commits_filter).replace('[',
                                                              r'\[').replace(
                                                                  ']', r'\]')
        commits_filter = self.build_steps.git_log_filtering_strategy()
        git_log_command_base = commits_filter + [
            '--grep=' + grep_filter, '--pretty=format:%h|%ct|%s'
        ]

        if self.config.start_commit is not None:
            commits_range = [
                self.config.start_commit + '..' + self.config.end_commit
            ]
        elif self.config.after is not None:
            commits_range = [
                '--after="' + self.config.after + '"',
                '--before="' + self.config.before + '"'
            ]
        else:
            commits_range = [
                '--after="' + self.build_steps.default_start_date() + '"',
                '--before="' + self.config.before + '"'
            ]

        mx.log('Commits Filtering Strategy: ' +
               ' '.join(commits_filter + commits_range))
        unparsed_commits = subprocess.check_output(
            git_log_command_base + commits_range,
            universal_newlines=True).splitlines()

        def commit_parser(c):
            commit_hash, date, msg = c.split('|', 2)
            commit = Commit(commit_hash, msg,
                            datetime.fromtimestamp(int(date)))
            mx.logv(commit)
            return commit

        mx.logv('---------- Commits in range to analyze')
        commits = list(map(commit_parser, unparsed_commits))
        mx.log('Commits in range to analyze: {}, hashes [{} - {}]'.format(
            len(commits), commits[len(commits) - 1].hash, commits[0].hash))
        return commits
Beispiel #17
0
def setupNodeEnvironment(args, add_graal_vm_args=True):
    javaHome = _getJdkHome()
    _setEnvVar('JAVA_HOME', javaHome)
    if mx.suite('compiler', fatalIfMissing=False) is None:
        _setEnvVar('GRAAL_SDK_JAR_PATH', mx.distribution('sdk:GRAAL_SDK').path)
        _setEnvVar('TRUFFLE_JAR_PATH',
                   mx.distribution('truffle:TRUFFLE_API').path)
    _setEnvVar('LAUNCHER_COMMON_JAR_PATH',
               mx.distribution('sdk:LAUNCHER_COMMON').path)
    _setEnvVar('TRUFFLENODE_JAR_PATH', mx.distribution('TRUFFLENODE').path)
    _setEnvVar(
        'NODE_JVM_CLASSPATH',
        mx.classpath(['TRUFFLENODE'] + (
            ['tools:CHROMEINSPECTOR', 'tools:TRUFFLE_PROFILER'] if mx.
            suite('tools', fatalIfMissing=False) is not None else [])))
    setLibraryPath()

    prevPATH = os.environ['PATH']
    _setEnvVar('PATH',
               "%s:%s" % (join(_suite.mxDir, 'fake_launchers'), prevPATH))

    args = args if args else []
    mode, vmArgs, progArgs = _parseArgs(args)

    if add_graal_vm_args:
        if mx.suite('graal-enterprise', fatalIfMissing=False):
            # explicitly require the enterprise compiler configuration
            vmArgs += ['-Dgraal.CompilerConfiguration=enterprise']
        if mx.suite('compiler', fatalIfMissing=False):
            vmArgs += [
                '-Djvmci.Compiler=graal', '-XX:+UnlockExperimentalVMOptions',
                '-XX:+EnableJVMCI'
            ]

    if isinstance(_suite, BinarySuite):
        mx.logv('%s is a binary suite' % _suite.name)
        tarfilepath = mx.distribution('TRUFFLENODE_GRAALVM_SUPPORT').path
        with tarfile.open(tarfilepath, 'r:') as tar:
            mx.logv('Extracting {} to {}'.format(tarfilepath, _suite.dir))
            tar.extractall(_suite.dir)

    return mode, vmArgs, progArgs
Beispiel #18
0
def python_build_watch(args):
    """
    Watch the suite and on any changes to .class, .jar, .h, or .c files rebuild.
    By default, rebuilds only the archives and non-Java projects.
    """
    parser = ArgumentParser(prog='mx python-build-watch')
    parser.add_argument('--full', action='store_true', help='Run a full mx build', required=False)
    parser.add_argument('--graalvm', action='store_true', help='Build a graalvm', required=False)
    parser.add_argument('--no-java', action='store_true', help='Build only archives and native projects [default]', required=False)
    args = parser.parse_args(args)
    if sum([args.full, args.graalvm, args.no_java]) > 1:
        mx.abort("Only one of --full, --graalvm, --no-java can be specified")
    if args.full:
        suffixes = [".c", ".h", ".class", ".jar", ".java"]
        excludes = [".*\\.py$"]
    elif args.graalvm:
        suffixes = [".c", ".h", ".class", ".jar", ".java", ".py"]
        excludes = ["mx_.*\\.py$"]
    else:
        suffixes = [".c", ".h", ".class", ".jar"]
        excludes = [".*\\.py$", ".*\\.java$"]

    cmd = ["inotifywait", "-q", "-e", "close_write,moved_to", "-r", "--format=%f"]
    for e in excludes:
        cmd += ["--exclude", e]
    cmd += ["@%s" % os.path.join(SUITE.dir, ".git"), SUITE.dir]

    while True:
        out = mx.OutputCapture()
        mx.run(cmd, out=out)
        changed_file = out.data.strip()
        mx.logv(changed_file)
        if any(changed_file.endswith(ext) for ext in [".c", ".h", ".class", ".jar"]):
            mx.log("Build needed ...")
            time.sleep(2)
            if args.full:
                mx.command_function("build")()
            elif args.graalvm:
                mx.log(python_gvm())
            else:
                nativebuild([])
        mx.log("Build done.")
Beispiel #19
0
def native_image_layout(dists, subdir, native_image_root, debug_gr_8964=False):
    if not dists:
        return
    dest_path = join(native_image_root, subdir)
    # Cleanup leftovers from previous call
    if exists(dest_path):
        if debug_gr_8964:
            mx.log('[mx_substratevm.native_image_layout: remove_tree: ' +
                   dest_path + ']')
        remove_tree(dest_path)
    mkpath(dest_path)

    # Create symlinks to conform with native-image directory layout scheme
    def symlink_jar(jar_path):
        if debug_gr_8964:

            def log_stat(prefix, file_name):
                file_stat = os.stat(file_name)
                mx.log('    ' + prefix + '.st_mode: ' + oct(file_stat.st_mode))
                mx.log('    ' + prefix + '.st_mtime: ' + time.strftime(
                    '%Y-%m-%dT%H:%M:%SZ', time.gmtime(file_stat.st_mtime)))

            dest_jar = join(dest_path, basename(jar_path))
            mx.log(
                '[mx_substratevm.native_image_layout.symlink_jar: symlink_or_copy'
            )
            mx.log('  src: ' + jar_path)
            log_stat('src', jar_path)
            mx.log('  dst: ' + dest_jar)
            symlink_or_copy(jar_path, dest_jar, debug_gr_8964)
            log_stat('dst', dest_jar)
            mx.log(']')
        else:
            symlink_or_copy(jar_path, join(dest_path, basename(jar_path)),
                            debug_gr_8964)

    for dist in dists:
        mx.logv('Add ' + type(dist).__name__ + ' ' + str(dist) + ' to ' +
                dest_path)
        symlink_jar(dist.path)
        if not dist.isBaseLibrary() and dist.sourcesPath:
            symlink_jar(dist.sourcesPath)
Beispiel #20
0
def native_image_layout(dists, subdir, native_image_root):
    dest_path = join(native_image_root, subdir)
    # Cleanup leftovers from previous call
    if exists(dest_path):
        remove_tree(dest_path)
    mkpath(dest_path)

    # Create symlinks to conform with native-image directory layout scheme
    def symlink_jar(jar_path):
        relsymlink(jar_path, join(dest_path, basename(jar_path)))

    for dist in dists:
        mx.logv('Add ' + type(dist).__name__ + ' ' + str(dist) + ' to ' +
                dest_path)
        if dist.isLibrary():
            dist.get_path(resolve=True)
            dist.get_source_path(resolve=True)
        symlink_jar(dist.path)
        if not dist.isLibrary():
            symlink_jar(dist.sourcesPath)
Beispiel #21
0
def checkCFiles(target, reason):
    error = False
    files_to_check = []
    if os.path.isfile(target):
        files_to_check.append(target)
    else:
        for path, _, files in os.walk(target):
            for f in files:
                if f.endswith('.c') or f.endswith('.cpp') or f.endswith(
                        '.h') or f.endswith('.hpp'):
                    files_to_check.append(os.path.join(path, f))
    if not files_to_check:
        mx.logv("clang-format: no files found {} ({})".format(target, reason))
        return True
    mx.logv("clang-format: checking {} ({}, {} files)".format(
        target, reason, len(files_to_check)))
    for f in files_to_check:
        if not checkCFile(f):
            error = True
    return not error
Beispiel #22
0
def _fetch_test_suite(dest, library_names):
    def _get_lib_path(_lib_name):
        return mx.library(_lib_name).get_path(resolve=True)

    _extract = False
    for _lib_name in library_names:
        if not exists(dest) or getmtime(
                _get_lib_path(_lib_name)) > getmtime(dest):
            mx.logv('{} needs to be extracted'.format(_lib_name))
            _extract = True
            break

    if _extract:
        if exists(dest):
            mx.logv('Deleting the old test directory {}'.format(dest))
            shutil.rmtree(dest)
            mx.ensure_dir_exists(dest)
        for _lib_name in library_names:
            with tarfile.open(_get_lib_path(_lib_name), 'r') as _tar:
                _tar.extractall(dest)
Beispiel #23
0
def _runmultimake(args):
    """run the JDK make process for one or more configurations"""

    jvmVariantsDefault = ','.join(_jdkJvmVariants)
    debugLevelsDefault = ','.join(_jdkDebugLevels)

    parser = ArgumentParser(prog='mx multimake')
    parser.add_argument('--jdk-jvm-variants', '--vms', help='a comma separated list of VMs to build (default: ' + jvmVariantsDefault + ')', metavar='<args>', default=jvmVariantsDefault)
    parser.add_argument('--jdk-debug-levels', '--builds', help='a comma separated list of JDK debug levels (default: ' + debugLevelsDefault + ')', metavar='<args>', default=debugLevelsDefault)
    parser.add_argument('-n', '--no-check', action='store_true', help='omit running "java -version" after each build')
    select = parser.add_mutually_exclusive_group()
    select.add_argument('-c', '--console', action='store_true', help='send build output to console instead of log files')
    select.add_argument('-d', '--output-dir', help='directory for log files instead of current working directory', default=os.getcwd(), metavar='<dir>')

    args = parser.parse_args(args)
    jvmVariants = args.jdk_jvm_variants.split(',')
    debugLevels = [_translateLegacyDebugLevel(dl) for dl in args.jdk_debug_levels.split(',')]

    allStart = time.time()
    for jvmVariant in jvmVariants:
        for debugLevel in debugLevels:
            if not args.console:
                logFile = join(mx.ensure_dir_exists(args.output_dir), jvmVariant + '-' + debugLevel + '.log')
                log = open(logFile, 'wb')
                start = time.time()
                mx.log('BEGIN: ' + jvmVariant + '-' + debugLevel + '\t(see: ' + logFile + ')')
                verbose = ['-v'] if mx.get_opts().verbose else []
                # Run as subprocess so that output can be directed to a file
                cmd = [sys.executable, '-u', mx.__file__] + verbose + ['--jdk-jvm-variant=' + jvmVariant, '--jdk-debug-level=' + debugLevel, 'make']
                mx.logv("executing command: " + str(cmd))
                subprocess.check_call(cmd, cwd=_suite.dir, stdout=log, stderr=subprocess.STDOUT)
                duration = datetime.timedelta(seconds=time.time() - start)
                mx.log('END:   ' + jvmVariant + '-' + debugLevel + '\t[' + str(duration) + ']')
            else:
                with VM(jvmVariant=jvmVariant, debugLevel=debugLevel):
                    _runmake([])
            if not args.no_check:
                with VM(jvmciMode='jit'):
                    run_vm(['-XX:-BootstrapJVMCI', '-version'])
    allDuration = datetime.timedelta(seconds=time.time() - allStart)
    mx.log('TOTAL TIME:   ' + '[' + str(allDuration) + ']')
Beispiel #24
0
def _runmultimake(args):
    """run the JDK make process for one or more configurations"""

    jvmVariantsDefault = ','.join(_jdkJvmVariants)
    debugLevelsDefault = ','.join(_jdkDebugLevels)

    parser = ArgumentParser(prog='mx multimake')
    parser.add_argument('--jdk-jvm-variants', '--vms', help='a comma separated list of VMs to build (default: ' + jvmVariantsDefault + ')', metavar='<args>', default=jvmVariantsDefault)
    parser.add_argument('--jdk-debug-levels', '--builds', help='a comma separated list of JDK debug levels (default: ' + debugLevelsDefault + ')', metavar='<args>', default=debugLevelsDefault)
    parser.add_argument('-n', '--no-check', action='store_true', help='omit running "java -version" after each build')
    select = parser.add_mutually_exclusive_group()
    select.add_argument('-c', '--console', action='store_true', help='send build output to console instead of log files')
    select.add_argument('-d', '--output-dir', help='directory for log files instead of current working directory', default=os.getcwd(), metavar='<dir>')

    args = parser.parse_args(args)
    jvmVariants = args.jdk_jvm_variants.split(',')
    debugLevels = [_translateLegacyDebugLevel(dl) for dl in args.jdk_debug_levels.split(',')]

    allStart = time.time()
    for jvmVariant in jvmVariants:
        for debugLevel in debugLevels:
            if not args.console:
                logFile = join(mx.ensure_dir_exists(args.output_dir), jvmVariant + '-' + debugLevel + '.log')
                log = open(logFile, 'wb')
                start = time.time()
                mx.log('BEGIN: ' + jvmVariant + '-' + debugLevel + '\t(see: ' + logFile + ')')
                verbose = ['-v'] if mx.get_opts().verbose else []
                # Run as subprocess so that output can be directed to a file
                cmd = [sys.executable, '-u', mx.__file__] + verbose + ['--jdk-jvm-variant=' + jvmVariant, '--jdk-debug-level=' + debugLevel, 'make']
                mx.logv("executing command: " + str(cmd))
                subprocess.check_call(cmd, cwd=_suite.dir, stdout=log, stderr=subprocess.STDOUT)
                duration = datetime.timedelta(seconds=time.time() - start)
                mx.log('END:   ' + jvmVariant + '-' + debugLevel + '\t[' + str(duration) + ']')
            else:
                with VM(jvmVariant=jvmVariant, debugLevel=debugLevel):
                    _runmake([])
            if not args.no_check:
                with VM(jvmciMode='jit'):
                    run_vm(['-XX:-BootstrapJVMCI', '-version'])
    allDuration = datetime.timedelta(seconds=time.time() - allStart)
    mx.log('TOTAL TIME:   ' + '[' + str(allDuration) + ']')
Beispiel #25
0
    def build(self):
        if not self.subject.build:
            mx.log("...skip build of {}".format(self.subject))
            return
        mx.log('...perform build of {}'.format(self.subject))

        rubyDir = _suite.dir

        # HACK: since the maven executable plugin does not configure the
        # java executable that is used we unfortunately need to append it to the PATH
        javaHome = os.getenv('JAVA_HOME')
        if javaHome:
            os.environ["PATH"] = os.environ["JAVA_HOME"] + '/bin' + os.pathsep + os.environ["PATH"]

        mx.logv('Setting PATH to {}'.format(os.environ["PATH"]))
        mx.logv('Calling java -version')
        mx.run(['java', '-version'])

        # Truffle version

        truffle = mx.suite('truffle')
        truffle_commit = truffle.vc.parent(truffle.dir)

        mx.run_mx(['maven-install'], suite=truffle)

        open(join(rubyDir, 'VERSION'), 'w').write('graal-vm\n')

        # Build jruby-truffle and

        mx.run(['find', '.'], nonZeroIsFatal=False, cwd=rubyDir)
        mx.run_maven(['--version'], nonZeroIsFatal=False, cwd=rubyDir)

        mx.log('Building without tests')

        mx.run_maven(['-DskipTests', '-Dtruffle.version=' + truffle_commit], cwd=rubyDir)

        mx.log('Building complete version')

        mx.run_maven(['-Pcomplete', '-DskipTests', '-Dtruffle.version=' + truffle_commit], cwd=rubyDir)
        mx.run(['zip', '-d', 'maven/jruby-complete/target/jruby-complete-graal-vm.jar', 'META-INF/jruby.home/lib/*'], cwd=rubyDir)
        mx.run(['bin/jruby', 'bin/gem', 'install', 'bundler', '-v', '1.10.6'], cwd=rubyDir)
Beispiel #26
0
def spotbugs(args,
             fbArgs=None,
             suite=None,
             projects=None,
             jarFileName='spotbugs.jar'):
    projectsToTest = [p for p in mx.projects() if _should_test_project(p)]
    projectsByVersion = {}
    for p in projectsToTest:
        compat = p.suite.getMxCompatibility()
        spotbugsVersion = compat.spotbugs_version()
        if spotbugsVersion not in projectsByVersion:
            projectsByVersion[spotbugsVersion] = []
        projectsByVersion[spotbugsVersion].append(p)
    resultcode = 0
    for spotbugsVersion, versionProjects in projectsByVersion.items():
        mx.logv('Running spotbugs version {} on projects {}'.format(
            spotbugsVersion, versionProjects))
        resultcode = max(
            resultcode,
            _spotbugs(args, fbArgs, suite, versionProjects, spotbugsVersion))
    return resultcode
Beispiel #27
0
def hsdis(args, copyToDir=None):
    """download the hsdis library

    This is needed to support HotSpot's assembly dumping features.
    By default it downloads the Intel syntax version, use the 'att' argument to install AT&T syntax."""
    flavor = 'intel'
    if 'att' in args:
        flavor = 'att'
    if mx.get_arch() == "sparcv9":
        flavor = "sparcv9"
    lib = mx.add_lib_suffix('hsdis-' + mx.get_arch())
    path = join(_suite.dir, 'lib', lib)

    sha1s = {
        'att/hsdis-amd64.dll': 'bcbd535a9568b5075ab41e96205e26a2bac64f72',
        'att/hsdis-amd64.so': '58919ba085d4ef7a513f25bae75e7e54ee73c049',
        'intel/hsdis-amd64.dll': '6a388372cdd5fe905c1a26ced614334e405d1f30',
        'intel/hsdis-amd64.so': '844ed9ffed64fe9599638f29a8450c50140e3192',
        'intel/hsdis-amd64.dylib': 'fdb13ef0d7d23d93dacaae9c98837bea0d4fc5a2',
        'sparcv9/hsdis-sparcv9.so': '970640a9af0bd63641f9063c11275b371a59ee60',
    }

    flavoredLib = flavor + "/" + lib
    if flavoredLib not in sha1s:
        mx.logv("hsdis not supported on this plattform or architecture")
        return

    if not exists(path):
        sha1 = sha1s[flavoredLib]
        sha1path = path + '.sha1'
        mx.download_file_with_sha1(
            'hsdis',
            path, ['https://lafo.ssw.uni-linz.ac.at/pub/hsdis/' + flavoredLib],
            sha1,
            sha1path,
            True,
            True,
            sources=False)
    if copyToDir is not None and exists(copyToDir):
        shutil.copy(path, copyToDir)
Beispiel #28
0
 def process_dep(dep, edge):
     if dep is proj:
         return
     if dep.isLibrary() or dep.isJARDistribution() or dep.isMavenProject():
         libraries.add(dep)
         moduleXml.element('orderEntry', attributes={'type': 'library', 'name': dep.name, 'level': 'project'})
     elif dep.isJavaProject():
         dependencies_project_packages.update(dep.defined_java_packages())
         referenced_modules.add(dep.name)
         moduleXml.element('orderEntry', attributes={'type': 'module', 'module-name': dep.name})
     elif dep.isJdkLibrary():
         jdk_libraries.add(dep)
         if jdk.javaCompliance < dep.jdkStandardizedSince:
             moduleXml.element('orderEntry', attributes={'type': 'library', 'name': dep.name, 'level': 'project'})
         else:
             mx.logv("{} skipping {} for {}".format(p, dep, jdk)) #pylint: disable=undefined-loop-variable
     elif dep.isJreLibrary():
         pass
     elif dep.isClasspathDependency():
         moduleXml.element('orderEntry', attributes={'type': 'library', 'name': dep.name, 'level': 'project'})
     else:
         mx.abort("Dependency not supported: {0} ({1})".format(dep, dep.__class__.__name__))
Beispiel #29
0
def mavenSetup():
    maven_args = []
    env = os.environ.copy()
    if not mx.get_opts().verbose:
        maven_args.append('-q')

    buildPack = join(_suite.dir, 'jruby-build-pack/maven')
    if isdir(buildPack):
        maven_args.append('-Dmaven.repo.local=' + buildPack)
    elif 'CI' in env and 'TRAVIS' not in env:
        maven_args.append('-Dmaven.repo.local=' +
                          join(_suite.dir, 'mxbuild/mvn'))

    if not mx.get_opts().verbose:
        env['JRUBY_BUILD_MORE_QUIET'] = 'true'
    # HACK: since the maven executable plugin does not configure the
    # java executable that is used we unfortunately need to prepend it to the PATH
    javaHome = os.getenv('JAVA_HOME')
    if javaHome:
        env["PATH"] = javaHome + '/bin' + os.pathsep + env["PATH"]
        mx.logv('Setting PATH to {}'.format(env["PATH"]))
    mx.run(['java', '-version'], env=env)
    return maven_args, env
Beispiel #30
0
def run_js(vmArgs, jsArgs, nonZeroIsFatal, out, err, cwd):
    bench_conf, should_bench_compile_server = _get_bench_conf(vmArgs)
    _, _, image_path = _bench_image_params(bench_conf)
    if should_bench_compile_server and not exists(image_path):
        for _ in range(_IMAGE_BENCH_REPETITIONS):
            with mx_substratevm.native_image_context(
                    config=mx_substratevm._graalvm_js_config,
                    build_if_missing=True):
                _bench_compile_server(bench_conf, out)

    image_path = bench_jsimage(bench_conf, out=out, err=err)
    if image_path:
        vmArgs = [
            vmArg for vmArg in vmArgs
            if not (vmArg.startswith(_conf_arg_prefix)
                    or vmArg == '--bench-compilation-server')
        ]

        all_args = vmArgs + jsArgs
        mx.logv("Running substratevm image '%s' with '%s' i.e.:" %
                (image_path, all_args))
        mx.logv(image_path + " " + " ".join(all_args))
        runner = mx_substratevm.ProcessRunner([image_path] + all_args,
                                              stderr=subprocess.STDOUT)

        timeout_factor, _ = _bench_configs[bench_conf]
        with _timedelta('IMAGERUN: ', out=out):
            returncode, stdoutdata, _ = runner.run(8 * 60 * timeout_factor)
            if runner.timedout:
                if nonZeroIsFatal:
                    mx.abort('Javascript benchmark timeout')
                return -1
            out(stdoutdata)
            return returncode
    if nonZeroIsFatal:
        mx.abort('Javascript image building for js-benchmarks failed')
    return -1
def native_image_layout(dists, subdir, native_image_root, debug_gr_8964=False):
    if not dists:
        return
    dest_path = join(native_image_root, subdir)
    # Cleanup leftovers from previous call
    if exists(dest_path):
        if debug_gr_8964:
            mx.log('[mx_substratevm.native_image_layout: remove_tree: ' +
                   dest_path + ']')
        remove_tree(dest_path)
    mkpath(dest_path)

    # Create symlinks to conform with native-image directory layout scheme
    # GR-8964: Copy the jar instead of symlinking to it.
    def symlink_jar(jar_path):
        if debug_gr_8964:
            dest_jar = join(dest_path, basename(jar_path))
            if debug_gr_8964:
                mx.log('[mx_substratevm.native_image_layout.symlink_jar: copy2' + \
                    '\n  src: ' + jar_path + \
                    '\n  dst: ' + dest_jar)
            copy2(jar_path, dest_jar)
            dest_stat = os.stat(dest_jar)
            if debug_gr_8964:
                mx.log('      ' + \
                    ' .st_mode: ' + oct(dest_stat.st_mode) + \
                    ' .st_mtime: ' + time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime(dest_stat.st_mtime)) + ']')
        else:
            relsymlink(jar_path, join(dest_path, basename(jar_path)))

    for dist in dists:
        mx.logv('Add ' + type(dist).__name__ + ' ' + str(dist) + ' to ' +
                dest_path)
        symlink_jar(dist.path)
        if not dist.isLibrary():
            symlink_jar(dist.sourcesPath)
Beispiel #32
0
def _pkgtest_args(args):
    graalvm_home = None
    if 'FASTR_GRAALVM' in os.environ:
        graalvm_home = os.environ['FASTR_GRAALVM']
    elif 'GRAALVM_FASTR' in os.environ:
        graalvm_home = os.environ['GRAALVM_FASTR']

    pkgtest_args = []
    pkgtest_args += ["--fastr-home"]
    pkgtest_args += [_fastr_suite.dir]
    if graalvm_home:
        # In GRAALVM mode, we assume FastR is not built so we need to
        _gnur_suite = mx.suite('gnur')
        pkgtest_args += ["--gnur-home"]
        pkgtest_args += [join(_gnur_suite.dir, 'gnur', _gnur_suite.extensions.r_version())]
        pkgtest_args += ["--graalvm-home"]
        pkgtest_args += [graalvm_home]
    else:
        pkgtest_args += ["--gnur-home"]
        pkgtest_args += [_gnur_path()]
    mx.log(args)
    full_args = pkgtest_args + list(args)
    mx.logv(full_args)
    return full_args
Beispiel #33
0
    def _run(self, *args, **kwargs):
        cmd = [self.binary, '-j', self.parallelism]
        if mx.get_opts().very_verbose:
            cmd += ['-v']
        cmd += args

        out = kwargs.get('out', mx.OutputCapture())
        err = kwargs.get('err', subprocess.STDOUT)
        if mx.get_opts().verbose:
            if callable(out) and '-n' not in args:
                out = mx.TeeOutputCapture(out)
            if callable(err):
                err = mx.TeeOutputCapture(err)

        try:
            rc = mx.run(cmd, nonZeroIsFatal=False, out=out, err=err, cwd=self.build_dir)
        except OSError as e:
            if e.errno != errno.EACCES:
                mx.abort('Error executing \'{}\': {}'.format(' '.join(cmd), str(e)))
            mx.logv('{} is not executable. Trying to change permissions...'.format(self.binary))
            os.chmod(self.binary, 0o755)
            self._run(*args, **kwargs)  # retry
        else:
            not rc or mx.abort(rc if mx.get_opts().verbose else out.data)  # pylint: disable=expression-not-assigned
Beispiel #34
0
def _parseArgs(args):
    arguments = list(args)
    debugArg = '--debug'
    if debugArg in arguments:
        mx.log('Running in debug mode. The --debug argument is handled by mx and not passed as program argument')
        arguments.remove(debugArg)
        mode = 'Debug'
    else:
        mode = 'Release'

    vmArgs, progArgs = parse_js_args(arguments)
    if mx.suite('compiler', fatalIfMissing=False):
        import mx_compiler
        vmArgs = mx_compiler._parseVmArgs(vmArgs)

    if '-d64' in vmArgs:
        mx.logv('[_parseArgs] removing -d64 from vmArgs')
        vmArgs.remove('-d64')

    mx.logv('[_parseArgs] mode: %s' % mode)
    mx.logv('[_parseArgs] vmArgs: %s' % vmArgs)
    mx.logv('[_parseArgs] progArgs: %s' % progArgs)

    return mode, vmArgs, progArgs
Beispiel #35
0
def _setEnvVar(name, val):
    if val:
        mx.logv('Setting environment variable %s=%s' % (name, val))
        os.environ[name] = val
Beispiel #36
0
def jlink_new_jdk(jdk, dst_jdk_dir, module_dists, root_module_names=None, missing_export_target_action='create', with_source=lambda x: True):
    """
    Uses jlink from `jdk` to create a new JDK image in `dst_jdk_dir` with `module_dists` and
    their dependencies added to the JDK image, replacing any existing modules of the same name.

    :param JDKConfig jdk: source JDK
    :param str dst_jdk_dir: path to use for the jlink --output option
    :param list module_dists: list of distributions defining modules
    :param list root_module_names: list of strings naming the module root set for the new JDK image.
                     The named modules must either be in `module_dists` or in `jdk`. If None, then
                     the root set will be all the modules in ``module_dists` and `jdk`.
    :param str missing_export_target_action: the action to perform for a qualifed export target that
                     is not present in `module_dists` and does not have a hash stored in java.base.
                     The choices are:
                       "create" - an empty module is created
                        "error" - raise an error
                           None - do nothing
    :param lambda with_source: returns True if the sources of a module distribution must be included in the new JDK
    """
    assert callable(with_source)

    if jdk.javaCompliance < '9':
        mx.abort('Cannot derive a new JDK from ' + jdk.home + ' with jlink since it is not JDK 9 or later')

    exploded_java_base_module = join(jdk.home, 'modules', 'java.base')
    if exists(exploded_java_base_module):
        mx.abort('Cannot derive a new JDK from ' + jdk.home + ' since it appears to be a developer build with exploded modules')

    jimage = join(jdk.home, 'lib', 'modules')
    jmods_dir = join(jdk.home, 'jmods')
    if not isfile(jimage):
        mx.abort('Cannot derive a new JDK from ' + jdk.home + ' since ' + jimage + ' is missing or is not an ordinary file')
    if not isdir(jmods_dir):
        mx.abort('Cannot derive a new JDK from ' + jdk.home + ' since ' + jmods_dir + ' is missing or is not a directory')

    jdk_modules = {jmd.name : jmd for jmd in jdk.get_modules()}
    modules = [as_java_module(dist, jdk) for dist in module_dists]
    all_module_names = frozenset(list(jdk_modules.keys()) + [m.name for m in modules])

    # Read hashes stored in java.base (the only module in the JDK where hashes are stored)
    out = mx.LinesOutputCapture()
    mx.run([jdk.exe_path('jmod'), 'describe', jdk_modules['java.base'].get_jmod_path()], out=out)
    lines = out.lines
    hashes = {}
    for line in lines:
        if line.startswith('hashes'):
            parts = line.split()
            assert len(parts) == 4, 'expected hashes line to have 4 fields, got {} fields: {}'.format(len(parts), line)
            _, module_name, algorithm, hash_value = parts
            hashes[module_name] = (algorithm, hash_value)

    build_dir = mx.ensure_dir_exists(join(dst_jdk_dir + ".build"))
    try:
        # Handle targets of qualified exports that are not present in `modules`
        target_requires = {}
        for jmd in modules:
            for targets in jmd.exports.values():
                for target in targets:
                    if target not in all_module_names and target not in hashes:
                        target_requires.setdefault(target, set()).add(jmd.name)
        if target_requires and missing_export_target_action is not None:
            if missing_export_target_action == 'error':
                mx.abort('Target(s) of qualified exports cannot be resolved: ' + '.'.join(target_requires.keys()))
            assert missing_export_target_action == 'create', 'invalid value for missing_export_target_action: ' + str(missing_export_target_action)

            extra_modules = []
            for name, requires in target_requires.items():
                module_jar = join(build_dir, name + '.jar')
                jmd = JavaModuleDescriptor(name, {}, requires={module : [] for module in requires}, uses=set(), provides={}, jarpath=module_jar)
                extra_modules.append(jmd)
                module_build_dir = mx.ensure_dir_exists(join(build_dir, name))
                module_info_java = join(module_build_dir, 'module-info.java')
                module_info_class = join(module_build_dir, 'module-info.class')
                with open(module_info_java, 'w') as fp:
                    print(jmd.as_module_info(), file=fp)
                mx.run([jdk.javac, '-d', module_build_dir, \
                        '--limit-modules=java.base,' + ','.join(jmd.requires.keys()), \
                        '--module-path=' + os.pathsep.join((m.jarpath for m in modules)), \
                        module_info_java])
                with ZipFile(module_jar, 'w') as zf:
                    zf.write(module_info_class, basename(module_info_class))
                if exists(jmd.get_jmod_path()):
                    os.remove(jmd.get_jmod_path())
                mx.run([jdk.javac.replace('javac', 'jmod'), 'create', '--class-path=' + module_build_dir, jmd.get_jmod_path()])

            modules.extend(extra_modules)
            all_module_names = frozenset(list(jdk_modules.keys()) + [m.name for m in modules])

        # Extract src.zip from source JDK
        jdk_src_zip = join(jdk.home, 'lib', 'src.zip')
        dst_src_zip_contents = {}
        if isfile(jdk_src_zip):
            mx.logv('[Extracting ' + jdk_src_zip + ']')
            with ZipFile(jdk_src_zip, 'r') as zf:
                for name in zf.namelist():
                    if not name.endswith('/'):
                        dst_src_zip_contents[name] = zf.read(name)
        else:
            mx.warn("'{}' does not exist or is not a file".format(jdk_src_zip))

        for jmd in modules:
            # Remove existing sources for all the modules that we include
            dst_src_zip_contents = {key : dst_src_zip_contents[key] for key in dst_src_zip_contents if not key.startswith(jmd.name)}

            if with_source(jmd.dist):
                # Add the sources that we can share.
                # Extract module sources
                jmd_src_zip = jmd.jarpath[0:-len('.jar')] + '.src.zip'
                if isfile(jmd_src_zip):
                    mx.logv('[Extracting ' + jmd_src_zip + ']')
                    with ZipFile(jmd_src_zip, 'r') as zf:
                        for name in zf.namelist():
                            if not name.endswith('/'):
                                dst_src_zip_contents[jmd.name + '/' + name] = zf.read(name)

                # Add module-info.java to sources
                dst_src_zip_contents[jmd.name + '/module-info.java'] = jmd.as_module_info(extras_as_comments=False)

        # Now build the new JDK image with jlink
        jlink = [jdk.javac.replace('javac', 'jlink')]

        if jdk_enables_jvmci_by_default(jdk):
            # On JDK 9+, +EnableJVMCI forces jdk.internal.vm.ci to be in the root set
            jlink.append('-J-XX:-EnableJVMCI')
        if root_module_names is not None:
            missing = frozenset(root_module_names) - all_module_names
            if missing:
                mx.abort('Invalid module(s): {}.\nAvailable modules: {}'.format(','.join(missing), ','.join(sorted(all_module_names))))
            jlink.append('--add-modules=' + ','.join(root_module_names))
        else:
            jlink.append('--add-modules=' + ','.join(sorted(all_module_names)))

        module_path = jmods_dir
        if modules:
            module_path = os.pathsep.join((m.get_jmod_path(respect_stripping=True) for m in modules)) + os.pathsep + module_path
        jlink.append('--module-path=' + module_path)
        jlink.append('--output=' + dst_jdk_dir)

        # These options are inspired by how OpenJDK runs jlink to produce the final runtime image.
        jlink.extend(['-J-XX:+UseSerialGC', '-J-Xms32M', '-J-Xmx512M', '-J-XX:TieredStopAtLevel=1'])
        jlink.append('-J-Dlink.debug=true')
        jlink.append('--dedup-legal-notices=error-if-not-same-content')
        jlink.append('--keep-packaged-modules=' + join(dst_jdk_dir, 'jmods'))

        # TODO: investigate the options below used by OpenJDK to see if they should be used:
        # --release-info: this allow extra properties to be written to the <jdk>/release file
        # --order-resources: specifies order of resources in generated lib/modules file.
        #       This is apparently not so important if a CDS archive is available.
        # --generate-jli-classes: pre-generates a set of java.lang.invoke classes.
        #       See https://github.com/openjdk/jdk/blob/master/make/GenerateLinkOptData.gmk
        mx.logv('[Creating JDK image]')
        mx.run(jlink)

        dst_src_zip = join(dst_jdk_dir, 'lib', 'src.zip')
        mx.logv('[Creating ' + dst_src_zip + ']')
        with ZipFile(dst_src_zip, 'w', compression=ZIP_DEFLATED, allowZip64=True) as zf:
            for name, contents in sorted(dst_src_zip_contents.items()):
                zf.writestr(name, contents)

        mx.logv('[Copying static libraries]')
        lib_prefix = mx.add_lib_prefix('')
        lib_suffix = '.lib' if mx.is_windows() else '.a'
        lib_directory = join(jdk.home, 'lib')
        dst_lib_directory = join(dst_jdk_dir, 'lib')
        for f in os.listdir(lib_directory):
            if f.startswith(lib_prefix) and f.endswith(lib_suffix):
                lib_path = join(lib_directory, f)
                if isfile(lib_path):
                    shutil.copy2(lib_path, dst_lib_directory)

        # Build the list of modules whose classes might have annotations
        # to be processed by native-image (GR-15192).
        with open(join(dst_jdk_dir, 'lib', 'native-image-modules.list'), 'w') as fp:
            print('# Modules whose classes might have annotations processed by native-image', file=fp)
            for m in modules:
                print(m.name, file=fp)

    finally:
        if not mx.get_opts().verbose:
            # Preserve build directory so that javac command can be re-executed
            # by cutting and pasting verbose output.
            shutil.rmtree(build_dir)

    # Create CDS archive (https://openjdk.java.net/jeps/341).
    out = mx.OutputCapture()
    mx.logv('[Creating CDS shared archive]')
    if mx.run([mx.exe_suffix(join(dst_jdk_dir, 'bin', 'java')), '-Xshare:dump', '-Xmx128M', '-Xms128M'], out=out, err=out, nonZeroIsFatal=False) != 0:
        mx.log(out.data)
        mx.abort('Error generating CDS shared archive')
Beispiel #37
0
    def __init__(self,
                 suite,
                 name,
                 short_name,
                 license_files,
                 third_party_license_files,
                 jar_distributions=None,
                 builder_jar_distributions=None,
                 support_distributions=None,
                 support_headers_distributions=None,
                 support_libraries_distributions=None,
                 dir_name=None,
                 launcher_configs=None,
                 library_configs=None,
                 provided_executables=None,
                 polyglot_lib_build_args=None,
                 polyglot_lib_jar_dependencies=None,
                 polyglot_lib_build_dependencies=None,
                 has_polyglot_lib_entrypoints=False,
                 boot_jars=None,
                 jvmci_parent_jars=None,
                 priority=None,
                 installable=False,
                 post_install_msg=None,
                 installable_id=None,
                 dependencies=None):
        """
        :param suite mx.Suite: the suite this component belongs to
        :type name: str
        :param str short_name: a short, unique name for this component
        :param str | None | False dir_name: the directory name in which this component lives. If `None`, the `short_name` is used. If `False`, files are copied to the root-dir for the component type.
        :param installable: Produce a distribution installable via `gu`
        :param post_install_msg: Post-installation message to be printed
        :param list[str] dependencies: a list of component names
        :type license_files: list[str]
        :type third_party_license_files: list[str]
        :type provided_executables: list[str]
        :type polyglot_lib_build_args: list[str]
        :type polyglot_lib_jar_dependencies: list[str]
        :type polyglot_lib_build_dependencies: list[str]
        :type has_polyglot_lib_entrypoints: bool
        :type boot_jars: list[str]
        :type jvmci_parent_jars: list[str]
        :type launcher_configs: list[LauncherConfig]
        :type library_configs: list[LibraryConfig]
        :type jar_distributions: list[str]
        :type builder_jar_distributions: list[str]
        :type support_distributions: list[str]
        :type support_headers_distributions: list[str]
        :type support_libraries_distributions: list[str]
        :param int priority: priority with a higher value means higher priority
        :type installable: bool
        :type installable_id: str
        :type post_install_msg: str
        """
        if dependencies is None:
            mx.logv('Component {} does not specify dependencies'.format(name))

        self.suite = suite
        self.name = name
        self.short_name = short_name
        self.dir_name = dir_name if dir_name is not None else short_name
        self.license_files = license_files
        self.third_party_license_files = third_party_license_files
        self.dependency_names = dependencies or []
        self.provided_executables = provided_executables or []
        self.polyglot_lib_build_args = polyglot_lib_build_args or []
        self.polyglot_lib_jar_dependencies = polyglot_lib_jar_dependencies or []
        self.polyglot_lib_build_dependencies = polyglot_lib_build_dependencies or []
        self.has_polyglot_lib_entrypoints = has_polyglot_lib_entrypoints
        self.boot_jars = boot_jars or []
        self.jvmci_parent_jars = jvmci_parent_jars or []
        self.jar_distributions = jar_distributions or []
        self.builder_jar_distributions = builder_jar_distributions or []
        self.support_distributions = support_distributions or []
        self.support_headers_distributions = support_headers_distributions or []
        self.support_libraries_distributions = support_libraries_distributions or []
        self.priority = priority or 0
        self.launcher_configs = launcher_configs or []
        self.library_configs = library_configs or []
        self.installable = installable
        self.post_install_msg = post_install_msg
        self.installable_id = installable_id or self.dir_name

        assert isinstance(self.jar_distributions, list)
        assert isinstance(self.builder_jar_distributions, list)
        assert isinstance(self.support_distributions, list)
        assert isinstance(self.support_headers_distributions, list)
        assert isinstance(self.support_libraries_distributions, list)
        assert isinstance(self.license_files, list)
        assert isinstance(self.third_party_license_files, list)
        assert isinstance(self.provided_executables, list)
        assert isinstance(self.polyglot_lib_build_args, list)
        assert isinstance(self.polyglot_lib_jar_dependencies, list)
        assert isinstance(self.polyglot_lib_build_dependencies, list)
        assert isinstance(self.boot_jars, list)
        assert isinstance(self.jvmci_parent_jars, list)
        assert isinstance(self.launcher_configs, list)
        assert isinstance(self.library_configs, list)
Beispiel #38
0
def truffle_language_ensure(language_flag,
                            version=None,
                            native_image_root=None):
    '''
    Ensures that we have a valid suite for the given language_flag, by downloading a binary if necessary
    and providing the suite distribution artifacts in the native-image directory hierachy (via symlinks).
    :param language_flag: native-image language_flag whose truffle-language we want to use
    :param version: if not specified and no TRUFFLE_<LANG>_VERSION set latest binary deployed master revision gets downloaded
    :param native_image_root: the native_image_root directory where the the artifacts get installed to
    :return: language suite for the given language_flag
    '''
    if not native_image_root:
        native_image_root = suite_native_image_root()

    version_env_var = 'TRUFFLE_' + language_flag.upper() + '_VERSION'
    if not version and os.environ.has_key(version_env_var):
        version = os.environ[version_env_var]

    if language_flag not in flag_suitename_map:
        mx.abort('No truffle-language uses language_flag \'' + language_flag +
                 '\'')
    if language_flag in session_language:
        language_suite = session_language[language_flag]
        mx.log('Reusing ' + language_flag + '.version=' +
               str(language_suite.version()))
        return language_suite

    language_suite_name = flag_suitename_map[language_flag][0]

    # Accessing truffle_language as source suite (--dynamicimports) has priority over binary suite import
    language_suite = suite.import_suite(language_suite_name)

    if not language_suite:
        # Get the truffle_language suite via binary suite import

        urlinfos = [
            mx.SuiteImportURLInfo(
                mx_urlrewrites.rewriteurl(
                    'https://curio.ssw.jku.at/nexus/content/repositories/snapshots'
                ), 'binary', mx.vc_system('binary'))
        ]

        if not version:
            # If no specific version requested use binary import of last recently deployed master version
            version = 'git-bref:binary'
            urlinfos.append(
                mx.SuiteImportURLInfo(
                    mx_urlrewrites.rewriteurl(
                        'https://github.com/graalvm/{0}.git'.format(
                            language_suite_name)), 'source',
                    mx.vc_system('git')))

        try:
            language_suite = suite.import_suite(language_suite_name,
                                                version=version,
                                                urlinfos=urlinfos,
                                                kind=None)
        except (urllib2.URLError, SystemExit):
            language_suite = suite.import_suite(language_suite_name)
            if language_suite:
                if version and session_language[
                        language_flag] and session_language[
                            language_flag].version() != version:
                    mx.abort('Cannot switch to ' + language_flag +
                             '.version=' + str(version) +
                             ' without maven access.')
                else:
                    mx.log('No maven access. Using already downloaded ' +
                           language_suite_name + ' binary suite.')
            else:
                mx.abort('No maven access and no local copy of ' +
                         language_suite_name + ' binary suite available.')

    if not language_suite:
        mx.abort('Binary suite not found and no local copy of ' +
                 language_suite_name + ' available.')

    language_dir = join('languages', language_flag)

    language_suite_depnames = flag_suitename_map[language_flag][1]
    language_deps = [
        dep for dep in language_suite.dists + language_suite.libs
        if dep.name in language_suite_depnames
    ]
    native_image_layout(language_deps, language_dir, native_image_root)

    language_suite_nativedistnames = flag_suitename_map[language_flag][2]
    language_nativedists = [
        dist for dist in language_suite.dists
        if dist.name in language_suite_nativedistnames
    ]
    native_image_extract(language_nativedists, language_dir, native_image_root)

    option_properties = join(language_suite.mxDir, 'native-image.properties')
    target_path = join(native_image_root, language_dir,
                       'native-image.properties')
    if islink(target_path):
        os.remove(target_path)
    if exists(option_properties):
        mx.logv('Add symlink to ' + str(option_properties))
        relsymlink(option_properties, target_path)
    else:
        native_image_option_properties('languages', language_flag,
                                       native_image_root)

    session_language[language_flag] = language_suite
    return language_suite
Beispiel #39
0
def bootstrap_native_image(native_image_root, svmDistribution,
                           graalDistribution, librarySupportDistribution):
    if not allow_native_image_build:
        mx.logv(
            'Detected building with ejc + --warning-as-error -> suppress bootstrap_native_image'
        )
        return

    bootstrap_command = list(GRAAL_COMPILER_FLAGS)
    bootstrap_command += locale_US_args()
    bootstrap_command += substratevm_version_args()
    bootstrap_command += ['-Dgraalvm.version=dev']

    builder_classpath = classpath(svmDistribution)
    imagecp_classpath = classpath(svmDistribution + ['substratevm:SVM_DRIVER'])
    bootstrap_command += [
        '-cp', builder_classpath,
        'com.oracle.svm.hosted.NativeImageGeneratorRunner', '-imagecp',
        imagecp_classpath, '-H:CLibraryPath=' + clibrary_libpath()
    ]

    bootstrap_command += [
        '-H:Path=' + dirname(native_image_path(native_image_root)),
        '-H:Class=com.oracle.svm.driver.NativeImage',
        '-H:Name=' + basename(native_image_path(native_image_root)),
        '-H:-TruffleFeature', '-H:-ParseRuntimeOptions',
        '-H:ReflectionConfigurationResources=com/oracle/svm/driver/ReflectionConfiguration.json'
    ]

    if mx._opts.strip_jars:
        bootstrap_command += ['-H:-VerifyNamingConventions']

    run_java(bootstrap_command)
    mx.logv('Built ' + native_image_path(native_image_root))

    def native_image_layout_dists(subdir, dists):
        native_image_dists = [mx.dependency(dist_name) for dist_name in dists]
        native_image_layout(native_image_dists, subdir, native_image_root)

    # Create native-image layout for sdk parts
    native_image_layout_dists(join('lib', 'boot'), ['sdk:GRAAL_SDK'])
    native_image_layout_dists(join('lib', 'graalvm'), ['sdk:LAUNCHER_COMMON'])

    # Create native-image layout for compiler & jvmci parts
    native_image_layout_dists(join('lib', 'jvmci'), graalDistribution)
    jdk_config = mx.get_jdk()
    jvmci_path = join(jdk_config.home, 'jre', 'lib', 'jvmci')
    for symlink_name in os.listdir(jvmci_path):
        relsymlink(join(jvmci_path, symlink_name),
                   join(native_image_root, 'lib', 'jvmci', symlink_name))

    # Create native-image layout for truffle parts
    native_image_layout_dists(join('lib', 'truffle'), ['truffle:TRUFFLE_API'])

    # Create native-image layout for tools parts
    native_image_layout_dists(join('tools', 'junit', 'builder'),
                              ['mx:JUNIT_TOOL', 'JUNIT', 'HAMCREST'])
    native_image_option_properties('tools', 'junit', native_image_root)
    native_image_option_properties('tools', 'truffle', native_image_root)
    native_image_layout_dists(join('tools', 'nfi', 'builder'),
                              ['truffle:TRUFFLE_NFI'])
    native_image_option_properties('tools', 'nfi', native_image_root)

    # Create native-image layout for svm parts
    svm_subdir = join('lib', 'svm')
    native_image_layout_dists(svm_subdir, librarySupportDistribution)
    native_image_layout_dists(
        join(svm_subdir, 'builder'),
        svmDistribution + ['substratevm:POINTSTO', 'substratevm:OBJECTFILE'])
    for clibrary_path in clibrary_paths():
        copy_tree(clibrary_path,
                  join(native_image_root, join(svm_subdir, 'clibraries')))

    # Finally create symlink to native-image in svm_suite().dir
    native_image_symlink_path = join(
        svm_suite().dir, basename(native_image_path(native_image_root)))
    if islink(native_image_symlink_path):
        os.remove(native_image_symlink_path)
    relsymlink(native_image_path(native_image_root), native_image_symlink_path)
Beispiel #40
0
def _setEnvVar(name, val, env=None):
    _env = env or os.environ
    if val:
        mx.logv('Setting environment variable %s=%s' % (name, val))
        _env[name] = val
Beispiel #41
0
def truffle_language_ensure(language_flag, version=None, native_image_root=None, early_exit=False, extract=True, debug_gr_8964=False):
    """
    Ensures that we have a valid suite for the given language_flag, by downloading a binary if necessary
    and providing the suite distribution artifacts in the native-image directory hierachy (via symlinks).
    :param language_flag: native-image language_flag whose truffle-language we want to use
    :param version: if not specified and no TRUFFLE_<LANG>_VERSION set latest binary deployed master revision gets downloaded
    :param native_image_root: the native_image_root directory where the the artifacts get installed to
    :return: language suite for the given language_flag
    """
    if not native_image_root:
        native_image_root = suite_native_image_root()

    version_env_var = 'TRUFFLE_' + language_flag.upper() + '_VERSION'
    if not version and os.environ.has_key(version_env_var):
        version = os.environ[version_env_var]

    if language_flag not in flag_suitename_map:
        mx.abort('No truffle-language uses language_flag \'' + language_flag + '\'')

    language_dir = join('languages', language_flag)
    if early_exit and exists(join(native_image_root, language_dir)):
        mx.logv('Early exit mode: Language subdir \'' + language_flag + '\' exists. Skip suite.import_suite.')
        return None

    language_entry = flag_suitename_map[language_flag]

    language_suite_name = language_entry[0]
    language_repo_name = language_entry[3] if len(language_entry) > 3 else None

    urlinfos = [
        mx.SuiteImportURLInfo(mx_urlrewrites.rewriteurl('https://curio.ssw.jku.at/nexus/content/repositories/snapshots'),
                              'binary',
                              mx.vc_system('binary'))
    ]

    failure_warning = None
    if not version and not mx.suite(language_suite_name, fatalIfMissing=False):
        # If no specific version requested use binary import of last recently deployed master version
        repo_suite_name = language_repo_name if language_repo_name else language_suite_name
        repo_url = mx_urlrewrites.rewriteurl('https://github.com/graalvm/{0}.git'.format(repo_suite_name))
        version = mx.SuiteImport.resolve_git_branchref(repo_url, 'binary', abortOnError=False)
        if not version:
            failure_warning = 'Resolving \'binary\' against ' + repo_url + ' failed'

    language_suite = suite.import_suite(
        language_suite_name,
        version=version,
        urlinfos=urlinfos,
        kind=None,
        in_subdir=bool(language_repo_name)
    )

    if not language_suite:
        if failure_warning:
            mx.warn(failure_warning)
        mx.abort('Binary suite not found and no local copy of ' + language_suite_name + ' available.')

    if not extract:
        if not exists(join(native_image_root, language_dir)):
            mx.abort('Language subdir \'' + language_flag + '\' should already exist with extract=False')
        return language_suite

    language_suite_depnames = language_entry[1]
    language_deps = language_suite.dists + language_suite.libs
    language_deps = [dep for dep in language_deps if dep.name in language_suite_depnames]
    native_image_layout(language_deps, language_dir, native_image_root, debug_gr_8964=debug_gr_8964)

    language_suite_nativedistnames = language_entry[2]
    language_nativedists = [dist for dist in language_suite.dists if dist.name in language_suite_nativedistnames]
    native_image_extract(language_nativedists, language_dir, native_image_root)

    option_properties = join(language_suite.mxDir, 'native-image.properties')
    target_path = remove_existing_symlink(join(native_image_root, language_dir, 'native-image.properties'))
    if exists(option_properties):
        if not exists(target_path):
            mx.logv('Add symlink to ' + str(option_properties))
            symlink_or_copy(option_properties, target_path, debug_gr_8964=debug_gr_8964)
    else:
        native_image_option_properties('languages', language_flag, native_image_root)
    return language_suite
 def _replace_host_vm(key):
     host_vm = dims.get("host-vm")
     if host_vm and host_vm.startswith(key):
         dims['host-vm'] = key
         mx.logv("[DEBUG] replace 'host-vm': '{key}-python' -> '{key}'".format(key=key))
Beispiel #43
0
def truffle_language_ensure(language_flag,
                            version=None,
                            native_image_root=None,
                            early_exit=False,
                            extract=True):
    """
    Ensures that we have a valid suite for the given language_flag, by downloading a binary if necessary
    and providing the suite distribution artifacts in the native-image directory hierachy (via symlinks).
    :param language_flag: native-image language_flag whose truffle-language we want to use
    :param version: if not specified and no TRUFFLE_<LANG>_VERSION set latest binary deployed master revision gets downloaded
    :param native_image_root: the native_image_root directory where the the artifacts get installed to
    :return: language suite for the given language_flag
    """
    if not native_image_root:
        native_image_root = suite_native_image_root()

    version_env_var = 'TRUFFLE_' + language_flag.upper() + '_VERSION'
    if not version and os.environ.has_key(version_env_var):
        version = os.environ[version_env_var]

    if language_flag not in flag_suitename_map:
        mx.abort('No truffle-language uses language_flag \'' + language_flag +
                 '\'')

    language_dir = join('languages', language_flag)
    if early_exit and exists(join(native_image_root, language_dir)):
        mx.logv('Early exit mode: Language subdir \'' + language_flag +
                '\' exists. Skip suite.import_suite.')
        return None

    language_entry = flag_suitename_map[language_flag]

    language_suite_name = language_entry[0]
    language_repo_name = language_entry[3] if len(language_entry) > 3 else None

    urlinfos = [
        mx.SuiteImportURLInfo(
            mx_urlrewrites.rewriteurl(
                'https://curio.ssw.jku.at/nexus/content/repositories/snapshots'
            ), 'binary', mx.vc_system('binary'))
    ]

    failure_warning = None
    if not version and not mx.suite(language_suite_name, fatalIfMissing=False):
        # If no specific version requested use binary import of last recently deployed master version
        repo_suite_name = language_repo_name if language_repo_name else language_suite_name
        repo_url = mx_urlrewrites.rewriteurl(
            'https://github.com/graalvm/{0}.git'.format(repo_suite_name))
        version = mx.SuiteImport.resolve_git_branchref(repo_url,
                                                       'binary',
                                                       abortOnError=False)
        if not version:
            failure_warning = 'Resolving \'binary\' against ' + repo_url + ' failed'

    language_suite = suite.import_suite(language_suite_name,
                                        version=version,
                                        urlinfos=urlinfos,
                                        kind=None,
                                        in_subdir=bool(language_repo_name))

    if not language_suite:
        if failure_warning:
            mx.warn(failure_warning)
        mx.abort('Binary suite not found and no local copy of ' +
                 language_suite_name + ' available.')

    if not extract:
        if not exists(join(native_image_root, language_dir)):
            mx.abort('Language subdir \'' + language_flag +
                     '\' should already exist with extract=False')
        return language_suite

    language_suite_depnames = language_entry[1]
    language_deps = language_suite.dists + language_suite.libs
    language_deps = [
        dep for dep in language_deps if dep.name in language_suite_depnames
    ]
    native_image_layout(language_deps, language_dir, native_image_root)

    language_suite_nativedistnames = language_entry[2]
    language_nativedists = [
        dist for dist in language_suite.dists
        if dist.name in language_suite_nativedistnames
    ]
    native_image_extract(language_nativedists, language_dir, native_image_root)

    option_properties = join(language_suite.mxDir, 'native-image.properties')
    target_path = remove_existing_symlink(
        join(native_image_root, language_dir, 'native-image.properties'))
    if exists(option_properties):
        if not exists(target_path):
            mx.logv('Add symlink to ' + str(option_properties))
            symlink_or_copy(option_properties, target_path)
    else:
        native_image_option_properties('languages', language_flag,
                                       native_image_root)
    return language_suite
Beispiel #44
0
def find_top(*args, **kwargs):
    full_args = _pkgtest_args(args)
    mx.logv(["find_top"] + full_args)
    return pkgtest_load().find_top(args)
Beispiel #45
0
    def build(self):
        source_dir = self.subject.getSourceDir()
        output_dir = self.subject.getOutputDir()
        if not emcc_dir:
            mx.abort("No EMCC_DIR specified - the source programs will not be compiled to .wasm.")
        emcc_cmd = os.path.join(emcc_dir, "emcc")
        gcc_cmd = os.path.join(gcc_dir, "gcc")
        if mx.run([emcc_cmd, "-v"], nonZeroIsFatal=False) != 0:
            mx.abort("Could not check the emcc version.")
        if mx.run([gcc_cmd, "--version"], nonZeroIsFatal=False) != 0:
            mx.abort("Could not check the gcc version.")
        if not wabt_dir:
            mx.abort("Set WABT_DIR if you want the binary to include .wat files.")
        mx.log("Building files from the source dir: " + source_dir)
        cc_flags = ["-O3", "-g2"]
        include_flags = []
        disable_test_api_flags = ["-DDISABLE_TEST_API"]
        if hasattr(self.project, "includeset"):
            include_flags = ["-I", os.path.join(_suite.dir, "includes", self.project.includeset)]
        emcc_flags = cc_flags
        if self.project.isBenchmarkProject():
            emcc_flags = emcc_flags + ["-s", "EXPORTED_FUNCTIONS=" + str(benchmark_methods).replace("'", "\"") + ""]
        subdir_program_names = defaultdict(lambda: [])
        for root, filename in self.subject.getProgramSources():
            subdir = os.path.relpath(root, self.subject.getSourceDir())
            mx.ensure_dir_exists(os.path.join(output_dir, subdir))

            basename = remove_extension(filename)
            source_path = os.path.join(root, filename)
            output_wasm_path = os.path.join(output_dir, subdir, basename + ".wasm")
            timestampedSource = mx.TimeStampFile(source_path)
            timestampedOutput = mx.TimeStampFile(output_wasm_path)
            mustRebuild = timestampedSource.isNewerThan(timestampedOutput) or not timestampedOutput.exists()

            # Step 1: build the .wasm binary.
            if mustRebuild:
                if filename.endswith(".c"):
                    # Step 1a: compile with the JS file, and store as files for running Node, if necessary.
                    output_js_path = os.path.join(output_dir, subdir, basename + ".js")
                    build_cmd_line = [emcc_cmd] + emcc_flags + disable_test_api_flags + [source_path, "-o", output_js_path] + include_flags
                    if mx.run(build_cmd_line, nonZeroIsFatal=False) != 0:
                        mx.abort("Could not build the JS output of " + filename + " with emcc.")
                    if self.subject.isBenchmarkProject():
                        node_dir = os.path.join(output_dir, subdir, NODE_BENCH_DIR)
                        mx.ensure_dir_exists(node_dir)
                        shutil.copyfile(output_js_path, os.path.join(node_dir, basename + ".js"))
                        shutil.copyfile(output_wasm_path, os.path.join(node_dir, basename + ".wasm"))

                    # Step 1b: extract the relevant information out of the JS file, and record it into an initialization file.
                    init_info = self.extractInitialization(output_js_path)
                    with open(os.path.join(output_dir, subdir, basename + ".init"), "w") as f:
                        f.write(init_info)

                    # Step 1c: compile to just a .wasm file, to avoid name mangling.
                    build_cmd_line = [emcc_cmd] + emcc_flags + ["-s", "ERROR_ON_UNDEFINED_SYMBOLS=0"] + [source_path, "-o", output_wasm_path] + include_flags
                    if mx.run(build_cmd_line, nonZeroIsFatal=False) != 0:
                        mx.abort("Could not build the wasm-only output of " + filename + " with emcc.")
                elif filename.endswith(".wat"):
                    # Step 1: compile the .wat file to .wasm.
                    wat2wasm_cmd = os.path.join(wabt_dir, "wat2wasm")
                    build_cmd_line = [wat2wasm_cmd, "-o", output_wasm_path, source_path]
                    if mx.run(build_cmd_line, nonZeroIsFatal=False) != 0:
                        mx.abort("Could not translate " + filename + " to binary format.")
            else:
                mx.logv("skipping, file is up-to-date: " + source_path)

            # Step 2: copy the result file if it exists.
            result_path = os.path.join(root, basename + ".result")
            if os.path.isfile(result_path):
                result_output_path = os.path.join(output_dir, subdir, basename + ".result")
                shutil.copyfile(result_path, result_output_path)

            # Step 3: copy the opts file if it exists.
            opts_path = os.path.join(root, basename + ".opts")
            if os.path.isfile(opts_path):
                opts_output_path = os.path.join(output_dir, subdir, basename + ".opts")
                shutil.copyfile(opts_path, opts_output_path)

            output_wat_path = os.path.join(output_dir, subdir, basename + ".wat")
            if mustRebuild:
                if filename.endswith(".c"):
                    # Step 4: produce the .wat files, for easier debugging.
                    wasm2wat_cmd = os.path.join(wabt_dir, "wasm2wat")
                    if mx.run([wasm2wat_cmd, "-o", output_wat_path, output_wasm_path], nonZeroIsFatal=False) != 0:
                        mx.abort("Could not compile .wat file for " + filename)
                elif filename.endswith(".wat"):
                    # Step 4: copy the .wat file, for easier debugging.
                    wat_path = os.path.join(root, basename + ".wat")
                    shutil.copyfile(wat_path, output_wat_path)

            # Step 5: if this is a benchmark project, create native binaries too.
            if mustRebuild:
                mx.ensure_dir_exists(os.path.join(output_dir, subdir, NATIVE_BENCH_DIR))
                if filename.endswith(".c"):
                    output_path = os.path.join(output_dir, subdir, NATIVE_BENCH_DIR, mx.exe_suffix(basename))
                    link_flags = ["-lm"]
                    gcc_cmd_line = [gcc_cmd] + cc_flags + disable_test_api_flags + [source_path, "-o", output_path] + include_flags + link_flags
                    if mx.run(gcc_cmd_line, nonZeroIsFatal=False) != 0:
                        mx.abort("Could not build the native binary of " + filename + ".")
                    os.chmod(output_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
                elif filename.endswith(".wat"):
                    mx.warn("The .wat files are not translated to native binaries: " + filename)

            # Remember the source name.
            subdir_program_names[subdir].append(basename)
        for subdir in subdir_program_names:
            with open(os.path.join(output_dir, subdir, "wasm_test_index"), "w") as f:
                for name in subdir_program_names[subdir]:
                    f.write(name)
                    f.write("\n")
Beispiel #46
0
    def build(self):
        pre_ts = GraalNodeJsBuildTask._get_newest_ts(self.subject.getResults(),
                                                     fatalIfMissing=False)

        build_env = os.environ.copy()
        _setEnvVar(
            'PATH', '%s%s%s' %
            (join(_suite.mxDir, 'python2'), pathsep, build_env['PATH']),
            build_env)

        debug = ['--debug'] if self._debug_mode else []
        shared_library = ['--enable-shared-library'] if hasattr(
            self.args, 'sharedlibrary') and self.args.sharedlibrary else []

        newest_config_file_ts = GraalNodeJsBuildTask._get_newest_ts(
            _config_files, fatalIfMissing=True)
        newest_generated_config_file_ts = GraalNodeJsBuildTask._get_newest_ts(
            _generated_config_files, fatalIfMissing=False)
        # Lazily generate config files only if `configure` and `configure.py` are older than the files they generate.
        # If we don't do this, the `Makefile` always considers `config.gypi` out of date, triggering a second, unnecessary configure.
        lazy_generator = ['--lazy-generator'
                          ] if newest_generated_config_file_ts.isNewerThan(
                              newest_config_file_ts) else []

        if _is_windows:
            processDevkitRoot(env=build_env)
            _setEnvVar(
                'PATH',
                pathsep.join([build_env['PATH']] + [
                    mx.library(lib_name).get_path(True)
                    for lib_name in ('NASM', 'NINJA')
                ]), build_env)
            extra_flags = [
                '--ninja', '--dest-cpu=x64', '--without-etw',
                '--without-snapshot'
            ]
        else:
            extra_flags = []

        _mxrun(python_cmd() + [
            join(_suite.dir,
                 'configure'), '--partly-static', '--without-dtrace',
            '--without-snapshot', '--without-node-snapshot',
            '--without-node-code-cache', '--java-home',
            _java_home(forBuild=True)
        ] + debug + shared_library + lazy_generator + extra_flags,
               cwd=_suite.dir,
               print_cmd=True,
               env=build_env)

        quiet_build = mx.is_continuous_integration(
        ) and not mx.get_opts().verbose
        if _is_windows:
            verbose = ['-v'] if mx.get_opts().verbose else []
            # The custom env is not used to resolve the location of the executable
            _mxrun([join(mx.library('NINJA').get_path(True), 'ninja.exe')] +
                   verbose +
                   ['-j%d' % self.parallelism, '-C', self._build_dir],
                   print_cmd=True,
                   quiet_if_successful=quiet_build,
                   env=build_env)
        else:
            verbose = 'V={}'.format('1' if mx.get_opts().verbose else '')
            _mxrun([mx.gmake_cmd(),
                    '-j%d' % self.parallelism, verbose],
                   cwd=_suite.dir,
                   print_cmd=True,
                   quiet_if_successful=quiet_build,
                   env=build_env)

        # put headers for native modules into out/headers
        _setEnvVar('HEADERS_ONLY', '1', build_env)
        _mxrun(python_cmd() + [
            join('tools', 'install.py'), 'install',
            join('out', 'headers'), sep
        ],
               quiet_if_successful=not mx.get_opts().verbose,
               env=build_env)

        post_ts = GraalNodeJsBuildTask._get_newest_ts(
            self.subject.getResults(), fatalIfMissing=True)
        mx.logv(
            'Newest time-stamp before building: {}\nNewest time-stamp after building: {}\nHas built? {}'
            .format(pre_ts, post_ts, post_ts.isNewerThan(pre_ts)))
        built = post_ts.isNewerThan(pre_ts)
        if built and _current_os == 'darwin':
            nodePath = join(self._build_dir, 'node')
            _mxrun([
                'install_name_tool', '-add_rpath',
                join(_java_home(), 'jre', 'lib'), '-add_rpath',
                join(_java_home(), 'lib'), nodePath
            ],
                   print_cmd=True,
                   env=build_env)
        return built
Beispiel #47
0
def testdownstream(suite, repoUrls, relTargetSuiteDir, mxCommands, branch=None):
    """
    Tests a downstream repo against the current working directory state of `suite`.

    :param mx.Suite suite: the suite to test against the downstream repo
    :param list repoUrls: URLs of downstream repos to clone, the first of which is the repo being tested
    :param str relTargetSuiteDir: directory of the downstream suite to test relative to the top level
           directory of the downstream repo being tested
    :param list mxCommands: argument lists for the mx commands run in downstream suite being tested
    :param str branch: name of branch to look for in downstream repo(s)
    """

    assert len(repoUrls) > 0
    workDir = join(suite.get_output_root(), 'testdownstream')

    # A mirror of each suites in the same repo as `suite` is created via copying
    rel_mirror = os.path.relpath(suite.dir, mx.SuiteModel.siblings_dir(suite.dir))
    in_subdir = os.sep in rel_mirror
    suites_in_repo = [suite]
    if in_subdir:
        base = os.path.dirname(suite.dir)
        for e in os.listdir(base):
            candidate = join(base, e)
            if candidate != suite.dir:
                mxDir = mx._is_suite_dir(candidate)
                if mxDir:
                    matches = [s for s in mx.suites() if s.dir == candidate]
                    if len(matches) == 0:
                        suites_in_repo.append(mx.SourceSuite(mxDir, primary=False, load=False))
                    else:
                        suites_in_repo.append(matches[0])

    for suite_in_repo in suites_in_repo:
        if suite_in_repo.vc_dir and suite_in_repo.dir != suite_in_repo.vc_dir:
            mirror = join(workDir, basename(suite_in_repo.vc_dir), suite_in_repo.name)
        else:
            mirror = join(workDir, suite_in_repo.name)
        if exists(mirror):
            shutil.rmtree(mirror)

        output_root = suite_in_repo.get_output_root()
        def ignore_output_root(d, names):
            mx.log('Copying ' + d)
            if d == os.path.dirname(output_root):
                mx.log('Omitting ' + output_root)
                return [os.path.basename(output_root)]
            return []
        shutil.copytree(suite_in_repo.dir, mirror, ignore=ignore_output_root)

    targetDir = None
    for repoUrl in repoUrls:
        # Deduce a target name from the target URL
        url = urlparse(repoUrl)
        targetName = url.path
        if targetName.rfind('/') != -1:
            targetName = targetName[targetName.rfind('/') + 1:]
        if targetName.endswith('.git'):
            targetName = targetName[0:-len('.git')]

        repoWorkDir = join(workDir, targetName)
        git = mx.GitConfig()
        if exists(repoWorkDir):
            git.pull(repoWorkDir)
        else:
            git.clone(repoUrl, repoWorkDir)

        # See if there's a matching (non-master) branch and use it if there is
        if not branch:
            branch = git.git_command(suite.dir, ['rev-parse', '--abbrev-ref', 'HEAD']).strip()
        if branch != 'master':
            git.git_command(repoWorkDir, ['checkout', branch], abortOnError=False)
        if not targetDir:
            targetDir = repoWorkDir

    assert not isabs(relTargetSuiteDir)
    targetSuiteDir = join(targetDir, relTargetSuiteDir)
    assert targetSuiteDir.startswith(targetDir)
    mxpy = None if suite != mx._mx_suite else join(mirror, 'mx.py')
    for command in mxCommands:
        mx.logv('[running "mx ' + ' '.join(command) + '" in ' + targetSuiteDir + ']')
        mx.run_mx(command, targetSuiteDir, mxpy=mxpy)
Beispiel #48
0
def testdownstream(suite, repoUrls, relTargetSuiteDir, mxCommands, branch=None):
    """
    Tests a downstream repo against the current working directory state of `suite`.

    :param mx.Suite suite: the suite to test against the downstream repo
    :param list repoUrls: URLs of downstream repos to clone, the first of which is the repo being tested
    :param str relTargetSuiteDir: directory of the downstream suite to test relative to the top level
           directory of the downstream repo being tested
    :param list mxCommands: argument lists for the mx commands run in downstream suite being tested
    :param str branch: name of branch to look for in downstream repo(s)
    """

    assert len(repoUrls) > 0
    workDir = join(suite.get_output_root(), 'testdownstream')

    # A mirror of the current suite is created with symlinks
    mirror = join(workDir, suite.name)
    if exists(mirror):
        shutil.rmtree(mirror)
    mx.ensure_dir_exists(mirror)
    for f in os.listdir(suite.dir):
        subDir = join(suite.dir, f)
        if subDir == suite.get_output_root():
            continue
        src = join(suite.dir, f)
        dst = join(mirror, f)
        mx.logv('[Creating symlink from {} to {}]'.format(dst, src))
        relsrc = os.path.relpath(src, os.path.dirname(dst))
        os.symlink(relsrc, dst)

    targetDir = None
    for repoUrl in repoUrls:
        # Deduce a target name from the target URL
        url = urlparse(repoUrl)
        targetName = url.path
        if targetName.rfind('/') != -1:
            targetName = targetName[targetName.rfind('/') + 1:]
        if targetName.endswith('.git'):
            targetName = targetName[0:-len('.git')]

        repoWorkDir = join(workDir, targetName)
        git = mx.GitConfig()
        if exists(repoWorkDir):
            git.pull(repoWorkDir)
        else:
            git.clone(repoUrl, repoWorkDir)

        # See if there's a matching (non-master) branch downstream and use it if there is
        if not branch:
            branch = git.git_command(suite.dir, ['rev-parse', '--abbrev-ref', 'HEAD']).strip()
        if branch != 'master':
            git.git_command(repoWorkDir, ['checkout', branch], abortOnError=False)
        if not targetDir:
            targetDir = repoWorkDir

    assert not isabs(relTargetSuiteDir)
    targetSuiteDir = join(targetDir, relTargetSuiteDir)
    assert targetSuiteDir.startswith(targetDir)
    mxpy = None if suite != mx._mx_suite else join(mirror, 'mx.py')
    for command in mxCommands:
        mx.logv('[running "mx ' + ' '.join(command) + '" in ' + targetSuiteDir + ']')
        mx.run_mx(command, targetSuiteDir, mxpy=mxpy)
Beispiel #49
0
def jdkartifactstats(args):
    """show stats about JDK deployed Graal artifacts"""
    artifacts = {}
    jdkDir = get_jvmci_jdk().home

    def _getDeployedJars():
        if JVMCI_VERSION < 9:
            for root, _, filenames in os.walk(join(jdkDir, "jre", "lib")):
                for f in filenames:
                    if f.endswith(".jar") and not f.endswith(".stripped.jar"):
                        yield join(root, f)
        else:
            for jdkDist in jdkDeployedDists:
                dist = jdkDist.dist()
                if isinstance(jdkDist, JvmciJDKDeployedDist):
                    yield dist.path

    for jar in _getDeployedJars():
        f = basename(jar)
        if "truffle" in f:
            if "enterprise" in f:
                artifacts.setdefault("GraalEnterpriseTruffle", []).append(jar)
            else:
                artifacts.setdefault("GraalTruffle", []).append(jar)
        elif "enterprise" in f:
            artifacts.setdefault("GraalEnterprise", []).append(jar)
        elif "jvmci" in f:
            artifacts.setdefault("JVMCI", []).append(jar)
        elif "graal" in f:
            artifacts.setdefault("Graal", []).append(jar)
        else:
            mx.logv("ignored: " + jar)

    print "{:>10}  {:>10}  {:>10}  {}".format("All", "NoVars", "None", "Jar")
    for category in sorted(artifacts.viewkeys()):
        jars = artifacts[category]
        if jars:
            totals = (0, 0, 0)
            print
            for j in jars:
                gSize = os.path.getsize(j)
                stripped = j[: -len(".jar")] + ".stripped.jar"
                mx.run(
                    [
                        mx.get_jdk().pack200,
                        "--repack",
                        "--quiet",
                        "-J-Djava.util.logging.config.file=",
                        "-DLocalVariableTypeTable=strip",
                        "-DLocalVariableTable=strip",
                        stripped,
                        j,
                    ]
                )
                gLinesSourceSize = os.path.getsize(stripped)
                mx.run(
                    [
                        mx.get_jdk().pack200,
                        "--repack",
                        "--quiet",
                        "-J-Djava.util.logging.config.file=",
                        "-G",
                        stripped,
                        j,
                    ]
                )
                gNoneSize = os.path.getsize(stripped)
                os.remove(stripped)
                print "{:10,}  {:10,}  {:10,}  {}:{}".format(gSize, gLinesSourceSize, gNoneSize, category, basename(j))
                t1, t2, t3 = totals
                totals = (t1 + gSize, t2 + gLinesSourceSize, t3 + gNoneSize)
            t1, t2, t3 = totals
            print "{:10,}  {:10,}  {:10,}  {}".format(t1, t2, t3, category)

    jvmLib = join(jdkDir, relativeVmLibDirInJdk(), get_vm(), mx.add_lib_suffix(mx.add_lib_prefix("jvm")))
    print
    if exists(jvmLib):
        print "{:10,}  {}".format(os.path.getsize(jvmLib), jvmLib)
    else:
        print "{:>10}  {}".format("<missing>", jvmLib)