예제 #1
0
def load_tests(config: Mapping[str, Any]) -> None:
    runtime_exe_path = build_runtime(config, "exe", test=True)
    data_path = pathify(config["data"], root_dir, cache_path, True, True)
    demo_data_path = pathify(config["demo_data"], root_dir, cache_path, True,
                             True)
    enable_offload_flag = config["enable_offload"]
    enable_alignment_flag = config["enable_alignment"]
    make(Path("common"), ["tests/run"])
    plugin_paths = threading_map(
        lambda plugin_config: build_one_plugin(
            config, plugin_config, test=True),
        [
            plugin_config for plugin_group in config["plugin_groups"]
            for plugin_config in plugin_group["plugin_group"]
        ],
        desc="Building plugins",
    )
    subprocess_run(
        [
            "catchsegv", "xvfb-run",
            str(runtime_exe_path), *map(str, plugin_paths)
        ],
        env_override=dict(
            ILLIXR_DATA=str(data_path),
            ILLIXR_DEMO_DATA=str(demo_data_path),
            ILLIXR_RUN_DURATION=str(config["action"].get(
                "ILLIXR_RUN_DURATION", 10)),
            ILLIXR_OFFLOAD_ENABLE=str(enable_offload_flag),
            ILLIXR_ALIGNMENT_ENABLE=str(enable_alignment_flag),
            ILLIXR_ENABLE_VERBOSE_ERRORS=str(config["enable_verbose_errors"]),
            KIMERA_ROOT=config["action"]["kimera_path"],
        ),
        check=True,
    )
예제 #2
0
파일: main.py 프로젝트: ILLIXR/ILLIXR
def clean_one_plugin(config: Mapping[str, Any], plugin_config: Mapping[str, Any]) -> Path:
    profile = config["profile"]
    path: Path = pathify(plugin_config["path"], root_dir, cache_path, True, True)
    path_str: str = str(path)
    name: str = plugin_config["name"] if plugin_config["name"] else os.path.basename(path_str)
    targets: List[str] = ["clean"]
    print(f"[Clean] Plugin '{name}' @ '{path_str}/'")
    env_override: Mapping[str, str] = dict(ILLIXR_INTEGRATION="yes")
    make(path, targets, plugin_config["config"], env_override=env_override)
    return path
예제 #3
0
def setup(config):
    workdir = os.path.abspath(os.getcwd())

    if not os.path.exists('llvm'):
        util.llvm.download(llvmver, os.getcwd())

    if not os.path.exists('llvm-' + llvmver):
        os.mkdir('llvm-' + llvmver)
        util.llvm.compile(src_dir=os.path.abspath('llvm'),
                          build_dir=os.path.abspath('llvm-' + llvmver),
                          install_dir=workdir)

    if config == configs[1]:
        if not os.path.exists('valgrind'):
            valgrind.download(os.path.abspath('valgrind'))

            if os.path.exists(os.path.join('bin', 'valgrind')):
                print('valgrind exists. skipping installation...')
            else:
                os.chdir('valgrind')
                util.configure('--prefix=' + workdir)
                util.make()
                util.make('install')
                os.chdir(workdir)

    # Create spec config file
    print 'creating spec config file...'

    spec_cfg = '-'.join([command, config]) + '.cfg'

    if os.path.exists(spec_cfg):
        os.remove(spec_cfg)

    cflags = list()

    valgrind_path = os.path.abspath(os.path.join('bin', 'valgrind'))
    spec_wrapper = ''
    if config == configs[1]:
        spec_wrapper = valgrind_path + ' --tool=memcheck'

    llvm_bin_path = os.path.join(workdir, 'llvm-' + llvmver, 'bin')

    cc = [os.path.join(llvm_bin_path, 'clang')]
    cc.extend(cflags)
    cxx = [os.path.join(llvm_bin_path, 'clang++')]
    cxx.extend(cflags)

    util.spec.create_config(dir=os.getcwd(),
                            file=spec_cfg,
                            name='-'.join([command, config]),
                            cc=' '.join(cc),
                            cxx=' '.join(cxx),
                            cld=' '.join(cxx),
                            spec_wrapper=spec_wrapper,
                            cxxportability=clang.common.cxxportability)
예제 #4
0
def build_runtime(
    config: Mapping[str, Any],
    suffix: str,
    test: bool = False,
) -> Path:
    profile = config["profile"]
    name = "main" if suffix == "exe" else "plugin"
    runtime_name = f"{name}.{profile}.{suffix}"
    runtime_config = config["runtime"]["config"]
    runtime_path: Path = pathify(config["runtime"]["path"], root_dir,
                                 cache_path, True, True)
    targets = [runtime_name] + (["tests/run"] if test else [])
    make(runtime_path, targets, runtime_config)
    return runtime_path / runtime_name
예제 #5
0
파일: main.py 프로젝트: ILLIXR/ILLIXR
def build_runtime(
    config: Mapping[str, Any],
    suffix: str,
    test: bool = False,
    is_mainline: bool = False,
) -> Path:
    profile = config["profile"]
    name = "main" if suffix == "exe" else "plugin"
    runtime_name = f"{name}.{profile}.{suffix}"
    runtime_config = config["runtime"]["config"]
    runtime_path: Path = pathify(config["runtime"]["path"], root_dir, cache_path, True, True)
    targets = [runtime_name] + (["tests/run"] if test else [])
    env_override: Mapping[str, str] = dict(ILLIXR_INTEGRATION="ON")
    if is_mainline:
        runtime_config.update(ILLIXR_MONADO_MAINLINE="ON")
    make(runtime_path, targets, runtime_config, env_override=env_override)
    return runtime_path / runtime_name
예제 #6
0
def build_one_plugin(
    config: Mapping[str, Any],
    plugin_config: Mapping[str, Any],
    test: bool = False,
) -> Path:
    profile = config["profile"]
    path: Path = pathify(plugin_config["path"], root_dir, cache_path, True,
                         True)
    if not (path / "common").exists():
        common_path = pathify(config["common"]["path"], root_dir, cache_path,
                              True, True)
        common_path = common_path.resolve()
        os.symlink(common_path, path / "common")
    plugin_so_name = f"plugin.{profile}.so"
    targets = [plugin_so_name] + (["tests/run"] if test else [])
    make(path, targets, plugin_config["config"])
    return path / plugin_so_name
예제 #7
0
def load_tests(config: Mapping[str, Any]) -> None:
    runtime_exe_path = build_runtime(config, "exe", test=True)
    data_path = pathify(config["data"], root_dir, cache_path, True, True)
    demo_data_path = pathify(config["demo_data"], root_dir, cache_path, True,
                             True)
    enable_offload_flag = config["enable_offload"]
    enable_alignment_flag = config["enable_alignment"]
    env_override: Mapping[str, str] = dict(ILLIXR_INTEGRATION="yes")
    make(Path("common"), ["tests/run"], env_override=env_override)
    realsense_cam_string = config["realsense_cam"]
    plugin_paths = threading_map(
        lambda plugin_config: build_one_plugin(
            config, plugin_config, test=True),
        [
            plugin_config for plugin_group in config["plugin_groups"]
            for plugin_config in plugin_group["plugin_group"]
        ],
        desc="Building plugins",
    )

    ## If pre-sleep is enabled, the application will pause and wait for a gdb process.
    ## If enabled, disable 'catchsegv' so that gdb can catch segfaults.
    enable_pre_sleep: bool = config["enable_pre_sleep"]
    cmd_list_tail: List[str] = [
        "xvfb-run", str(runtime_exe_path), *map(str, plugin_paths)
    ]
    cmd_list: List[str] = (["catchsegv"]
                           if not enable_pre_sleep else list()) + cmd_list_tail

    subprocess_run(
        cmd_list,
        env_override=dict(
            ILLIXR_DATA=str(data_path),
            ILLIXR_DEMO_DATA=str(demo_data_path),
            ILLIXR_RUN_DURATION=str(config["action"].get(
                "ILLIXR_RUN_DURATION", 10)),
            ILLIXR_OFFLOAD_ENABLE=str(enable_offload_flag),
            ILLIXR_ALIGNMENT_ENABLE=str(enable_alignment_flag),
            ILLIXR_ENABLE_VERBOSE_ERRORS=str(config["enable_verbose_errors"]),
            ILLIXR_ENABLE_PRE_SLEEP=str(enable_pre_sleep),
            KIMERA_ROOT=config["action"]["kimera_path"],
            REALSENSE_CAM=str(realsense_cam_string),
        ),
        check=True,
    )
예제 #8
0
파일: main.py 프로젝트: ILLIXR/ILLIXR
def build_one_plugin(
    config: Mapping[str, Any],
    plugin_config: Mapping[str, Any],
    test: bool = False,
) -> Path:
    profile = config["profile"]
    path: Path = pathify(plugin_config["path"], root_dir, cache_path, True, True)
    if not (path / "common").exists():
        common_path = pathify(config["common"]["path"], root_dir, cache_path, True, True)
        common_path = common_path.resolve()
        os.symlink(common_path, path / "common")
    plugin_so_name = f"plugin.{profile}.so"
    targets = [plugin_so_name] + (["tests/run"] if test else [])

    ## When building using runner, enable ILLIXR integrated mode (compilation)
    env_override: Mapping[str, str] = dict(ILLIXR_INTEGRATION="yes")
    make(path, targets, plugin_config["config"], env_override=env_override)

    return path / plugin_so_name
예제 #9
0
파일: test3.py 프로젝트: dshi7/537-sp14
def main():
    # 1. check if make generate an executable server file
    util.make()

    # 2. ok you got an excutable server, run it
    print ""
    print "Starting ./server 2010 1 1 FIFO"
    util.run("./server 2010 1 1 FIFO")

    clientcmd = "./testclient localhost 2010 /testdata/testfile.txt"
    print "Client: " + clientcmd
    response = commands.getoutput(clientcmd)
    if response.find("hey this is a test file") == -1:
        util.error("ouchs! client can not get the right file")
    print "Client got expected response"


    print ""
    print "Starting server 2010 1 1 SFF"
    util.run("./server 2010 1 1 SFF")
    print "Client: " + clientcmd
    response = commands.getoutput(clientcmd)
    if response.find("hey this is a test file") == -1:
        util.error("ouchs! client can not get the right file")
    print "Client got expected response"
 
    print ""
    print "Starting server 2010 1 1 SFF-BS"
    util.run("./server 2010 1 1 SFF-BS 1")
    print "Client: " + clientcmd
    response = commands.getoutput(clientcmd)
    if response.find("hey this is a test file") == -1:
        util.error("ouchs! client can not get the right file")
    print "Client got expected response"

    util.good(" Your server worked with simple requests")

    sys.exit(0)
def test(config):
    cwd = os.getcwd()

    os.chdir('test')

    if not os.path.exists('shrc'):
        raise Exception('no shrc found in directory %s' % testdir)

    try:
        f = open('shrc', 'r')
        lines = f.readlines()
        for line in lines:
            (key, _, value) = line.partition('=')
            os.environ[key.split()[1]] = value.rstrip().replace('"', '')
    finally:
        f.close()

    util.make()

    os.environ['LD_LIBRARY_PATH'] = os.path.join(cwd, 'lib')
    util.make('test')

    os.chdir(cwd)
예제 #11
0
def main():
    # 1. check if make generate an executable server file
    util.make()

    # 2. ok you got an excutable server, run it
    print ""
    print "Starting ./server 2010 1 1 FIFO"
    util.run("./server 2010 1 1 FIFO")

    clientcmd = "./testclient localhost 2010 /testdata/testfile.txt"
    print "Client: " + clientcmd
    response = commands.getoutput(clientcmd)
    if response.find("hey this is a test file") == -1:
        util.error("ouchs! client can not get the right file")
    print "Client got expected response"

    print ""
    print "Starting server 2010 1 1 SFF"
    util.run("./server 2010 1 1 SFF")
    print "Client: " + clientcmd
    response = commands.getoutput(clientcmd)
    if response.find("hey this is a test file") == -1:
        util.error("ouchs! client can not get the right file")
    print "Client got expected response"

    print ""
    print "Starting server 2010 1 1 SFF-BS"
    util.run("./server 2010 1 1 SFF-BS 1")
    print "Client: " + clientcmd
    response = commands.getoutput(clientcmd)
    if response.find("hey this is a test file") == -1:
        util.error("ouchs! client can not get the right file")
    print "Client got expected response"

    util.good(" Your server worked with simple requests")

    sys.exit(0)
def prepare_gold(version):
    curdir = os.getcwd()
    binutils_tarball = binutils_tarball_format.replace('VERSION', version)
    binutils_url = binutils_url_format.replace('TARBALL', binutils_tarball)
    util.download(binutils_url)
    util.untar(binutils_tarball)
    os.chdir('binutils-VERSION'.replace('VERSION', version))
    util.configure(
        '--prefix=%s --enable-gold --enable-plugins --disable-werror' % curdir)
    util.make('-j4')
    util.make('-j4 all-gold')
    util.make('install')
    os.chdir(curdir)
예제 #13
0
    def run(self):
        clientcmd = "./testclient localhost 2010 "
        if self.id == 0:
            file = "/output.cgi"
        else:
            file = "/testdata/file%s.txt" % str(self.id)

        clientcmd = clientcmd + file + " >> /tmp/file" + str(self.id) + " &"
        util.info(self.clientname + ": " + clientcmd)

        os.system(clientcmd)
        if self.id != 0:
            os.system("./testclient localhost 2010 /output.cgi > /dev/null &")


util.make()

# test SFF server
for i in range(0, NUM_LOOPS):
    print ""
    print "#############################"
    print "- %s iteration" % str(i)
    print "#############################"
    print ""
    test("./server 2010 1 7 FIFO")

util.good("great! FIFO is done correctly")

sys.exit(0)
예제 #14
0
def run_test(testcase):
    commands.getoutput("killall -9 server")
    commands.getoutput("killall -9 testclient")
    if int(testcase) == 1:
        if os.path.exists("README") or os.path.exists("README.txt"):
            return 0 # pass
        print "ERROR: no README or README.txt"
        return -1 # failed

    if int(testcase) == 2:
        if os.path.exists("Makefile") or os.path.exists("makefile"):
            return 0
        print "ERROR: no Makefile or makefile"
        return -1
    
    if int(testcase) == 3:
        commands.getoutput("rm -rf server")
        commands.getoutput("rm -rf libmfs.so")
        commands.getoutput("make")
        if not os.path.exists("server") or not os.path.exists("libmfs.so"):
            print "ERROR: make does not generate server and libmfs.so"
            return -1
        return 0

    if int(testcase) > 3 and int(testcase) <=39:
        rc = util.make()
        if rc == -1:
            return -1
        rc = util.compile_testclient(testcase)
        if rc == -1:
            return -1
        
        rc = util.start_fresh_server(port)
        if rc == -1:
            return -1

        status, out = commands.getstatusoutput("./testclient %s %s"
                                               % (port, host))
#        print out
        commands.getoutput("killall -9 server")
        return status


    if int(testcase) == 40: # test a 
        rc = util.make()
        if rc == -1:
            return -1;
        
        # start server with a fresh file
        commands.getoutput("rm -rf fsimage")
        commands.getoutput("rm -rf server-out.txt")
        commands.getoutput("killall -9 server")
        os.system("./server %s fsimage > /dev/null &" % port)
        #time.sleep(1)
        time.sleep(3) # added on 5/3 9am
        rc = util.compile_testclient(str(testcase) + "pre")
        if rc == -1:
            return -1
        status, out = commands.getstatusoutput("./testclient %s %s"
                                               % (port, host))
#        print out
        if status == -1:
            print "can not pass PRE test"
            return -1

        commands.getoutput("killall -9 server")
        time.sleep(1) # added on 5/3 9am

        os.system("./server %s fsimage > /dev/null &" % port)
        time.sleep(0.5)
        rc = util.compile_testclient(str(testcase) + "post")
        if rc == -1:
            return -1
        status, out = commands.getstatusoutput("./testclient %s %s"
                                               % (port, host))
#        print out
        commands.getoutput("killall -9 server")
        return status

    if int(testcase) >= 41:
        rc = util.make()
        if rc == -1:
            return -1
        rc = util.compile_testclient(testcase)
        if rc == -1:
            return -1
        clientout = "out.txt"
        commands.getoutput("rm -rf out.txt")
        os.system("./testclient %s %s > %s &" % (port, host, clientout))
        time.sleep(ctimeout/2)
        rc = util.start_fresh_server(port)
        if rc == -1:
            return -1
        time.sleep(ctimeout)
        ps = commands.getoutput("ps -ax")
        if ps.find("./testclient %s %s" % (port, host)) >= 0:
            # the testclient is still running
            print "ERROR: your client did not implement timeout correctly"
            print "ERROR: testclient should have finished"
            return -1
        else: # now testclient finish, let check the return status
            cat = commands.getoutput("cat %s" % clientout)
            if cat.find("PASSED") >= 0:
                return 0
            return -1
def setup(config):
    workdir = os.path.abspath(os.getcwd())

    if not os.path.exists('dangsan'):
        util.git.clone(dangsan_repo)

    if not os.path.exists('llvm-svn'):
        util.llvm.checkout_svn(llvm_svn_commit)
        os.chdir('llvm-svn')
        util.patch(
            os.path.join(workdir, 'dangsan', 'patches',
                         'LLVM-gold-plugins-3.8.diff'), '-p0')
        util.patch(
            os.path.join(workdir, 'dangsan', 'patches',
                         'LLVM-safestack-3.8.diff'), '-p0')
        os.chdir('projects/compiler-rt')
        util.patch(
            os.path.join(workdir, 'dangsan', 'patches',
                         'COMPILERRT-safestack-3.8.diff'), '-p0')
        os.chdir(workdir)

    if not os.path.exists('bin/ld'):
        prepare_gold(binutils_version)
        os.remove('bin/ld')
        shutil.copy('bin/ld.gold', 'bin/ld')

    if not os.path.exists('llvm-build'):
        os.mkdir('llvm-build')
        util.llvm.compile(
            src_dir=os.path.abspath('llvm-svn'),
            build_dir=os.path.abspath('llvm-build'),
            install_dir=os.getcwd(),
            options=['-DLLVM_BINUTILS_INCDIR=%s/include' % workdir])
        util.llvm.install(os.path.abspath('llvm-build'))

    if not glob.glob('gperftools*'):
        util.git.clone(gperftools_repo)
        os.chdir('gperftools')
        util.git.checkout(gperftools_commit)

        # gperftools patch for both baseline and DangSan
        util.git.patch(
            os.path.join(workdir, 'dangsan', 'patches',
                         'GPERFTOOLS_SPEEDUP.patch'))

        if config == configs[1] or config == configs[2]:
            os.chdir(workdir)

            # Build metapagetable
            if not os.path.exists('gperftools-metalloc'):
                shutil.move('gperftools', 'gperftools-metalloc')

            if not os.path.exists('metapagetable'):
                config_fixedcompression = 'false'
                config_metadatabytes = 8
                config_deepmetadata = 'false'
                config_alloc_size_hook = 'dang_alloc_size_hook'
                metalloc_options = []
                metalloc_options.append('-DFIXEDCOMPRESSION=%s' %
                                        config_fixedcompression)
                metalloc_options.append('-DMETADATABYTES=%d' %
                                        config_metadatabytes)
                metalloc_options.append('-DDEEPMETADATA=%s' %
                                        config_deepmetadata)
                metalloc_options.append('-DALLOC_SIZE_HOOK=%s' %
                                        config_alloc_size_hook)
                os.environ['METALLOC_OPTIONS'] = ' '.join(metalloc_options)

                shutil.copytree(os.path.join('dangsan', 'metapagetable'),
                                'metapagetable')
                os.chdir('metapagetable')
                util.make('config')
                util.make()
                os.chdir(workdir)

            # Apply DanSan patches for gperftool
            os.chdir('gperftools-metalloc')
            util.git.patch(
                os.path.join(os.path.dirname(__file__),
                             'GPERFTOOLS_DANGSAN.patch'))

        util.run('autoreconf -fi')
        util.configure('--prefix=%s' % workdir)
        util.make('-j4')
        util.make('install')
        os.chdir(workdir)

    # build llvm-plugins
    llvmplugins_dir = os.path.join(workdir, 'llvm-plugins')
    libplugins_so = os.path.join(llvmplugins_dir, 'libplugins.so')
    if config == configs[1] or config == configs[2]:
        if not os.path.exists('staticlib'):
            os.environ['PATH'] = ':'.join(
                [os.path.join(workdir, 'bin'), os.environ['PATH']])
            shutil.copytree('dangsan/staticlib', 'staticlib')
            os.chdir('staticlib')
            util.make('METAPAGETABLEDIR=%s' %
                      os.path.join(workdir, 'metapagetable', 'obj'))
            os.chdir(workdir)

        if not os.path.exists(libplugins_so):
            os.environ['PATH'] = ':'.join(
                [os.path.join(workdir, 'bin'), os.environ['PATH']])
            os.chdir('dangsan/llvm-plugins')
            util.make('-j4 GOLDINSTDIR=%s TARGETDIR=%s' %
                      (workdir, llvmplugins_dir))
            os.chdir(workdir)

    # Create spec config file
    print 'creating spec config file...'

    spec_cfg = '-'.join([command, config]) + '.cfg'

    if os.path.exists(spec_cfg):
        os.remove(spec_cfg)

    cflags = ['-flto']
    cxxflags = ['-flto']
    ldflags = []
    if config == configs[1] or config == configs[2]:
        cflags.append('-fsanitize=safe-stack')
        cxxflags.append('-fsanitize=safe-stack')
        cxxflags.append('-DSOPLEX_DANGSAN_MASK')
        ldflags.append('-Wl,-plugin-opt=-load=%s' % libplugins_so)
        ldflags.append('-Wl,-plugin-opt=%s' % '-largestack=false')
        # ldflags.append('-Wl,-plugin-opt=%s' % '-stats') # option not found
        ldflags.append('-Wl,-plugin-opt=%s' % '-mergedstack=false')
        ldflags.append('-Wl,-plugin-opt=%s' % '-stacktracker')
        ldflags.append('-Wl,-plugin-opt=%s' % '-globaltracker')
        ldflags.append('-Wl,-plugin-opt=%s' % '-pointertracker')
        ldflags.append('-Wl,-plugin-opt=%s' % '-FreeSentryLoop')
        ldflags.append('-Wl,-plugin-opt=%s' % '-custominline')
        ldflags.append('-Wl,-whole-archive,-l:libmetadata.a,-no-whole-archive'
                       )  # staticlib
        ldflags.append(
            '@%s' %
            os.path.join(workdir, 'metapagetable', 'obj', 'linker-options'))

    extra_libs = ['-ltcmalloc', '-lpthread', '-lunwind']
    extra_libs.append('-L%s' % os.path.join(workdir, 'lib'))
    if config == configs[1] or config == configs[2]:
        extra_libs.append('-ldl')
        extra_libs.append('-L%s' % os.path.join(workdir, 'staticlib', 'obj'))

    if config == configs[2]:
        cflags.append('-g')
        cxxflags.append('-g')

    cc = [os.path.abspath(os.path.join(workdir, 'bin', 'clang'))]
    cc.extend(cflags)
    cxx = [os.path.abspath(os.path.join(workdir, 'bin', 'clang++'))]
    cxx.extend(cxxflags)
    cld = list(cc)
    cld.extend(ldflags)
    cxxld = list(cxx)
    cxxld.extend(ldflags)

    util.spec.create_config(dir=os.getcwd(),
                            file=spec_cfg,
                            name='-'.join([command, config]),
                            cc=' '.join(cc),
                            cxx=' '.join(cxx),
                            cld=' '.join(cld),
                            cxxld=' '.join(cxxld),
                            extra_libs=' '.join(extra_libs))

    # create shrc for testing
    print('creating test directory...')

    if not os.path.exists('test'):
        testdir = os.path.join(os.path.dirname(__file__), 'test')
        shutil.copytree(testdir, os.path.join(workdir, 'test'))

    if os.path.exists('test/shrc'):
        os.remove('test/shrc')

    util.test.create_shrc(dir=os.path.abspath('test'),
                          cc=' '.join(cc),
                          cxx=' '.join(cxx),
                          ld=' '.join(cld),
                          ldflags=' '.join(extra_libs))
예제 #16
0
def run_test(testcase):
    commands.getoutput("killall -9 server")
    commands.getoutput("killall -9 testclient")
    if int(testcase) == 1:
        if os.path.exists("README") or os.path.exists("README.txt"):
            return 0 # pass
        print "ERROR: no README or README.txt"
        return -1 # failed

    if int(testcase) == 2:
        if os.path.exists("Makefile") or os.path.exists("makefile"):
            return 0
        print "ERROR: no Makefile or makefile"
        return -1
    
    if int(testcase) == 3:
        commands.getoutput("rm -rf server")
        commands.getoutput("rm -rf libmfs.so")
        commands.getoutput("make")
        if not os.path.exists("server") or not os.path.exists("libmfs.so"):
            print "ERROR: make does not generate server and libmfs.so"
            return -1
        return 0

    if int(testcase) > 3 and int(testcase) <=39:
        rc = util.make()
        if rc == -1:
            return -1
        rc = util.compile_testclient(testcase)
        if rc == -1:
            return -1
        
        rc = util.start_fresh_server(port)
        if rc == -1:
            return -1

        status, out = commands.getstatusoutput("./testclient %s %s"
                                               % (port, host))
#        print out
        commands.getoutput("killall -9 server")
        return status


    if int(testcase) == 40: # test a 
        rc = util.make()
        if rc == -1:
            return -1;
        
        # start server with a fresh file
        commands.getoutput("rm -rf fsimage")
        commands.getoutput("rm -rf server-out.txt")
        commands.getoutput("killall -9 server")
        os.system("./server %s fsimage > /dev/null &" % port)
        #time.sleep(1)
        time.sleep(3) # added on 5/3 9am
        rc = util.compile_testclient(str(testcase) + "pre")
        if rc == -1:
            return -1
        status, out = commands.getstatusoutput("./testclient %s %s"
                                               % (port, host))
#        print out
        if status == -1:
            print "can not pass PRE test"
            return -1

        commands.getoutput("killall -9 server")
        time.sleep(1) # added on 5/3 9am

        os.system("./server %s fsimage > /dev/null &" % port)
        time.sleep(0.5)
        rc = util.compile_testclient(str(testcase) + "post")
        if rc == -1:
            return -1
        status, out = commands.getstatusoutput("./testclient %s %s"
                                               % (port, host))
#        print out
        commands.getoutput("killall -9 server")
        return status

    if int(testcase) >= 41:
        rc = util.make()
        if rc == -1:
            return -1
        rc = util.compile_testclient(testcase)
        if rc == -1:
            return -1
        clientout = "out.txt"
        commands.getoutput("rm -rf out.txt")
        os.system("./testclient %s %s > %s &" % (port, host, clientout))
        time.sleep(ctimeout/2)
        rc = util.start_fresh_server(port)
        if rc == -1:
            return -1
        time.sleep(ctimeout)
        ps = commands.getoutput("ps -ax")
        if ps.find("./testclient %s %s" % (port, host)) >= 0:
            # the testclient is still running
            print "ERROR: your client did not implement timeout correctly"
            print "ERROR: testclient should have finished"
            return -1
        else: # now testclient finish, let check the return status
            cat = commands.getoutput("cat %s" % clientout)
            if cat.find("PASSED") >= 0:
                return 0
            return -1
예제 #17
0
파일: setup.py 프로젝트: wangyf/SORDlatest
def build(mode=None, optimize=None):
    """
    Build SORD code.
    """
    cf = util.namespace(configure.configure())
    if not optimize:
        optimize = cf.optimize
    if not mode:
        mode = cf.mode
    if not mode:
        mode = 'sm'
    base = (
        'globals.f90',
        'diffcn.f90',
        'diffnc.f90',
        'hourglass.f90',
        'bc.f90',
        'surfnormals.f90',
        'util.f90',
        'frio.f90',
    )
    common = (
        'arrays.f90',
        'fieldio.f90',
        'stats.f90',
        'parameters.f90',
        'setup.f90',
        'gridgen.f90',
        'attenuation.f90',
        'material.f90',
        'source.f90',
        'inivolstress.f90',
        'rupture.f90',
        'resample.f90',
        'checkpoint.f90',
        'timestep.f90',
        'stress.f90',
        'acceleration.f90',
        'sord.f90',
    )
    cwd = os.getcwd()
    path = os.path.realpath(os.path.dirname(__file__))
    f = os.path.join(path, 'bin')
    if not os.path.isdir(f):
        os.mkdir(f)
    new = False
    os.chdir(os.path.join(path, 'src'))
    if 's' in mode:
        source = base + ('serial.f90', ) + common
        for opt in optimize:
            object_ = os.path.join('..', 'bin', 'sord-s' + opt)
            compiler = cf.fortran_serial + cf.fortran_flags[opt]
            new |= util.make(compiler, object_, source)
    if 'm' in mode and cf.fortran_mpi:
        source = base + ('mpi.f90', ) + common
        for opt in optimize:
            object_ = os.path.join('..', 'bin', 'sord-m' + opt)
            compiler = cf.fortran_mpi + cf.fortran_flags[opt]
            new |= util.make(compiler, object_, source)
    os.chdir(path)
    if new:
        try:
            import bzrlib
        except ImportError:
            print(
                'Warning: bzr not installed. Install bzr if you want to save a\
                copy of the source code for posterity with each run.')
        else:
            os.system('bzr export sord.tgz')
    os.chdir(cwd)
    return
def setup(config):
    workdir = os.path.abspath(os.getcwd())

    if not os.path.exists('typesan'):
        util.git.clone(typesan_repo)

    if not os.path.exists('llvm'):
        util.llvm.checkout(llvm_commit, clang_commit, compilerrt_commit)

    if not os.path.exists('llvm-build'):
        os.mkdir('llvm-build')

        # Apply TypeSan patches
        if config == configs[1]:
            llvm_patch = os.path.join(os.path.dirname(__file__), 'llvm.patch')
            clang_patch = os.path.join(os.path.dirname(__file__),
                                       'clang.patch')
            compilerrt_patch = os.path.join(os.path.dirname(__file__),
                                            'compiler-rt.patch')

            os.chdir('llvm')
            util.git.patch(llvm_patch)
            os.chdir(os.path.join(workdir, 'llvm/tools/clang'))
            util.git.patch(clang_patch)
            os.chdir(os.path.join(workdir, 'llvm/projects/compiler-rt'))
            util.git.patch(compilerrt_patch)
            os.chdir(workdir)

        util.llvm.compile(src_dir=os.path.abspath('llvm'),
                          build_dir=os.path.abspath('llvm-build'),
                          install_dir=os.getcwd())

    if not glob.glob('gperftools*'):
        util.git.clone(gperftools_repo)
        os.chdir('gperftools')
        util.git.checkout(gperftools_commit)
        util.git.patch(
            os.path.join(os.path.dirname(__file__),
                         'GPERFTOOLS_SPEEDUP.patch'))

        if config == configs[1]:
            os.chdir(workdir)

            # Build metapagetable
            if not os.path.exists('gperftools-metalloc'):
                shutil.move('gperftools', 'gperftools-metalloc')

            if not os.path.exists('metapagetable'):
                config_fixedcompression = 'false'
                config_metadatabytes = 16
                config_deepmetadata = 'false'
                os.environ[
                    'METALLOC_OPTIONS'] = '-DFIXEDCOMPRESSION=%s -DMETADATABYTES=%d -DDEEPMETADATA=%s' % (
                        config_fixedcompression, config_metadatabytes,
                        config_deepmetadata)

                shutil.copytree(os.path.join('typesan', 'metapagetable'),
                                'metapagetable')
                os.chdir('metapagetable')
                util.make('config')
                util.make()
                os.chdir(workdir)

            # Apply TypeSan patches for gperftool
            os.chdir('gperftools-metalloc')
            util.git.patch(
                os.path.join(os.path.dirname(__file__),
                             'GPERFTOOLS_TYPESAN.patch'))

        util.run('autoreconf -fi')
        util.configure('--prefix=' + workdir)
        util.make()
        util.make('install')
        os.chdir(workdir)

    # Create spec config file
    print 'creating spec config file...'

    spec_cfg = '-'.join([command, config]) + '.cfg'

    if os.path.exists(spec_cfg):
        os.remove(spec_cfg)

    path = os.path.join('llvm-build', 'bin')

    cflags = []
    extra_libs = []
    if config == configs[1]:
        cflags = ['-fsanitize=typesan']

    extra_libs.extend(['-ltcmalloc', '-lpthread', '-lunwind'])
    extra_libs.append('-L' + os.path.join(workdir, 'lib'))  # tcmalloc

    cc = [os.path.abspath(os.path.join(path, 'clang'))]
    cc.extend(cflags)
    cxx = [os.path.abspath(os.path.join(path, 'clang++'))]
    cxx.extend(cflags)

    util.spec.create_config(dir=os.getcwd(),
                            file=spec_cfg,
                            name='-'.join([command, config]),
                            cc=' '.join(cc),
                            cxx=' '.join(cxx),
                            extra_libs=' '.join(extra_libs))

    # create shrc for testing
    print('creating test directory...')

    if not os.path.exists('test'):
        testdir = os.path.join(os.path.dirname(__file__), 'test')
        shutil.copytree(testdir, os.path.join(workdir, 'test'))

    if os.path.exists('test/shrc'):
        os.remove('test/shrc')

    util.test.create_shrc(dir=os.path.abspath('test'),
                          cc=' '.join(cc),
                          cxx=' '.join(cxx),
                          ld=' '.join(cxx),
                          ldflags=' '.join(extra_libs))
예제 #19
0
###############################################

def test(cmd):

    print ""
    print "- Starting " + cmd
    util.run2(cmd)
    
    if util.is_server_alive(cmd) != -1:
        util.error("Ouch! Server is still alive!")

    print ""
    print "GOOD! ..."
    print ""
        
util.make()

# test FIFO server
test("./server 2010 1 FIFO")
test("./server 1 2 FIFO")
# test SFF server
test("./server 2010 2 SFF")
test("./server 2010 2 1")
test("./server 1 SFF")
# test SFF-BS
test("./server 2010 2 2 SSF-BS")


sys.exit(0)

예제 #20
0
def setup(config):
    workdir = os.path.abspath(os.getcwd())
    configdir = os.path.join(os.path.dirname(__file__), 'config')
    INSTRUMENTATION_PATH = os.path.abspath('llvm/lib/Transforms/Instrumentation')

    if not os.path.exists('llvm'):
        util.llvm.download(llvmver, os.getcwd())

        # Apply patches common to both baseline and LowFat
        bug81066_patch = os.path.join(os.path.dirname(__file__), 'bug81066.patch')
        os.chdir('llvm/projects/compiler-rt')
        util.patch(os.path.abspath(bug81066_patch), '-p0')
        os.chdir(workdir)

    if not os.path.exists('llvm-4.0.0') and not os.path.exists('llvm-lowfat'):
        os.mkdir('llvm-4.0.0')
        util.llvm.compile(src_dir=os.path.abspath('llvm'),
                          build_dir=os.path.abspath('llvm-4.0.0'),
                          install_dir=workdir)

    if config != configs[0]:
        os.environ['CC'] = os.path.abspath('llvm-4.0.0/bin/clang')
        os.environ['CXX'] = os.path.abspath('llvm-4.0.0/bin/clang++')

        if not os.path.exists('llvm-lowfat'):
            print 'patching llvm/clang/compiler-rt version ' + llvmver + '...'

            llvm_patch = os.path.join(os.path.dirname(__file__), 'llvm.patch')
            clang_patch = os.path.join(os.path.dirname(__file__), 'clang.patch')
            compilerrt_patch = os.path.join(os.path.dirname(__file__), 'compiler-rt.patch')

            legacy_compilerrt_patch = os.path.join(os.path.dirname(__file__), 'legacy_compiler-rt.patch')
            modern_compilerrt_patch = os.path.join(os.path.dirname(__file__), 'modern_compiler-rt.patch')

            os.chdir('llvm')
            util.patch(os.path.abspath(llvm_patch))
            os.chdir(workdir)

            os.chdir('llvm/tools/clang')
            util.patch(os.path.abspath(clang_patch))
            os.chdir(workdir)

            os.chdir('llvm/projects/compiler-rt')
            util.patch(os.path.abspath(compilerrt_patch))

            cpuinfo = util.stdout(['cat', '/proc/cpuinfo']).strip().split('\n')

            legacy = False

            if not [s for s in cpuinfo if ' bmi1' in s]:
                legacy = True

            if not [s for s in cpuinfo if ' bmi2' in s]:
                legacy = True

            if not [s for s in cpuinfo if ' abm' in s]:
                legacy = True

            if legacy:
                util.patch(os.path.abspath(legacy_compilerrt_patch))
            else:
                util.patch(os.path.abspath(modern_compilerrt_patch))

            os.chdir(workdir)

            print 'building the LowFat config builder...'

            CONFIG = 'sizes.cfg 32'
            os.chdir(os.path.join(os.path.dirname(__file__), 'config'))
            util.make()
            util.run('./lowfat-config ' + CONFIG)
            util.make('lowfat-check-config')
            util.run('./lowfat-check-config')
            os.chdir(workdir)

            print 'copying the LowFat config files...'

            RUNTIME_PATH = os.path.abspath('llvm/projects/compiler-rt/lib/lowfat')
            shutil.copy(os.path.join(configdir, 'lowfat_config.h'), RUNTIME_PATH)
            shutil.copy(os.path.join(configdir, 'lowfat_config.c'), RUNTIME_PATH)

            CLANGLIB_PATH = os.path.abspath('llvm/tools/clang/lib/Basic')
            shutil.copy(os.path.join(RUNTIME_PATH, 'lowfat_config.c'),
                        os.path.join(INSTRUMENTATION_PATH, 'lowfat_config.inc'))
            shutil.copy(os.path.join(RUNTIME_PATH, 'lowfat_config.h'),
                        os.path.join(INSTRUMENTATION_PATH, 'lowfat_config.h'))
            shutil.copy(os.path.join(RUNTIME_PATH, 'lowfat_config.h'),
                        os.path.join(CLANGLIB_PATH, 'lowfat_config.h'))
            shutil.copy(os.path.join(RUNTIME_PATH, 'lowfat.h'),
                        os.path.join(INSTRUMENTATION_PATH, 'lowfat.h'))

            os.mkdir('llvm-lowfat')

            util.llvm.compile(src_dir=os.path.abspath('llvm'),
                              build_dir=os.path.abspath('llvm-lowfat'),
                              install_dir=workdir)

        os.chdir('llvm-lowfat')

        if not os.path.exists('lib/LowFat'):
            if not os.path.exists('lib'):
                os.mkdir('lib')
            os.mkdir('lib/LowFat')
            shutil.copy(os.path.join(configdir, 'lowfat.ld'),
                        os.path.join('lib/LowFat', 'lowfat.ld'))

        if not os.path.exists('bin/lowfat-ptr-info'):
            os.chdir(configdir)
            util.make('lowfat-ptr-info')
            shutil.copy('lowfat-ptr-info', os.path.join(workdir, 'llvm-lowfat', 'bin'))

        os.chdir(workdir)

        if not os.path.exists('LowFat.so'):
            clangxx = os.path.join('llvm-4.0.0', 'bin', 'clang++')
            llvm_config = os.path.join('llvm-4.0.0', 'bin', 'llvm-config')

            cxxflags = util.stdout([llvm_config, '--cxxflags']).rstrip('\n')
            includedir = util.stdout([llvm_config, '--includedir']).rstrip('\n')
            cmd = ' '.join([clangxx, '-DLOWFAT_PLUGIN',
                            os.path.join(INSTRUMENTATION_PATH, 'LowFat.cpp'),
                            '-c -Wall -O2',
                            '-I' + INSTRUMENTATION_PATH,
                            '-o LowFat.o',
                            cxxflags, includedir])
            util.run(cmd)

            ldflags = util.stdout([llvm_config, '--ldflags']).rstrip('\n')
            cmd = ' '.join([clangxx, '-shared -rdynamic -o LowFat.so LowFat.o', ldflags])
            util.run(cmd)

        os.chdir(workdir)

    # Create spec config file
    print 'creating spec config file...'

    spec_cfg = '-'.join([command, config]) + '.cfg'

    if os.path.exists(spec_cfg):
        os.remove(spec_cfg)

    if config == configs[0]:
        path = os.path.join('llvm-4.0.0', 'bin')
        cflags = []
    else:
        path = os.path.join('llvm-lowfat', 'bin')
        cflags = ['-fsanitize=lowfat']
        blacklist = os.path.join(os.path.dirname(__file__), 'blacklist.txt')
        cflags.extend(['-mllvm', '-lowfat-no-check-blacklist=%s' % blacklist])

    cflags.append('-DPATCH_PERLBENCH_OVERFLOW')
    cflags.append('-DPATCH_H264REF_OVERFLOW')

    if config == configs[2]:
        cflags.append('-g')
        cflags.append('-fno-inline-functions')

    if config == configs[3]:
        cflags.append('-mllvm -lowfat-no-abort')

    cc = [os.path.abspath(os.path.join(path, 'clang'))]
    cc.extend(cflags)
    cxx = [os.path.abspath(os.path.join(path, 'clang++'))]
    cxx.extend(cflags)

    util.spec.create_config(dir=os.getcwd(), file=spec_cfg,
                            name='-'.join([command, config]),
                            cc=' '.join(cc), cxx=' '.join(cxx))