def load_tests(config: Mapping[str, Any]) -> None:
runtime_exe_path = build_runtime(config, "exe", test=True)
data_path = pathify(config["data"], root_dir, cache_path, True, True)
demo_data_path = pathify(config["demo_data"], root_dir, cache_path, True,
True)
enable_offload_flag = config["enable_offload"]
enable_alignment_flag = config["enable_alignment"]
make(Path("common"), ["tests/run"])
plugin_paths = threading_map(
lambda plugin_config: build_one_plugin(
config, plugin_config, test=True),
[
plugin_config for plugin_group in config["plugin_groups"]
for plugin_config in plugin_group["plugin_group"]
],
desc="Building plugins",
)
subprocess_run(
[
"catchsegv", "xvfb-run",
str(runtime_exe_path), *map(str, plugin_paths)
],
env_override=dict(
ILLIXR_DATA=str(data_path),
ILLIXR_DEMO_DATA=str(demo_data_path),
ILLIXR_RUN_DURATION=str(config["action"].get(
"ILLIXR_RUN_DURATION", 10)),
ILLIXR_OFFLOAD_ENABLE=str(enable_offload_flag),
ILLIXR_ALIGNMENT_ENABLE=str(enable_alignment_flag),
ILLIXR_ENABLE_VERBOSE_ERRORS=str(config["enable_verbose_errors"]),
KIMERA_ROOT=config["action"]["kimera_path"],
),
check=True,
)
def clean_one_plugin(config: Mapping[str, Any], plugin_config: Mapping[str, Any]) -> Path:
profile = config["profile"]
path: Path = pathify(plugin_config["path"], root_dir, cache_path, True, True)
path_str: str = str(path)
name: str = plugin_config["name"] if plugin_config["name"] else os.path.basename(path_str)
targets: List[str] = ["clean"]
print(f"[Clean] Plugin '{name}' @ '{path_str}/'")
env_override: Mapping[str, str] = dict(ILLIXR_INTEGRATION="yes")
make(path, targets, plugin_config["config"], env_override=env_override)
return path
def setup(config):
workdir = os.path.abspath(os.getcwd())
if not os.path.exists('llvm'):
util.llvm.download(llvmver, os.getcwd())
if not os.path.exists('llvm-' + llvmver):
os.mkdir('llvm-' + llvmver)
util.llvm.compile(src_dir=os.path.abspath('llvm'),
build_dir=os.path.abspath('llvm-' + llvmver),
install_dir=workdir)
if config == configs[1]:
if not os.path.exists('valgrind'):
valgrind.download(os.path.abspath('valgrind'))
if os.path.exists(os.path.join('bin', 'valgrind')):
print('valgrind exists. skipping installation...')
else:
os.chdir('valgrind')
util.configure('--prefix=' + workdir)
util.make()
util.make('install')
os.chdir(workdir)
# Create spec config file
print 'creating spec config file...'
spec_cfg = '-'.join([command, config]) + '.cfg'
if os.path.exists(spec_cfg):
os.remove(spec_cfg)
cflags = list()
valgrind_path = os.path.abspath(os.path.join('bin', 'valgrind'))
spec_wrapper = ''
if config == configs[1]:
spec_wrapper = valgrind_path + ' --tool=memcheck'
llvm_bin_path = os.path.join(workdir, 'llvm-' + llvmver, 'bin')
cc = [os.path.join(llvm_bin_path, 'clang')]
cc.extend(cflags)
cxx = [os.path.join(llvm_bin_path, 'clang++')]
cxx.extend(cflags)
util.spec.create_config(dir=os.getcwd(),
file=spec_cfg,
name='-'.join([command, config]),
cc=' '.join(cc),
cxx=' '.join(cxx),
cld=' '.join(cxx),
spec_wrapper=spec_wrapper,
cxxportability=clang.common.cxxportability)
def build_runtime(
config: Mapping[str, Any],
suffix: str,
test: bool = False,
) -> Path:
profile = config["profile"]
name = "main" if suffix == "exe" else "plugin"
runtime_name = f"{name}.{profile}.{suffix}"
runtime_config = config["runtime"]["config"]
runtime_path: Path = pathify(config["runtime"]["path"], root_dir,
cache_path, True, True)
targets = [runtime_name] + (["tests/run"] if test else [])
make(runtime_path, targets, runtime_config)
return runtime_path / runtime_name
def build_runtime(
config: Mapping[str, Any],
suffix: str,
test: bool = False,
is_mainline: bool = False,
) -> Path:
profile = config["profile"]
name = "main" if suffix == "exe" else "plugin"
runtime_name = f"{name}.{profile}.{suffix}"
runtime_config = config["runtime"]["config"]
runtime_path: Path = pathify(config["runtime"]["path"], root_dir, cache_path, True, True)
targets = [runtime_name] + (["tests/run"] if test else [])
env_override: Mapping[str, str] = dict(ILLIXR_INTEGRATION="ON")
if is_mainline:
runtime_config.update(ILLIXR_MONADO_MAINLINE="ON")
make(runtime_path, targets, runtime_config, env_override=env_override)
return runtime_path / runtime_name
def build_one_plugin(
config: Mapping[str, Any],
plugin_config: Mapping[str, Any],
test: bool = False,
) -> Path:
profile = config["profile"]
path: Path = pathify(plugin_config["path"], root_dir, cache_path, True,
True)
if not (path / "common").exists():
common_path = pathify(config["common"]["path"], root_dir, cache_path,
True, True)
common_path = common_path.resolve()
os.symlink(common_path, path / "common")
plugin_so_name = f"plugin.{profile}.so"
targets = [plugin_so_name] + (["tests/run"] if test else [])
make(path, targets, plugin_config["config"])
return path / plugin_so_name
def load_tests(config: Mapping[str, Any]) -> None:
runtime_exe_path = build_runtime(config, "exe", test=True)
data_path = pathify(config["data"], root_dir, cache_path, True, True)
demo_data_path = pathify(config["demo_data"], root_dir, cache_path, True,
True)
enable_offload_flag = config["enable_offload"]
enable_alignment_flag = config["enable_alignment"]
env_override: Mapping[str, str] = dict(ILLIXR_INTEGRATION="yes")
make(Path("common"), ["tests/run"], env_override=env_override)
realsense_cam_string = config["realsense_cam"]
plugin_paths = threading_map(
lambda plugin_config: build_one_plugin(
config, plugin_config, test=True),
[
plugin_config for plugin_group in config["plugin_groups"]
for plugin_config in plugin_group["plugin_group"]
],
desc="Building plugins",
)
## If pre-sleep is enabled, the application will pause and wait for a gdb process.
## If enabled, disable 'catchsegv' so that gdb can catch segfaults.
enable_pre_sleep: bool = config["enable_pre_sleep"]
cmd_list_tail: List[str] = [
"xvfb-run", str(runtime_exe_path), *map(str, plugin_paths)
]
cmd_list: List[str] = (["catchsegv"]
if not enable_pre_sleep else list()) + cmd_list_tail
subprocess_run(
cmd_list,
env_override=dict(
ILLIXR_DATA=str(data_path),
ILLIXR_DEMO_DATA=str(demo_data_path),
ILLIXR_RUN_DURATION=str(config["action"].get(
"ILLIXR_RUN_DURATION", 10)),
ILLIXR_OFFLOAD_ENABLE=str(enable_offload_flag),
ILLIXR_ALIGNMENT_ENABLE=str(enable_alignment_flag),
ILLIXR_ENABLE_VERBOSE_ERRORS=str(config["enable_verbose_errors"]),
ILLIXR_ENABLE_PRE_SLEEP=str(enable_pre_sleep),
KIMERA_ROOT=config["action"]["kimera_path"],
REALSENSE_CAM=str(realsense_cam_string),
),
check=True,
)
def build_one_plugin(
config: Mapping[str, Any],
plugin_config: Mapping[str, Any],
test: bool = False,
) -> Path:
profile = config["profile"]
path: Path = pathify(plugin_config["path"], root_dir, cache_path, True, True)
if not (path / "common").exists():
common_path = pathify(config["common"]["path"], root_dir, cache_path, True, True)
common_path = common_path.resolve()
os.symlink(common_path, path / "common")
plugin_so_name = f"plugin.{profile}.so"
targets = [plugin_so_name] + (["tests/run"] if test else [])
## When building using runner, enable ILLIXR integrated mode (compilation)
env_override: Mapping[str, str] = dict(ILLIXR_INTEGRATION="yes")
make(path, targets, plugin_config["config"], env_override=env_override)
return path / plugin_so_name
def main():
# 1. check if make generate an executable server file
util.make()
# 2. ok you got an excutable server, run it
print ""
print "Starting ./server 2010 1 1 FIFO"
util.run("./server 2010 1 1 FIFO")
clientcmd = "./testclient localhost 2010 /testdata/testfile.txt"
print "Client: " + clientcmd
response = commands.getoutput(clientcmd)
if response.find("hey this is a test file") == -1:
util.error("ouchs! client can not get the right file")
print "Client got expected response"
print ""
print "Starting server 2010 1 1 SFF"
util.run("./server 2010 1 1 SFF")
print "Client: " + clientcmd
response = commands.getoutput(clientcmd)
if response.find("hey this is a test file") == -1:
util.error("ouchs! client can not get the right file")
print "Client got expected response"
print ""
print "Starting server 2010 1 1 SFF-BS"
util.run("./server 2010 1 1 SFF-BS 1")
print "Client: " + clientcmd
response = commands.getoutput(clientcmd)
if response.find("hey this is a test file") == -1:
util.error("ouchs! client can not get the right file")
print "Client got expected response"
util.good(" Your server worked with simple requests")
sys.exit(0)
def test(config):
cwd = os.getcwd()
os.chdir('test')
if not os.path.exists('shrc'):
raise Exception('no shrc found in directory %s' % testdir)
try:
f = open('shrc', 'r')
lines = f.readlines()
for line in lines:
(key, _, value) = line.partition('=')
os.environ[key.split()[1]] = value.rstrip().replace('"', '')
finally:
f.close()
util.make()
os.environ['LD_LIBRARY_PATH'] = os.path.join(cwd, 'lib')
util.make('test')
os.chdir(cwd)
def main():
# 1. check if make generate an executable server file
util.make()
# 2. ok you got an excutable server, run it
print ""
print "Starting ./server 2010 1 1 FIFO"
util.run("./server 2010 1 1 FIFO")
clientcmd = "./testclient localhost 2010 /testdata/testfile.txt"
print "Client: " + clientcmd
response = commands.getoutput(clientcmd)
if response.find("hey this is a test file") == -1:
util.error("ouchs! client can not get the right file")
print "Client got expected response"
print ""
print "Starting server 2010 1 1 SFF"
util.run("./server 2010 1 1 SFF")
print "Client: " + clientcmd
response = commands.getoutput(clientcmd)
if response.find("hey this is a test file") == -1:
util.error("ouchs! client can not get the right file")
print "Client got expected response"
print ""
print "Starting server 2010 1 1 SFF-BS"
util.run("./server 2010 1 1 SFF-BS 1")
print "Client: " + clientcmd
response = commands.getoutput(clientcmd)
if response.find("hey this is a test file") == -1:
util.error("ouchs! client can not get the right file")
print "Client got expected response"
util.good(" Your server worked with simple requests")
sys.exit(0)
def prepare_gold(version):
curdir = os.getcwd()
binutils_tarball = binutils_tarball_format.replace('VERSION', version)
binutils_url = binutils_url_format.replace('TARBALL', binutils_tarball)
util.download(binutils_url)
util.untar(binutils_tarball)
os.chdir('binutils-VERSION'.replace('VERSION', version))
util.configure(
'--prefix=%s --enable-gold --enable-plugins --disable-werror' % curdir)
util.make('-j4')
util.make('-j4 all-gold')
util.make('install')
os.chdir(curdir)
def run(self):
clientcmd = "./testclient localhost 2010 "
if self.id == 0:
file = "/output.cgi"
else:
file = "/testdata/file%s.txt" % str(self.id)
clientcmd = clientcmd + file + " >> /tmp/file" + str(self.id) + " &"
util.info(self.clientname + ": " + clientcmd)
os.system(clientcmd)
if self.id != 0:
os.system("./testclient localhost 2010 /output.cgi > /dev/null &")
util.make()
# test SFF server
for i in range(0, NUM_LOOPS):
print ""
print "#############################"
print "- %s iteration" % str(i)
print "#############################"
print ""
test("./server 2010 1 7 FIFO")
util.good("great! FIFO is done correctly")
sys.exit(0)
def run_test(testcase):
commands.getoutput("killall -9 server")
commands.getoutput("killall -9 testclient")
if int(testcase) == 1:
if os.path.exists("README") or os.path.exists("README.txt"):
return 0 # pass
print "ERROR: no README or README.txt"
return -1 # failed
if int(testcase) == 2:
if os.path.exists("Makefile") or os.path.exists("makefile"):
return 0
print "ERROR: no Makefile or makefile"
return -1
if int(testcase) == 3:
commands.getoutput("rm -rf server")
commands.getoutput("rm -rf libmfs.so")
commands.getoutput("make")
if not os.path.exists("server") or not os.path.exists("libmfs.so"):
print "ERROR: make does not generate server and libmfs.so"
return -1
return 0
if int(testcase) > 3 and int(testcase) <=39:
rc = util.make()
if rc == -1:
return -1
rc = util.compile_testclient(testcase)
if rc == -1:
return -1
rc = util.start_fresh_server(port)
if rc == -1:
return -1
status, out = commands.getstatusoutput("./testclient %s %s"
% (port, host))
# print out
commands.getoutput("killall -9 server")
return status
if int(testcase) == 40: # test a
rc = util.make()
if rc == -1:
return -1;
# start server with a fresh file
commands.getoutput("rm -rf fsimage")
commands.getoutput("rm -rf server-out.txt")
commands.getoutput("killall -9 server")
os.system("./server %s fsimage > /dev/null &" % port)
#time.sleep(1)
time.sleep(3) # added on 5/3 9am
rc = util.compile_testclient(str(testcase) + "pre")
if rc == -1:
return -1
status, out = commands.getstatusoutput("./testclient %s %s"
% (port, host))
# print out
if status == -1:
print "can not pass PRE test"
return -1
commands.getoutput("killall -9 server")
time.sleep(1) # added on 5/3 9am
os.system("./server %s fsimage > /dev/null &" % port)
time.sleep(0.5)
rc = util.compile_testclient(str(testcase) + "post")
if rc == -1:
return -1
status, out = commands.getstatusoutput("./testclient %s %s"
% (port, host))
# print out
commands.getoutput("killall -9 server")
return status
if int(testcase) >= 41:
rc = util.make()
if rc == -1:
return -1
rc = util.compile_testclient(testcase)
if rc == -1:
return -1
clientout = "out.txt"
commands.getoutput("rm -rf out.txt")
os.system("./testclient %s %s > %s &" % (port, host, clientout))
time.sleep(ctimeout/2)
rc = util.start_fresh_server(port)
if rc == -1:
return -1
time.sleep(ctimeout)
ps = commands.getoutput("ps -ax")
if ps.find("./testclient %s %s" % (port, host)) >= 0:
# the testclient is still running
print "ERROR: your client did not implement timeout correctly"
print "ERROR: testclient should have finished"
return -1
else: # now testclient finish, let check the return status
cat = commands.getoutput("cat %s" % clientout)
if cat.find("PASSED") >= 0:
return 0
return -1
def setup(config):
workdir = os.path.abspath(os.getcwd())
if not os.path.exists('dangsan'):
util.git.clone(dangsan_repo)
if not os.path.exists('llvm-svn'):
util.llvm.checkout_svn(llvm_svn_commit)
os.chdir('llvm-svn')
util.patch(
os.path.join(workdir, 'dangsan', 'patches',
'LLVM-gold-plugins-3.8.diff'), '-p0')
util.patch(
os.path.join(workdir, 'dangsan', 'patches',
'LLVM-safestack-3.8.diff'), '-p0')
os.chdir('projects/compiler-rt')
util.patch(
os.path.join(workdir, 'dangsan', 'patches',
'COMPILERRT-safestack-3.8.diff'), '-p0')
os.chdir(workdir)
if not os.path.exists('bin/ld'):
prepare_gold(binutils_version)
os.remove('bin/ld')
shutil.copy('bin/ld.gold', 'bin/ld')
if not os.path.exists('llvm-build'):
os.mkdir('llvm-build')
util.llvm.compile(
src_dir=os.path.abspath('llvm-svn'),
build_dir=os.path.abspath('llvm-build'),
install_dir=os.getcwd(),
options=['-DLLVM_BINUTILS_INCDIR=%s/include' % workdir])
util.llvm.install(os.path.abspath('llvm-build'))
if not glob.glob('gperftools*'):
util.git.clone(gperftools_repo)
os.chdir('gperftools')
util.git.checkout(gperftools_commit)
# gperftools patch for both baseline and DangSan
util.git.patch(
os.path.join(workdir, 'dangsan', 'patches',
'GPERFTOOLS_SPEEDUP.patch'))
if config == configs[1] or config == configs[2]:
os.chdir(workdir)
# Build metapagetable
if not os.path.exists('gperftools-metalloc'):
shutil.move('gperftools', 'gperftools-metalloc')
if not os.path.exists('metapagetable'):
config_fixedcompression = 'false'
config_metadatabytes = 8
config_deepmetadata = 'false'
config_alloc_size_hook = 'dang_alloc_size_hook'
metalloc_options = []
metalloc_options.append('-DFIXEDCOMPRESSION=%s' %
config_fixedcompression)
metalloc_options.append('-DMETADATABYTES=%d' %
config_metadatabytes)
metalloc_options.append('-DDEEPMETADATA=%s' %
config_deepmetadata)
metalloc_options.append('-DALLOC_SIZE_HOOK=%s' %
config_alloc_size_hook)
os.environ['METALLOC_OPTIONS'] = ' '.join(metalloc_options)
shutil.copytree(os.path.join('dangsan', 'metapagetable'),
'metapagetable')
os.chdir('metapagetable')
util.make('config')
util.make()
os.chdir(workdir)
# Apply DanSan patches for gperftool
os.chdir('gperftools-metalloc')
util.git.patch(
os.path.join(os.path.dirname(__file__),
'GPERFTOOLS_DANGSAN.patch'))
util.run('autoreconf -fi')
util.configure('--prefix=%s' % workdir)
util.make('-j4')
util.make('install')
os.chdir(workdir)
# build llvm-plugins
llvmplugins_dir = os.path.join(workdir, 'llvm-plugins')
libplugins_so = os.path.join(llvmplugins_dir, 'libplugins.so')
if config == configs[1] or config == configs[2]:
if not os.path.exists('staticlib'):
os.environ['PATH'] = ':'.join(
[os.path.join(workdir, 'bin'), os.environ['PATH']])
shutil.copytree('dangsan/staticlib', 'staticlib')
os.chdir('staticlib')
util.make('METAPAGETABLEDIR=%s' %
os.path.join(workdir, 'metapagetable', 'obj'))
os.chdir(workdir)
if not os.path.exists(libplugins_so):
os.environ['PATH'] = ':'.join(
[os.path.join(workdir, 'bin'), os.environ['PATH']])
os.chdir('dangsan/llvm-plugins')
util.make('-j4 GOLDINSTDIR=%s TARGETDIR=%s' %
(workdir, llvmplugins_dir))
os.chdir(workdir)
# Create spec config file
print 'creating spec config file...'
spec_cfg = '-'.join([command, config]) + '.cfg'
if os.path.exists(spec_cfg):
os.remove(spec_cfg)
cflags = ['-flto']
cxxflags = ['-flto']
ldflags = []
if config == configs[1] or config == configs[2]:
cflags.append('-fsanitize=safe-stack')
cxxflags.append('-fsanitize=safe-stack')
cxxflags.append('-DSOPLEX_DANGSAN_MASK')
ldflags.append('-Wl,-plugin-opt=-load=%s' % libplugins_so)
ldflags.append('-Wl,-plugin-opt=%s' % '-largestack=false')
# ldflags.append('-Wl,-plugin-opt=%s' % '-stats') # option not found
ldflags.append('-Wl,-plugin-opt=%s' % '-mergedstack=false')
ldflags.append('-Wl,-plugin-opt=%s' % '-stacktracker')
ldflags.append('-Wl,-plugin-opt=%s' % '-globaltracker')
ldflags.append('-Wl,-plugin-opt=%s' % '-pointertracker')
ldflags.append('-Wl,-plugin-opt=%s' % '-FreeSentryLoop')
ldflags.append('-Wl,-plugin-opt=%s' % '-custominline')
ldflags.append('-Wl,-whole-archive,-l:libmetadata.a,-no-whole-archive'
) # staticlib
ldflags.append(
'@%s' %
os.path.join(workdir, 'metapagetable', 'obj', 'linker-options'))
extra_libs = ['-ltcmalloc', '-lpthread', '-lunwind']
extra_libs.append('-L%s' % os.path.join(workdir, 'lib'))
if config == configs[1] or config == configs[2]:
extra_libs.append('-ldl')
extra_libs.append('-L%s' % os.path.join(workdir, 'staticlib', 'obj'))
if config == configs[2]:
cflags.append('-g')
cxxflags.append('-g')
cc = [os.path.abspath(os.path.join(workdir, 'bin', 'clang'))]
cc.extend(cflags)
cxx = [os.path.abspath(os.path.join(workdir, 'bin', 'clang++'))]
cxx.extend(cxxflags)
cld = list(cc)
cld.extend(ldflags)
cxxld = list(cxx)
cxxld.extend(ldflags)
util.spec.create_config(dir=os.getcwd(),
file=spec_cfg,
name='-'.join([command, config]),
cc=' '.join(cc),
cxx=' '.join(cxx),
cld=' '.join(cld),
cxxld=' '.join(cxxld),
extra_libs=' '.join(extra_libs))
# create shrc for testing
print('creating test directory...')
if not os.path.exists('test'):
testdir = os.path.join(os.path.dirname(__file__), 'test')
shutil.copytree(testdir, os.path.join(workdir, 'test'))
if os.path.exists('test/shrc'):
os.remove('test/shrc')
util.test.create_shrc(dir=os.path.abspath('test'),
cc=' '.join(cc),
cxx=' '.join(cxx),
ld=' '.join(cld),
ldflags=' '.join(extra_libs))
def run_test(testcase):
commands.getoutput("killall -9 server")
commands.getoutput("killall -9 testclient")
if int(testcase) == 1:
if os.path.exists("README") or os.path.exists("README.txt"):
return 0 # pass
print "ERROR: no README or README.txt"
return -1 # failed
if int(testcase) == 2:
if os.path.exists("Makefile") or os.path.exists("makefile"):
return 0
print "ERROR: no Makefile or makefile"
return -1
if int(testcase) == 3:
commands.getoutput("rm -rf server")
commands.getoutput("rm -rf libmfs.so")
commands.getoutput("make")
if not os.path.exists("server") or not os.path.exists("libmfs.so"):
print "ERROR: make does not generate server and libmfs.so"
return -1
return 0
if int(testcase) > 3 and int(testcase) <=39:
rc = util.make()
if rc == -1:
return -1
rc = util.compile_testclient(testcase)
if rc == -1:
return -1
rc = util.start_fresh_server(port)
if rc == -1:
return -1
status, out = commands.getstatusoutput("./testclient %s %s"
% (port, host))
# print out
commands.getoutput("killall -9 server")
return status
if int(testcase) == 40: # test a
rc = util.make()
if rc == -1:
return -1;
# start server with a fresh file
commands.getoutput("rm -rf fsimage")
commands.getoutput("rm -rf server-out.txt")
commands.getoutput("killall -9 server")
os.system("./server %s fsimage > /dev/null &" % port)
#time.sleep(1)
time.sleep(3) # added on 5/3 9am
rc = util.compile_testclient(str(testcase) + "pre")
if rc == -1:
return -1
status, out = commands.getstatusoutput("./testclient %s %s"
% (port, host))
# print out
if status == -1:
print "can not pass PRE test"
return -1
commands.getoutput("killall -9 server")
time.sleep(1) # added on 5/3 9am
os.system("./server %s fsimage > /dev/null &" % port)
time.sleep(0.5)
rc = util.compile_testclient(str(testcase) + "post")
if rc == -1:
return -1
status, out = commands.getstatusoutput("./testclient %s %s"
% (port, host))
# print out
commands.getoutput("killall -9 server")
return status
if int(testcase) >= 41:
rc = util.make()
if rc == -1:
return -1
rc = util.compile_testclient(testcase)
if rc == -1:
return -1
clientout = "out.txt"
commands.getoutput("rm -rf out.txt")
os.system("./testclient %s %s > %s &" % (port, host, clientout))
time.sleep(ctimeout/2)
rc = util.start_fresh_server(port)
if rc == -1:
return -1
time.sleep(ctimeout)
ps = commands.getoutput("ps -ax")
if ps.find("./testclient %s %s" % (port, host)) >= 0:
# the testclient is still running
print "ERROR: your client did not implement timeout correctly"
print "ERROR: testclient should have finished"
return -1
else: # now testclient finish, let check the return status
cat = commands.getoutput("cat %s" % clientout)
if cat.find("PASSED") >= 0:
return 0
return -1
def build(mode=None, optimize=None):
"""
Build SORD code.
"""
cf = util.namespace(configure.configure())
if not optimize:
optimize = cf.optimize
if not mode:
mode = cf.mode
if not mode:
mode = 'sm'
base = (
'globals.f90',
'diffcn.f90',
'diffnc.f90',
'hourglass.f90',
'bc.f90',
'surfnormals.f90',
'util.f90',
'frio.f90',
)
common = (
'arrays.f90',
'fieldio.f90',
'stats.f90',
'parameters.f90',
'setup.f90',
'gridgen.f90',
'attenuation.f90',
'material.f90',
'source.f90',
'inivolstress.f90',
'rupture.f90',
'resample.f90',
'checkpoint.f90',
'timestep.f90',
'stress.f90',
'acceleration.f90',
'sord.f90',
)
cwd = os.getcwd()
path = os.path.realpath(os.path.dirname(__file__))
f = os.path.join(path, 'bin')
if not os.path.isdir(f):
os.mkdir(f)
new = False
os.chdir(os.path.join(path, 'src'))
if 's' in mode:
source = base + ('serial.f90', ) + common
for opt in optimize:
object_ = os.path.join('..', 'bin', 'sord-s' + opt)
compiler = cf.fortran_serial + cf.fortran_flags[opt]
new |= util.make(compiler, object_, source)
if 'm' in mode and cf.fortran_mpi:
source = base + ('mpi.f90', ) + common
for opt in optimize:
object_ = os.path.join('..', 'bin', 'sord-m' + opt)
compiler = cf.fortran_mpi + cf.fortran_flags[opt]
new |= util.make(compiler, object_, source)
os.chdir(path)
if new:
try:
import bzrlib
except ImportError:
print(
'Warning: bzr not installed. Install bzr if you want to save a\
copy of the source code for posterity with each run.')
else:
os.system('bzr export sord.tgz')
os.chdir(cwd)
return
def setup(config):
workdir = os.path.abspath(os.getcwd())
if not os.path.exists('typesan'):
util.git.clone(typesan_repo)
if not os.path.exists('llvm'):
util.llvm.checkout(llvm_commit, clang_commit, compilerrt_commit)
if not os.path.exists('llvm-build'):
os.mkdir('llvm-build')
# Apply TypeSan patches
if config == configs[1]:
llvm_patch = os.path.join(os.path.dirname(__file__), 'llvm.patch')
clang_patch = os.path.join(os.path.dirname(__file__),
'clang.patch')
compilerrt_patch = os.path.join(os.path.dirname(__file__),
'compiler-rt.patch')
os.chdir('llvm')
util.git.patch(llvm_patch)
os.chdir(os.path.join(workdir, 'llvm/tools/clang'))
util.git.patch(clang_patch)
os.chdir(os.path.join(workdir, 'llvm/projects/compiler-rt'))
util.git.patch(compilerrt_patch)
os.chdir(workdir)
util.llvm.compile(src_dir=os.path.abspath('llvm'),
build_dir=os.path.abspath('llvm-build'),
install_dir=os.getcwd())
if not glob.glob('gperftools*'):
util.git.clone(gperftools_repo)
os.chdir('gperftools')
util.git.checkout(gperftools_commit)
util.git.patch(
os.path.join(os.path.dirname(__file__),
'GPERFTOOLS_SPEEDUP.patch'))
if config == configs[1]:
os.chdir(workdir)
# Build metapagetable
if not os.path.exists('gperftools-metalloc'):
shutil.move('gperftools', 'gperftools-metalloc')
if not os.path.exists('metapagetable'):
config_fixedcompression = 'false'
config_metadatabytes = 16
config_deepmetadata = 'false'
os.environ[
'METALLOC_OPTIONS'] = '-DFIXEDCOMPRESSION=%s -DMETADATABYTES=%d -DDEEPMETADATA=%s' % (
config_fixedcompression, config_metadatabytes,
config_deepmetadata)
shutil.copytree(os.path.join('typesan', 'metapagetable'),
'metapagetable')
os.chdir('metapagetable')
util.make('config')
util.make()
os.chdir(workdir)
# Apply TypeSan patches for gperftool
os.chdir('gperftools-metalloc')
util.git.patch(
os.path.join(os.path.dirname(__file__),
'GPERFTOOLS_TYPESAN.patch'))
util.run('autoreconf -fi')
util.configure('--prefix=' + workdir)
util.make()
util.make('install')
os.chdir(workdir)
# Create spec config file
print 'creating spec config file...'
spec_cfg = '-'.join([command, config]) + '.cfg'
if os.path.exists(spec_cfg):
os.remove(spec_cfg)
path = os.path.join('llvm-build', 'bin')
cflags = []
extra_libs = []
if config == configs[1]:
cflags = ['-fsanitize=typesan']
extra_libs.extend(['-ltcmalloc', '-lpthread', '-lunwind'])
extra_libs.append('-L' + os.path.join(workdir, 'lib')) # tcmalloc
cc = [os.path.abspath(os.path.join(path, 'clang'))]
cc.extend(cflags)
cxx = [os.path.abspath(os.path.join(path, 'clang++'))]
cxx.extend(cflags)
util.spec.create_config(dir=os.getcwd(),
file=spec_cfg,
name='-'.join([command, config]),
cc=' '.join(cc),
cxx=' '.join(cxx),
extra_libs=' '.join(extra_libs))
# create shrc for testing
print('creating test directory...')
if not os.path.exists('test'):
testdir = os.path.join(os.path.dirname(__file__), 'test')
shutil.copytree(testdir, os.path.join(workdir, 'test'))
if os.path.exists('test/shrc'):
os.remove('test/shrc')
util.test.create_shrc(dir=os.path.abspath('test'),
cc=' '.join(cc),
cxx=' '.join(cxx),
ld=' '.join(cxx),
ldflags=' '.join(extra_libs))
###############################################
def test(cmd):
print ""
print "- Starting " + cmd
util.run2(cmd)
if util.is_server_alive(cmd) != -1:
util.error("Ouch! Server is still alive!")
print ""
print "GOOD! ..."
print ""
util.make()
# test FIFO server
test("./server 2010 1 FIFO")
test("./server 1 2 FIFO")
# test SFF server
test("./server 2010 2 SFF")
test("./server 2010 2 1")
test("./server 1 SFF")
# test SFF-BS
test("./server 2010 2 2 SSF-BS")
sys.exit(0)
def setup(config):
workdir = os.path.abspath(os.getcwd())
configdir = os.path.join(os.path.dirname(__file__), 'config')
INSTRUMENTATION_PATH = os.path.abspath('llvm/lib/Transforms/Instrumentation')
if not os.path.exists('llvm'):
util.llvm.download(llvmver, os.getcwd())
# Apply patches common to both baseline and LowFat
bug81066_patch = os.path.join(os.path.dirname(__file__), 'bug81066.patch')
os.chdir('llvm/projects/compiler-rt')
util.patch(os.path.abspath(bug81066_patch), '-p0')
os.chdir(workdir)
if not os.path.exists('llvm-4.0.0') and not os.path.exists('llvm-lowfat'):
os.mkdir('llvm-4.0.0')
util.llvm.compile(src_dir=os.path.abspath('llvm'),
build_dir=os.path.abspath('llvm-4.0.0'),
install_dir=workdir)
if config != configs[0]:
os.environ['CC'] = os.path.abspath('llvm-4.0.0/bin/clang')
os.environ['CXX'] = os.path.abspath('llvm-4.0.0/bin/clang++')
if not os.path.exists('llvm-lowfat'):
print 'patching llvm/clang/compiler-rt version ' + llvmver + '...'
llvm_patch = os.path.join(os.path.dirname(__file__), 'llvm.patch')
clang_patch = os.path.join(os.path.dirname(__file__), 'clang.patch')
compilerrt_patch = os.path.join(os.path.dirname(__file__), 'compiler-rt.patch')
legacy_compilerrt_patch = os.path.join(os.path.dirname(__file__), 'legacy_compiler-rt.patch')
modern_compilerrt_patch = os.path.join(os.path.dirname(__file__), 'modern_compiler-rt.patch')
os.chdir('llvm')
util.patch(os.path.abspath(llvm_patch))
os.chdir(workdir)
os.chdir('llvm/tools/clang')
util.patch(os.path.abspath(clang_patch))
os.chdir(workdir)
os.chdir('llvm/projects/compiler-rt')
util.patch(os.path.abspath(compilerrt_patch))
cpuinfo = util.stdout(['cat', '/proc/cpuinfo']).strip().split('\n')
legacy = False
if not [s for s in cpuinfo if ' bmi1' in s]:
legacy = True
if not [s for s in cpuinfo if ' bmi2' in s]:
legacy = True
if not [s for s in cpuinfo if ' abm' in s]:
legacy = True
if legacy:
util.patch(os.path.abspath(legacy_compilerrt_patch))
else:
util.patch(os.path.abspath(modern_compilerrt_patch))
os.chdir(workdir)
print 'building the LowFat config builder...'
CONFIG = 'sizes.cfg 32'
os.chdir(os.path.join(os.path.dirname(__file__), 'config'))
util.make()
util.run('./lowfat-config ' + CONFIG)
util.make('lowfat-check-config')
util.run('./lowfat-check-config')
os.chdir(workdir)
print 'copying the LowFat config files...'
RUNTIME_PATH = os.path.abspath('llvm/projects/compiler-rt/lib/lowfat')
shutil.copy(os.path.join(configdir, 'lowfat_config.h'), RUNTIME_PATH)
shutil.copy(os.path.join(configdir, 'lowfat_config.c'), RUNTIME_PATH)
CLANGLIB_PATH = os.path.abspath('llvm/tools/clang/lib/Basic')
shutil.copy(os.path.join(RUNTIME_PATH, 'lowfat_config.c'),
os.path.join(INSTRUMENTATION_PATH, 'lowfat_config.inc'))
shutil.copy(os.path.join(RUNTIME_PATH, 'lowfat_config.h'),
os.path.join(INSTRUMENTATION_PATH, 'lowfat_config.h'))
shutil.copy(os.path.join(RUNTIME_PATH, 'lowfat_config.h'),
os.path.join(CLANGLIB_PATH, 'lowfat_config.h'))
shutil.copy(os.path.join(RUNTIME_PATH, 'lowfat.h'),
os.path.join(INSTRUMENTATION_PATH, 'lowfat.h'))
os.mkdir('llvm-lowfat')
util.llvm.compile(src_dir=os.path.abspath('llvm'),
build_dir=os.path.abspath('llvm-lowfat'),
install_dir=workdir)
os.chdir('llvm-lowfat')
if not os.path.exists('lib/LowFat'):
if not os.path.exists('lib'):
os.mkdir('lib')
os.mkdir('lib/LowFat')
shutil.copy(os.path.join(configdir, 'lowfat.ld'),
os.path.join('lib/LowFat', 'lowfat.ld'))
if not os.path.exists('bin/lowfat-ptr-info'):
os.chdir(configdir)
util.make('lowfat-ptr-info')
shutil.copy('lowfat-ptr-info', os.path.join(workdir, 'llvm-lowfat', 'bin'))
os.chdir(workdir)
if not os.path.exists('LowFat.so'):
clangxx = os.path.join('llvm-4.0.0', 'bin', 'clang++')
llvm_config = os.path.join('llvm-4.0.0', 'bin', 'llvm-config')
cxxflags = util.stdout([llvm_config, '--cxxflags']).rstrip('\n')
includedir = util.stdout([llvm_config, '--includedir']).rstrip('\n')
cmd = ' '.join([clangxx, '-DLOWFAT_PLUGIN',
os.path.join(INSTRUMENTATION_PATH, 'LowFat.cpp'),
'-c -Wall -O2',
'-I' + INSTRUMENTATION_PATH,
'-o LowFat.o',
cxxflags, includedir])
util.run(cmd)
ldflags = util.stdout([llvm_config, '--ldflags']).rstrip('\n')
cmd = ' '.join([clangxx, '-shared -rdynamic -o LowFat.so LowFat.o', ldflags])
util.run(cmd)
os.chdir(workdir)
# Create spec config file
print 'creating spec config file...'
spec_cfg = '-'.join([command, config]) + '.cfg'
if os.path.exists(spec_cfg):
os.remove(spec_cfg)
if config == configs[0]:
path = os.path.join('llvm-4.0.0', 'bin')
cflags = []
else:
path = os.path.join('llvm-lowfat', 'bin')
cflags = ['-fsanitize=lowfat']
blacklist = os.path.join(os.path.dirname(__file__), 'blacklist.txt')
cflags.extend(['-mllvm', '-lowfat-no-check-blacklist=%s' % blacklist])
cflags.append('-DPATCH_PERLBENCH_OVERFLOW')
cflags.append('-DPATCH_H264REF_OVERFLOW')
if config == configs[2]:
cflags.append('-g')
cflags.append('-fno-inline-functions')
if config == configs[3]:
cflags.append('-mllvm -lowfat-no-abort')
cc = [os.path.abspath(os.path.join(path, 'clang'))]
cc.extend(cflags)
cxx = [os.path.abspath(os.path.join(path, 'clang++'))]
cxx.extend(cflags)
util.spec.create_config(dir=os.getcwd(), file=spec_cfg,
name='-'.join([command, config]),
cc=' '.join(cc), cxx=' '.join(cxx))
