def keep_results(output, sout, serr, epmc): scpcmd = "scp " + ssh_master[0] + ":" + test_dir + "/" + sample_filename + " " + tc.get_odir() + "/sample.txt" tc.log_print(scpcmd) sp.check_call(scpcmd, shell=True) # parse the output file for percentile and average load qps, lat = mp.parse_mut_sample(tc.get_odir() + "/sample.txt") avg_qps = np.mean(qps) output = build_memparse(lat, qps) f = open(tc.get_odir() + "/" + epmc + ".txt", "w") f.write(output) f.close() tc.log_print(output) mvcmd = "mv " + tc.get_odir() + "/sample.txt " + tc.get_odir() + "/l" + str(int(avg_qps)) + ".sample" tc.log_print(mvcmd) sp.check_call(mvcmd, shell=True) if pmc: pmc_cmd = "pmcstat -R /tmp/tests.d/result.pmc -z 32 -G /tmp/tests.d/result.stack" tc.log_print(pmc_cmd) tc.remote_exec(ssh_server, pmc_cmd) scpcmd = "scp " + ssh_server[0] + ":" + test_dir + "/result.stack " + tc.get_odir() + "/" + epmc + ".stack" tc.log_print(scpcmd) sp.check_call(scpcmd, shell=True) if lockstat and len(serr) > 0: f = open(tc.get_odir() + "/l" + str(int(avg_qps)) + ".lstat", "w") f.write(serr) f.close() if dump and len (sout) > 0: f = open(tc.get_odir() + "/l" + str(int(avg_qps)) + ".kstat", "w") f.write(sout) f.close() return int(avg_qps)
def stop_all():
# stop clients
tc.log_print("Stopping clients...")
tc.remote_exec(clients, "sudo killall dismember", check=False)
if not client_only:
# stop server
tc.log_print("Stopping server...")
tc.remote_exec(server, "sudo killall ppd", check=False)
# stop master
tc.log_print("Stopping master...")
tc.remote_exec(master, "sudo killall dismember", check=False)
tc.log_print("Stopping hpc...")
tc.remote_exec(hpc, "sudo killall dismember", check=False)
def stop_all():
# stop clients
tc.log_print("Stopping clients...")
tc.remote_exec(clients_ssh, "killall -9 mutilate", check=False)
if not client_only:
# stop server
tc.log_print("Stopping server...")
tc.remote_exec(server_ssh, "killall -9 memcached", check=False)
# stop master
tc.log_print("Stopping master...")
tc.remote_exec(master_ssh, "killall -9 mutilate", check=False)
def stop_all():
# stop clients
tc.log_print("Stopping clients...")
tc.remote_exec(clients_ssh, "killall -9 dismember", check=False)
if not client_only:
# stop server
tc.log_print("Stopping server...")
tc.remote_exec(
server_ssh,
"sudo killall -9 httpd; sudo killall -9 nginx; sudo killall -9 lockstat",
check=False)
# stop master
tc.log_print("Stopping master...")
tc.remote_exec(master_ssh, "killall -9 dismember", check=False)
def run_exp(sc, ld, lstat, epmc):
while True:
if client_only:
ssrv = None
else:
# start server
tc.log_print("Starting server...")
server_cmd = test_dir + "/pingpong/build/ppd -a -t " + str(threads) + " -p " + str(server_port) + " -OENTRIES=" + str(cacheline) + " -M 0"
if pmc:
server_cmd = "sudo pmcstat -S " + epmc + " -n 1000 -O /tmp/tests.d/result.pmc " + test_dir + \
"/pingpong/build/ppd -a -t" + str(threads) + " -p " + str(server_port) + " -M 0 "
if lstat:
server_cmd = "sudo lockstat -A -P -n65536 " + server_cmd
if sc != -1:
server_cmd = server_cmd + " -m " + str(sc)
if dump:
server_cmd += " -d 1 "
tc.log_print(server_cmd)
ssrv = tc.remote_exec(ssh_server, server_cmd, blocking=False)
# start clients
tc.log_print("Starting clients...")
client_cmd = tc.get_cpuset_core(client_threads) + " " + test_dir + "/pingpong/build/dismember -A"
tc.log_print(client_cmd)
sclt = tc.remote_exec(ssh_clients, client_cmd, blocking=False)
time.sleep(1)
# start master
tc.log_print("Starting master...")
master_cmd = tc.get_cpuset_core(client_threads) + " " + test_dir + "/pingpong/build/dismember " + \
get_client_str(clients) + \
" -s " + server[0] + \
" -p " + str(server_port) + \
" -q " + str(ld) + \
" -c " + str(conn_per_thread) + \
" -o " + test_dir + "/" + sample_filename + \
" -t " + str(client_threads) + \
" -w " + str(duration) + \
" -W " + str(warmup) + \
" -T " + str(client_threads) + \
" -i exponential " + \
" -C " + str(client_threads) + \
" -Q 1000 " + \
" -l 0 " + \
" -OGEN=fixed:" + str(cacheline) + \
" -OUPDATE=0"
tc.log_print(master_cmd)
sp = tc.remote_exec(ssh_master, master_cmd, blocking=False)
p = sp[0]
success = False
cur = 0
while True:
# either failed or timeout
# we use failure detection to save time for long durations
if False \
or not tc.scan_stderr(sp, exclude=[".*warn.*", ".*WARN.*", ".*DEBUG.*"]) \
or not tc.scan_stderr(ssrv, exclude=[".*warn.*", ".*WARN.*", ".*DEBUG.*"]) \
or not tc.scan_stderr(sclt, exclude=[".*warn.*", ".*WARN.*", ".*DEBUG.*"]) \
or cur >= int(warmup + duration) * 2 \
:
break
if p.poll() != None:
success = True
break
time.sleep(1)
cur = cur + 1
stop_all()
print("Cooling down...")
time.sleep(cooldown)
output = p.communicate()[0].decode(sys.getfilesystemencoding())
if not client_only:
stdout, stderr = ssrv[0].communicate()
stdout = stdout.decode(sys.getfilesystemencoding())
stderr = stderr.decode(sys.getfilesystemencoding())
else:
stdout = ""
stderr = ""
if success:
# return mout, sout, serr (master output, server output, server serr)
return output, stdout, stderr
def run_exp(sc, ld, lstat):
while True:
if client_only:
ssrv = None
else:
# start server
tc.log_print("Starting server...")
server_cmd = None
if sc == -1:
server_cmd = tc.get_cpuset_core(
threads
) + " " + test_dir + "/memcached/memcached -m 1024 -c 65536 -b 4096 -t " + str(
threads)
elif sc == -2:
server_cmd = "limit core 0; sudo " + tc.get_cpuset_core(threads) + " " + \
test_dir + "/memcached_linox/memcached -u oscar -m 1024 -c 65536 -b 4096 -t " + str(threads)
elif sc == -3:
server_cmd = "limit core 0; sudo " + tc.get_cpuset_core(threads) + " " + test_dir + "/memcached-A/memcached -u oscar -m 1024 -c 65536 -b 4096 -t 1 " + \
"--minNumCores 2 --maxNumCores " + str(threads - 1)
else:
server_cmd = test_dir + "/mem/memcached -e -m 1024 -c 65536 -b 4096 -t " + str(
threads) + " -q " + str(sc) + (" -j 1 " if dump else "")
if lstat:
server_cmd = "sudo lockstat -A -P -s16 -n16777216 " + server_cmd + " -u " + tc.get_username(
)
tc.log_print(server_cmd)
ssrv = tc.remote_exec(server_ssh, server_cmd, blocking=False)
# pre-load server
time.sleep(1)
tc.log_print("Preloading server...")
preload_cmd = test_dir + "/mutilate/mutilate --loadonly -s localhost"
tc.log_print(preload_cmd)
tc.remote_exec(server_ssh, preload_cmd, blocking=True)
time.sleep(1)
# start clients
tc.log_print("Starting clients...")
client_cmd = tc.get_cpuset_core(
client_threads
) + " " + test_dir + "/mutilate/mutilate -A -T " + str(client_threads)
tc.log_print(client_cmd)
sclt = tc.remote_exec(clients_ssh, client_cmd, blocking=False)
time.sleep(1)
# start master
tc.log_print("Starting master...")
master_cmd = test_dir + "/mutilate/mutilate --noload -K fb_key -V fb_value -i fb_ia -u 0.03 -Q 1000 " + \
" -T " + str(client_threads) + \
" -C 1 " + \
" -c " + str(conn_per_thread) + \
" -w " + str(warmup) + \
" -t " + str(duration) + \
" -s " + server[0] + " " + get_client_str(clients) + \
" -q " + str(ld) + \
" --save " + test_dir + "/" + sample_filename
tc.log_print(master_cmd)
sp = tc.remote_exec(master_ssh, master_cmd, blocking=False)
p = sp[0]
success = False
cur = 0
while True:
# either failed or timeout
# we use failure detection to save time for long durations
if False \
or not tc.scan_stderr(ssrv, exclude=[".*warn.*", ".*DEBUG.*", ".*"]) \
or not tc.scan_stderr(sclt) \
or cur >= int(warmup + duration) * 3 \
or not tc.scan_stderr(sp, exclude=[".*mutex.hpp.*"]) \
:
break
if p.poll() != None:
success = True
break
time.sleep(1)
cur = cur + 1
stop_all()
tc.log_print("Cooling down...")
time.sleep(cooldown)
output = p.communicate()[0].decode(sys.getfilesystemencoding())
if not client_only:
stdout, stderr = ssrv[0].communicate()
stdout = stdout.decode(sys.getfilesystemencoding())
stderr = stderr.decode(sys.getfilesystemencoding())
else:
stdout = ""
stderr = ""
if success:
return output, stdout, stderr
def run_exp(sc, ld, lstat):
while True:
if client_only:
ssrv = None
else:
# start server
tc.log_print("Starting server...")
if sc == -2:
server_cmd = "sudo cpuset -l 0-23 nginx -c " + test_dir + "/scripts/wrk/nginx.conf"
else:
server_cmd = test_dir + "/celestis/build/tools/httpd/httpd -w " + str(
threads) + " -p " + str(
server_port) + " -d " + test_dir + "/scripts/wrk/"
if lstat:
server_cmd = "sudo lockstat -A -P -s4 -n16777216 " + server_cmd
if sc != -1:
server_cmd = server_cmd + " -m " + str(sc)
else:
server_cmd = server_cmd + " -l "
if dump:
server_cmd = server_cmd + " -D "
tc.log_print(server_cmd)
ssrv = tc.remote_exec(server_ssh, server_cmd, blocking=False)
# start clients
tc.log_print("Starting clients...")
client_cmd = tc.get_cpuset_core(
client_threads) + " " + test_dir + "/pingpong/build/dismember -A"
tc.log_print(client_cmd)
sclt = tc.remote_exec(clients_ssh, client_cmd, blocking=False)
time.sleep(1)
# start master
tc.log_print("Starting master...")
master_cmd = tc.get_cpuset_core(client_threads) + " " + test_dir + "/pingpong/build/dismember " + \
get_client_str(clients) + \
" -s " + server[0] + \
" -q " + str(ld) + \
" -p " + str(server_port) + \
" -c " + str(conn_per_thread) + \
" -o " + test_dir + "/" + sample_filename + \
" -t " + str(client_threads) + \
" -w " + str(duration) + \
" -W " + str(warmup) + \
" -T " + str(client_threads) + \
" -i fb_ia " + \
" -C " + str(conn_per_thread) + \
" -Q 1000 " + \
" -l 3 "
tc.log_print(master_cmd)
sp = tc.remote_exec(master_ssh, master_cmd, blocking=False)
p = sp[0]
success = False
cur = 0
while True:
# either failed or timeout
# we use failure detection to save time for long durations
if False \
or not tc.scan_stderr(sp, exclude=[".*warn.*", ".*WARN.*", ".*DEBUG.*"]) \
or not tc.scan_stderr(ssrv, exclude=[".*warn.*", ".*WARN.*", ".*DEBUG.*", ".*Bad file descriptor.*"]) \
or not tc.scan_stderr(sclt, exclude=[".*warn.*", ".*WARN.*", ".*DEBUG.*"]) \
or cur >= int(warmup + duration) * 2 \
:
break
if p.poll() != None:
success = True
break
time.sleep(1)
cur = cur + 1
stop_all()
print("Cooling down...")
time.sleep(cooldown)
output = p.communicate()[0].decode(sys.getfilesystemencoding())
if not client_only:
stdout, stderr = ssrv[0].communicate()
stdout = stdout.decode(sys.getfilesystemencoding())
stderr = stderr.decode(sys.getfilesystemencoding())
else:
stdout = ""
stderr = ""
if success:
# return mout, sout, serr (master output, server output, server serr)
return output, stdout, stderr
def run_exp(sc, ld, lstat):
while True:
if client_only:
ssrv = None
else:
# start server
tc.log_print("Starting server...")
server_cmd = test_dir + "/pingpong/build/ppd -a -t " + str(
threads) + " -p " + str(server_port) + " -M 0 -F 100000 "
if priority:
server_cmd += " -r " + (hpc[0] if measure_lp else master[0])
if sc != -1:
server_cmd = server_cmd + " -m " + str(sc)
tc.log_print(server_cmd)
ssrv = tc.remote_exec(server, server_cmd, blocking=False)
# start clients
tc.log_print("Starting clients...")
client_cmd = tc.get_cpuset_core(
client_threads) + " " + test_dir + "/pingpong/build/dismember -A"
tc.log_print(client_cmd)
sclt = tc.remote_exec(clients, client_cmd, blocking=False)
hpclt = None
time.sleep(1)
# start hpc
tc.log_print("Starting hpc...")
hpc_cmd = tc.get_cpuset_core(client_threads) + " " + test_dir + "/pingpong/build/dismember " + \
" -s " + server[0] + \
" -p " + str(server_port) + \
" -q 30000 " + \
" -c 1 " + \
" -t " + str(client_threads) + \
" -w 999" + \
" -W 0" + \
" -i exponential " + \
" -OGEN=fixed:0 " + \
" -OCDELAY=0" + \
" -l 0"
tc.log_print(hpc_cmd)
hpclt = tc.remote_exec(hpc, hpc_cmd, blocking=False)
time.sleep(1)
# start master
tc.log_print("Starting master...")
master_cmd = tc.get_cpuset_core(client_threads) + " " + test_dir + "/pingpong/build/dismember " + \
get_client_str(clients) + \
" -s " + server[0] + \
" -p " + str(server_port) + \
" -q " + str(ld) + \
" -c " + str(conn_per_thread) + \
" -o " + test_dir + "/" + sample_filename + \
" -t " + str(client_threads) + \
" -w " + str(duration) + \
" -W " + str(warmup) + \
" -T " + str(client_threads) + \
" -i exponential " + \
" -C 1" + \
" -Q 30000" + \
" -l 0 " + \
" -OGEN=fixed:0 " + \
" -OCDELAY=" + ("1" if conn_delay else "0")
tc.log_print(master_cmd)
sp = tc.remote_exec(master, master_cmd, blocking=False)
p = sp[0]
success = False
cur = 0
while True:
# either failed or timeout
# we use failure detection to save time for long durations
#or (measure_lp and (not tc.scan_stderr(hpclt, exclude=[".*warn.*", ".*WARN.*", ".*DEBUG.*"]))) \
if False \
or not tc.scan_stderr(sp, exclude=[".*warn.*", ".*WARN.*", ".*DEBUG.*"]) \
or not tc.scan_stderr(ssrv, exclude=[".*warn.*", ".*WARN.*", ".*DEBUG.*"]) \
or not tc.scan_stderr(sclt, exclude=[".*warn.*", ".*WARN.*", ".*DEBUG.*"]) \
or cur >= int(warmup + duration) * 2 \
:
break
if p.poll() != None:
success = True
break
time.sleep(1)
cur = cur + 1
stop_all()
print("Cooling down...")
time.sleep(cooldown)
output = p.communicate()[0].decode(sys.getfilesystemencoding())
if not client_only:
stdout, stderr = ssrv[0].communicate()
stdout = stdout.decode(sys.getfilesystemencoding())
stderr = stderr.decode(sys.getfilesystemencoding())
else:
stdout = ""
stderr = ""
if success:
# return mout, sout, serr (master output, server output, server serr)
return output, stdout, stderr