def keep_results(output, sout, serr):
scpcmd = "scp " + master[
0] + ":" + test_dir + "/" + sample_filename + " " + tc.get_odir(
) + "/sample.txt"
tc.log_print(scpcmd)
sp.check_call(scpcmd, shell=True)
# parse the output file for percentile and average load
qps, lat = mp.parse_mut_sample(tc.get_odir() + "/sample.txt")
avg_qps = np.mean(qps)
output = build_memparse(lat, qps)
f = open(tc.get_odir() + "/l" + str(int(avg_qps)) + ".txt", "w")
f.write(output)
f.close()
tc.log_print(output)
mvcmd = "mv " + tc.get_odir() + "/sample.txt " + tc.get_odir(
) + "/l" + str(int(avg_qps)) + ".sample"
tc.log_print(mvcmd)
sp.check_call(mvcmd, shell=True)
if lockstat and len(serr) > 0:
f = open(tc.get_odir() + "/l" + str(int(avg_qps)) + ".lstat", "w")
f.write(serr)
f.close()
if dump and len(sout) > 0:
f = open(tc.get_odir() + "/l" + str(int(avg_qps)) + ".kstat", "w")
f.write(sout)
f.close()
return int(avg_qps)
def keep_results(output, sout, serr, epmc): scpcmd = "scp " + ssh_master[0] + ":" + test_dir + "/" + sample_filename + " " + tc.get_odir() + "/sample.txt" tc.log_print(scpcmd) sp.check_call(scpcmd, shell=True) # parse the output file for percentile and average load qps, lat = mp.parse_mut_sample(tc.get_odir() + "/sample.txt") avg_qps = np.mean(qps) output = build_memparse(lat, qps) f = open(tc.get_odir() + "/" + epmc + ".txt", "w") f.write(output) f.close() tc.log_print(output) mvcmd = "mv " + tc.get_odir() + "/sample.txt " + tc.get_odir() + "/l" + str(int(avg_qps)) + ".sample" tc.log_print(mvcmd) sp.check_call(mvcmd, shell=True) if pmc: pmc_cmd = "pmcstat -R /tmp/tests.d/result.pmc -z 32 -G /tmp/tests.d/result.stack" tc.log_print(pmc_cmd) tc.remote_exec(ssh_server, pmc_cmd) scpcmd = "scp " + ssh_server[0] + ":" + test_dir + "/result.stack " + tc.get_odir() + "/" + epmc + ".stack" tc.log_print(scpcmd) sp.check_call(scpcmd, shell=True) if lockstat and len(serr) > 0: f = open(tc.get_odir() + "/l" + str(int(avg_qps)) + ".lstat", "w") f.write(serr) f.close() if dump and len (sout) > 0: f = open(tc.get_odir() + "/l" + str(int(avg_qps)) + ".kstat", "w") f.write(sout) f.close() return int(avg_qps)
def keep_results(ld, output, sout, serr):
f = open(tc.get_odir() + "/l" + ld + ".txt", "w")
f.write(output)
f.close()
scpcmd = "scp " + master_ssh[
0] + ":" + test_dir + "/" + sample_filename + " " + tc.get_odir(
) + "/l" + ld + ".sample"
tc.log_print(scpcmd)
sp.check_call(scpcmd, shell=True)
if lockstat and len(sout) > 0:
f = open(
tc.get_odir() + "/l" + ld + (".lstat" if lockstat else ".truss"),
"w")
f.write(sout)
f.close()
if dump and len(sout) > 0:
f = open(tc.get_odir() + "/l" + ld + ".kstat", "w")
f.write(sout)
f.close()
def stop_all():
# stop clients
tc.log_print("Stopping clients...")
tc.remote_exec(clients, "sudo killall dismember", check=False)
if not client_only:
# stop server
tc.log_print("Stopping server...")
tc.remote_exec(server, "sudo killall ppd", check=False)
# stop master
tc.log_print("Stopping master...")
tc.remote_exec(master, "sudo killall dismember", check=False)
tc.log_print("Stopping hpc...")
tc.remote_exec(hpc, "sudo killall dismember", check=False)
def stop_all():
# stop clients
tc.log_print("Stopping clients...")
tc.remote_exec(clients_ssh, "killall -9 mutilate", check=False)
if not client_only:
# stop server
tc.log_print("Stopping server...")
tc.remote_exec(server_ssh, "killall -9 memcached", check=False)
# stop master
tc.log_print("Stopping master...")
tc.remote_exec(master_ssh, "killall -9 mutilate", check=False)
def stop_all():
# stop clients
tc.log_print("Stopping clients...")
tc.remote_exec(clients_ssh, "killall -9 dismember", check=False)
if not client_only:
# stop server
tc.log_print("Stopping server...")
tc.remote_exec(
server_ssh,
"sudo killall -9 httpd; sudo killall -9 nginx; sudo killall -9 lockstat",
check=False)
# stop master
tc.log_print("Stopping master...")
tc.remote_exec(master_ssh, "killall -9 dismember", check=False)
def run_exp(sc, ld, lstat, epmc):
while True:
if client_only:
ssrv = None
else:
# start server
tc.log_print("Starting server...")
server_cmd = test_dir + "/pingpong/build/ppd -a -t " + str(threads) + " -p " + str(server_port) + " -OENTRIES=" + str(cacheline) + " -M 0"
if pmc:
server_cmd = "sudo pmcstat -S " + epmc + " -n 1000 -O /tmp/tests.d/result.pmc " + test_dir + \
"/pingpong/build/ppd -a -t" + str(threads) + " -p " + str(server_port) + " -M 0 "
if lstat:
server_cmd = "sudo lockstat -A -P -n65536 " + server_cmd
if sc != -1:
server_cmd = server_cmd + " -m " + str(sc)
if dump:
server_cmd += " -d 1 "
tc.log_print(server_cmd)
ssrv = tc.remote_exec(ssh_server, server_cmd, blocking=False)
# start clients
tc.log_print("Starting clients...")
client_cmd = tc.get_cpuset_core(client_threads) + " " + test_dir + "/pingpong/build/dismember -A"
tc.log_print(client_cmd)
sclt = tc.remote_exec(ssh_clients, client_cmd, blocking=False)
time.sleep(1)
# start master
tc.log_print("Starting master...")
master_cmd = tc.get_cpuset_core(client_threads) + " " + test_dir + "/pingpong/build/dismember " + \
get_client_str(clients) + \
" -s " + server[0] + \
" -p " + str(server_port) + \
" -q " + str(ld) + \
" -c " + str(conn_per_thread) + \
" -o " + test_dir + "/" + sample_filename + \
" -t " + str(client_threads) + \
" -w " + str(duration) + \
" -W " + str(warmup) + \
" -T " + str(client_threads) + \
" -i exponential " + \
" -C " + str(client_threads) + \
" -Q 1000 " + \
" -l 0 " + \
" -OGEN=fixed:" + str(cacheline) + \
" -OUPDATE=0"
tc.log_print(master_cmd)
sp = tc.remote_exec(ssh_master, master_cmd, blocking=False)
p = sp[0]
success = False
cur = 0
while True:
# either failed or timeout
# we use failure detection to save time for long durations
if False \
or not tc.scan_stderr(sp, exclude=[".*warn.*", ".*WARN.*", ".*DEBUG.*"]) \
or not tc.scan_stderr(ssrv, exclude=[".*warn.*", ".*WARN.*", ".*DEBUG.*"]) \
or not tc.scan_stderr(sclt, exclude=[".*warn.*", ".*WARN.*", ".*DEBUG.*"]) \
or cur >= int(warmup + duration) * 2 \
:
break
if p.poll() != None:
success = True
break
time.sleep(1)
cur = cur + 1
stop_all()
print("Cooling down...")
time.sleep(cooldown)
output = p.communicate()[0].decode(sys.getfilesystemencoding())
if not client_only:
stdout, stderr = ssrv[0].communicate()
stdout = stdout.decode(sys.getfilesystemencoding())
stderr = stderr.decode(sys.getfilesystemencoding())
else:
stdout = ""
stderr = ""
if success:
# return mout, sout, serr (master output, server output, server serr)
return output, stdout, stderr
def main():
global hostfile
global server
global master
global clients
global dump
global lockstat
global client_only
options = getopt.getopt(sys.argv[1:], 'h:sldcp')[0]
for opt, arg in options:
if opt in ('-h'):
hostfile = arg
elif opt in ('-s'):
stop_all()
return
elif opt in ('-l'):
lockstat=True
elif opt in ('-d'):
dump=True
elif opt in ('-c'):
client_only=True
tc.init("~/results.d/rss/" + str(threads) + "+" + str(len(clients)) + "x" + str(client_threads) + "x" + str(conn_per_thread))
tc.log_print("Configuration:\n" + \
"Hostfile: " + ("None" if hostfile == None else hostfile) + "\n" \
"Lockstat: " + str(lockstat) + "\n" \
"KQ dump: " + str(dump) + "\n" \
"Client only: " + str(client_only) + "\n")
if hostfile != None:
hosts = tc.parse_hostfile(hostfile)
server = tc.process_hostnames(server, hosts)
clients = tc.process_hostnames(clients, hosts)
master = tc.process_hostnames(master, hosts)
stop_all()
for i in range(0, len(sched), 2):
esched = sched[i+1]
ename = sched[i]
tc.begin(ename)
stop_all()
for j in range(0, len(pmc)):
epmc = pmc[j]
tc.log_print("============ Sched: " + str(ename) + " Flag: " + format(esched, '#04x') + " PMC: " + epmc + " Load: " + str(qps) + " ============")
output, sout, serr = run_exp(esched, qps, lockstat, epmc)
keep_results(output, sout, serr, epmc)
tc.log_print("")
tc.end()
stop_all()
def main():
global hostfile
global server
global clients
global dump
global lockstat
global client_only
global priority
global master
options = getopt.getopt(sys.argv[1:], 'h:sldcp')[0]
for opt, arg in options:
if opt in ('-h'):
hostfile = arg
elif opt in ('-s'):
stop_all()
return
elif opt in ('-l'):
lockstat = True
elif opt in ('-d'):
dump = True
elif opt in ('-c'):
client_only = True
elif opt in ('-p'):
priority = True
tc.init("~/results.d/pingpong/" + str(threads) + "+" + str(len(clients)) +
"x" + str(client_threads) + "x" + str(conn_per_thread))
tc.log_print("Configuration:\n" + \
"Hostfile: " + ("None" if hostfile == None else hostfile) + "\n" \
"Lockstat: " + str(lockstat) + "\n" \
"KQ dump: " + str(dump) + "\n" \
"Client only: " + str(client_only) + "\n" + \
"Conn delay: " + str(conn_delay) + "\n"
"Priority: " + str(priority) + "\n")
if hostfile != None:
hosts = tc.parse_hostfile(hostfile)
server = tc.process_hostnames(server, hosts)
clients = tc.process_hostnames(clients, hosts)
master = tc.process_hostnames(master, hosts)
stop_all()
for i in range(0, len(sched), 2):
esched = sched[i + 1]
ename = sched[i]
step_mul = 100
last_load = 0
cur_load = init_step
tc.begin(ename)
tc.log_print("============ Sched: " + str(ename) + " Flag: " +
format(esched, '#04x') + " Load: MAX" + " ============")
output, sout, serr = run_exp(esched, 600000, lockstat)
keep_results(output, sout, serr)
stop_all()
while True:
tc.log_print("============ Sched: " + str(ename) + " Flag: " +
format(esched, '#04x') + " Load: " + str(cur_load) +
" ============")
output, sout, serr = run_exp(esched, cur_load, lockstat)
qps = keep_results(output, sout, serr)
pct = int((qps - last_load) / init_step * 100)
tc.log_print("last_load: " + str(last_load) + " this_load: " +
str(qps) + " inc_pct: " + str(pct) + "%")
if pct <= term_pct:
tc.log_print("inc_pct less than TERM_PCT " + str(term_pct) +
"%. Done.")
break
if pct <= inc_pct:
step_mul += step_inc_pct
tc.log_print("inc_pct less than INC_PCT " + str(inc_pct) +
"%. Increasing step multiplier to " +
str(step_mul) + "%")
last_load = qps
cur_load += int(init_step * step_mul / 100)
tc.log_print("")
tc.end()
stop_all()
def run_exp(sc, ld, lstat):
while True:
if client_only:
ssrv = None
else:
# start server
tc.log_print("Starting server...")
server_cmd = None
if sc == -1:
server_cmd = tc.get_cpuset_core(
threads
) + " " + test_dir + "/memcached/memcached -m 1024 -c 65536 -b 4096 -t " + str(
threads)
elif sc == -2:
server_cmd = "limit core 0; sudo " + tc.get_cpuset_core(threads) + " " + \
test_dir + "/memcached_linox/memcached -u oscar -m 1024 -c 65536 -b 4096 -t " + str(threads)
elif sc == -3:
server_cmd = "limit core 0; sudo " + tc.get_cpuset_core(threads) + " " + test_dir + "/memcached-A/memcached -u oscar -m 1024 -c 65536 -b 4096 -t 1 " + \
"--minNumCores 2 --maxNumCores " + str(threads - 1)
else:
server_cmd = test_dir + "/mem/memcached -e -m 1024 -c 65536 -b 4096 -t " + str(
threads) + " -q " + str(sc) + (" -j 1 " if dump else "")
if lstat:
server_cmd = "sudo lockstat -A -P -s16 -n16777216 " + server_cmd + " -u " + tc.get_username(
)
tc.log_print(server_cmd)
ssrv = tc.remote_exec(server_ssh, server_cmd, blocking=False)
# pre-load server
time.sleep(1)
tc.log_print("Preloading server...")
preload_cmd = test_dir + "/mutilate/mutilate --loadonly -s localhost"
tc.log_print(preload_cmd)
tc.remote_exec(server_ssh, preload_cmd, blocking=True)
time.sleep(1)
# start clients
tc.log_print("Starting clients...")
client_cmd = tc.get_cpuset_core(
client_threads
) + " " + test_dir + "/mutilate/mutilate -A -T " + str(client_threads)
tc.log_print(client_cmd)
sclt = tc.remote_exec(clients_ssh, client_cmd, blocking=False)
time.sleep(1)
# start master
tc.log_print("Starting master...")
master_cmd = test_dir + "/mutilate/mutilate --noload -K fb_key -V fb_value -i fb_ia -u 0.03 -Q 1000 " + \
" -T " + str(client_threads) + \
" -C 1 " + \
" -c " + str(conn_per_thread) + \
" -w " + str(warmup) + \
" -t " + str(duration) + \
" -s " + server[0] + " " + get_client_str(clients) + \
" -q " + str(ld) + \
" --save " + test_dir + "/" + sample_filename
tc.log_print(master_cmd)
sp = tc.remote_exec(master_ssh, master_cmd, blocking=False)
p = sp[0]
success = False
cur = 0
while True:
# either failed or timeout
# we use failure detection to save time for long durations
if False \
or not tc.scan_stderr(ssrv, exclude=[".*warn.*", ".*DEBUG.*", ".*"]) \
or not tc.scan_stderr(sclt) \
or cur >= int(warmup + duration) * 3 \
or not tc.scan_stderr(sp, exclude=[".*mutex.hpp.*"]) \
:
break
if p.poll() != None:
success = True
break
time.sleep(1)
cur = cur + 1
stop_all()
tc.log_print("Cooling down...")
time.sleep(cooldown)
output = p.communicate()[0].decode(sys.getfilesystemencoding())
if not client_only:
stdout, stderr = ssrv[0].communicate()
stdout = stdout.decode(sys.getfilesystemencoding())
stderr = stderr.decode(sys.getfilesystemencoding())
else:
stdout = ""
stderr = ""
if success:
return output, stdout, stderr
def run_exp(sc, ld, lstat):
while True:
if client_only:
ssrv = None
else:
# start server
tc.log_print("Starting server...")
if sc == -2:
server_cmd = "sudo cpuset -l 0-23 nginx -c " + test_dir + "/scripts/wrk/nginx.conf"
else:
server_cmd = test_dir + "/celestis/build/tools/httpd/httpd -w " + str(
threads) + " -p " + str(
server_port) + " -d " + test_dir + "/scripts/wrk/"
if lstat:
server_cmd = "sudo lockstat -A -P -s4 -n16777216 " + server_cmd
if sc != -1:
server_cmd = server_cmd + " -m " + str(sc)
else:
server_cmd = server_cmd + " -l "
if dump:
server_cmd = server_cmd + " -D "
tc.log_print(server_cmd)
ssrv = tc.remote_exec(server_ssh, server_cmd, blocking=False)
# start clients
tc.log_print("Starting clients...")
client_cmd = tc.get_cpuset_core(
client_threads) + " " + test_dir + "/pingpong/build/dismember -A"
tc.log_print(client_cmd)
sclt = tc.remote_exec(clients_ssh, client_cmd, blocking=False)
time.sleep(1)
# start master
tc.log_print("Starting master...")
master_cmd = tc.get_cpuset_core(client_threads) + " " + test_dir + "/pingpong/build/dismember " + \
get_client_str(clients) + \
" -s " + server[0] + \
" -q " + str(ld) + \
" -p " + str(server_port) + \
" -c " + str(conn_per_thread) + \
" -o " + test_dir + "/" + sample_filename + \
" -t " + str(client_threads) + \
" -w " + str(duration) + \
" -W " + str(warmup) + \
" -T " + str(client_threads) + \
" -i fb_ia " + \
" -C " + str(conn_per_thread) + \
" -Q 1000 " + \
" -l 3 "
tc.log_print(master_cmd)
sp = tc.remote_exec(master_ssh, master_cmd, blocking=False)
p = sp[0]
success = False
cur = 0
while True:
# either failed or timeout
# we use failure detection to save time for long durations
if False \
or not tc.scan_stderr(sp, exclude=[".*warn.*", ".*WARN.*", ".*DEBUG.*"]) \
or not tc.scan_stderr(ssrv, exclude=[".*warn.*", ".*WARN.*", ".*DEBUG.*", ".*Bad file descriptor.*"]) \
or not tc.scan_stderr(sclt, exclude=[".*warn.*", ".*WARN.*", ".*DEBUG.*"]) \
or cur >= int(warmup + duration) * 2 \
:
break
if p.poll() != None:
success = True
break
time.sleep(1)
cur = cur + 1
stop_all()
print("Cooling down...")
time.sleep(cooldown)
output = p.communicate()[0].decode(sys.getfilesystemencoding())
if not client_only:
stdout, stderr = ssrv[0].communicate()
stdout = stdout.decode(sys.getfilesystemencoding())
stderr = stderr.decode(sys.getfilesystemencoding())
else:
stdout = ""
stderr = ""
if success:
# return mout, sout, serr (master output, server output, server serr)
return output, stdout, stderr
def run_exp(sc, ld, lstat):
while True:
if client_only:
ssrv = None
else:
# start server
tc.log_print("Starting server...")
server_cmd = test_dir + "/pingpong/build/ppd -a -t " + str(
threads) + " -p " + str(server_port) + " -M 0 -F 100000 "
if priority:
server_cmd += " -r " + (hpc[0] if measure_lp else master[0])
if sc != -1:
server_cmd = server_cmd + " -m " + str(sc)
tc.log_print(server_cmd)
ssrv = tc.remote_exec(server, server_cmd, blocking=False)
# start clients
tc.log_print("Starting clients...")
client_cmd = tc.get_cpuset_core(
client_threads) + " " + test_dir + "/pingpong/build/dismember -A"
tc.log_print(client_cmd)
sclt = tc.remote_exec(clients, client_cmd, blocking=False)
hpclt = None
time.sleep(1)
# start hpc
tc.log_print("Starting hpc...")
hpc_cmd = tc.get_cpuset_core(client_threads) + " " + test_dir + "/pingpong/build/dismember " + \
" -s " + server[0] + \
" -p " + str(server_port) + \
" -q 30000 " + \
" -c 1 " + \
" -t " + str(client_threads) + \
" -w 999" + \
" -W 0" + \
" -i exponential " + \
" -OGEN=fixed:0 " + \
" -OCDELAY=0" + \
" -l 0"
tc.log_print(hpc_cmd)
hpclt = tc.remote_exec(hpc, hpc_cmd, blocking=False)
time.sleep(1)
# start master
tc.log_print("Starting master...")
master_cmd = tc.get_cpuset_core(client_threads) + " " + test_dir + "/pingpong/build/dismember " + \
get_client_str(clients) + \
" -s " + server[0] + \
" -p " + str(server_port) + \
" -q " + str(ld) + \
" -c " + str(conn_per_thread) + \
" -o " + test_dir + "/" + sample_filename + \
" -t " + str(client_threads) + \
" -w " + str(duration) + \
" -W " + str(warmup) + \
" -T " + str(client_threads) + \
" -i exponential " + \
" -C 1" + \
" -Q 30000" + \
" -l 0 " + \
" -OGEN=fixed:0 " + \
" -OCDELAY=" + ("1" if conn_delay else "0")
tc.log_print(master_cmd)
sp = tc.remote_exec(master, master_cmd, blocking=False)
p = sp[0]
success = False
cur = 0
while True:
# either failed or timeout
# we use failure detection to save time for long durations
#or (measure_lp and (not tc.scan_stderr(hpclt, exclude=[".*warn.*", ".*WARN.*", ".*DEBUG.*"]))) \
if False \
or not tc.scan_stderr(sp, exclude=[".*warn.*", ".*WARN.*", ".*DEBUG.*"]) \
or not tc.scan_stderr(ssrv, exclude=[".*warn.*", ".*WARN.*", ".*DEBUG.*"]) \
or not tc.scan_stderr(sclt, exclude=[".*warn.*", ".*WARN.*", ".*DEBUG.*"]) \
or cur >= int(warmup + duration) * 2 \
:
break
if p.poll() != None:
success = True
break
time.sleep(1)
cur = cur + 1
stop_all()
print("Cooling down...")
time.sleep(cooldown)
output = p.communicate()[0].decode(sys.getfilesystemencoding())
if not client_only:
stdout, stderr = ssrv[0].communicate()
stdout = stdout.decode(sys.getfilesystemencoding())
stderr = stderr.decode(sys.getfilesystemencoding())
else:
stdout = ""
stderr = ""
if success:
# return mout, sout, serr (master output, server output, server serr)
return output, stdout, stderr
def main():
global hostfile
global server
global clients
global dump
global lockstat
global client_only
global server_delay
options = getopt.getopt(sys.argv[1:], 'h:sldcDp')[0]
for opt, arg in options:
if opt in ('-h'):
hostfile = arg
elif opt in ('-s'):
stop_all()
return
elif opt in ('-l'):
lockstat=True
elif opt in ('-d'):
dump=True
elif opt in ('-c'):
client_only=True
elif opt in ('-D'):
server_delay=True
tc.init("scale_" + str(len(clients)) + "x" + str(client_threads) + "x" + str(conn_per_thread))
tc.log_print("Configuration:\n" + \
"Hostfile: " + ("None" if hostfile == None else hostfile) + "\n" \
"Lockstat: " + str(lockstat) + "\n" \
"KQ dump: " + str(dump) + "\n" \
"Client only: " + str(client_only) + "\n" + \
"Server delay: " + str(server_delay) + "\n" + \
"Conn delay: " + str(conn_delay) + "\n")
if hostfile != None:
hosts = tc.parse_hostfile(hostfile)
server = tc.process_hostnames(server, hosts)
clients = tc.process_hostnames(clients, hosts)
master = tc.process_hostnames(master, hosts)
stop_all()
for i in range(0, len(sched), 2):
esched = sched[i+1]
ename = sched[i]
# output, sout, serr = run_exp(esched, 12, lockstat)
# keep_results(99, output, sout, serr)
# stop_all()
tc.begin(ename)
for j in range(0, len(threads)):
ethread = threads[j]
tc.log_print("============ Sched: " + str(ename) + " Thread: " + str(ethread) + " ============")
output, sout, serr = run_exp(esched, ethread, lockstat)
keep_results(ethread, output, sout, serr)
tc.log_print("")
tc.end()
stop_all()
