def run_exp_old(exps, network_test=False, delay=''):
if env.shmem:
schema_path = "/dev/shm/"
else:
schema_path = "{}/".format(env.rem_homedir)
good_hosts = []
if not network_test and EXECUTE_EXPS:
good_hosts = get_good_hosts()
with color():
puts("good host list =\n{}".format(
pprint.pformat(good_hosts, depth=3)),
show_prefix=True)
execute(copy_schema)
fmt, experiments = experiment_map[exps]()
batch_size = 0
nids = {}
outfiles = {}
exps = {}
runfiles = {}
for e in experiments:
print(e)
cfgs = get_cfgs(fmt, e)
output_fbase = get_outfile_name(cfgs, fmt, env.hosts)
output_exec_fname = get_execfile_name(cfgs, fmt, env.hosts)
output_f = output_fbase + STRNOW
last_exp = experiments.index(e) == len(experiments) - 1
skip_exp = False
# Check whether experiment has been already been run in this batch
if SKIP:
if len(glob.glob('{}*{}*.out'.format(env.result_dir,
output_fbase))) > 0:
with color("warn"):
puts("experiment exists in results folder... skipping",
show_prefix=True)
if last_exp:
skip_exp = True
else:
continue
global CC_ALG
CC_ALG = cfgs["CC_ALG"]
if EXECUTE_EXPS:
cfg_srcpath = "{}cfg".format(
os.path.join("binaries", output_exec_fname))
cfg_destpath = "{}.cfg".format(
os.path.join(env.result_dir, output_exec_fname + STRNOW))
local("cp {} {}".format(cfg_srcpath, cfg_destpath))
nnodes = cfgs["NODE_CNT"]
nclnodes = cfgs["CLIENT_NODE_CNT"]
try:
ntotal = nnodes + nclnodes
except TypeError:
nclnodes = cfgs[cfgs["CLIENT_NODE_CNT"]]
ntotal = nnodes + nclnodes
# if CC_ALG == 'CALVIN':
# ntotal += 1
if env.same_node:
ntotal = 1
if env.overlap:
ntotal = max(nnodes, nclnodes)
if env.cram:
ntotal = max(max(nnodes, nclnodes) / 8, 1)
if env.remote:
if not network_test:
set_hosts(good_hosts)
# if ntotal > len(env.hosts):
# msg = "Not enough nodes to run experiment!\n"
# msg += "\tRequired nodes: {}, ".format(ntotal)
# msg += "Actual nodes: {}".format(len(env.hosts))
# with color():
# puts(msg,show_prefix=True)
# cmd = "rm -f config.h {}".format(cfg_destpath)
# local(cmd)
# continue
if not skip_exp:
if env.batch_mode:
# If full, execute all exps in batch and reset everything
full = (batch_size + ntotal) > len(env.hosts)
if full:
if env.cluster != 'istc' and not env.dry_run:
# Sync clocks before each experiment
execute(sync_clocks)
with color():
puts("Batch is full, deploying batch...{}/{}".
format(batch_size, len(good_hosts)),
show_prefix=True)
with color("debug"):
puts(pprint.pformat(outfiles, depth=3),
show_prefix=False)
set_hosts(env.hosts[:batch_size])
with color():
puts("Starttime: {}".format(
datetime.datetime.now().strftime(
"%H:%M:%S")),
show_prefix=True)
execute(deploy, schema_path, nids, exps, runfiles,
fmt)
with color():
puts("Endtime: {}".format(
datetime.datetime.now().strftime(
"%H:%M:%S")),
show_prefix=True)
execute(get_results, outfiles, nids)
if not env.dry_run:
good_hosts = get_good_hosts()
env.roledefs = None
batch_size = 0
nids = {}
exps = {}
runfiles = {}
outfiles = {}
set_hosts(good_hosts)
else:
with color():
puts("Adding experiment to current batch: {}".
format(output_f),
show_prefix=True)
machines = env.hosts[batch_size:batch_size + ntotal]
batch_size += ntotal
else:
machines = env.hosts[:ntotal]
set_hosts(machines)
new_roles = execute(assign_roles,
nnodes,
nclnodes,
append=env.batch_mode)[env.host]
new_nids, new_exps, new_runfiles = execute(
write_ifconfig, new_roles, e,
output_exec_fname)[env.host]
nids.update(new_nids)
exps.update(new_exps)
runfiles.update(new_runfiles)
for host, nid in new_nids.iteritems():
outfiles[host] = "{}.out".format(output_f)
# if env.same_node:
# outfiles[host] = "{}.out".format(output_f)
# else:
# outfiles[host] = "{}_{}.out".format(nid[0],output_f)
print(nids)
if cfgs["WORKLOAD"] == "TPCC":
schema = "benchmarks/TPCC_full_schema.txt"
# schema = "benchmarks/TPCC_short_schema.txt"
elif cfgs["WORKLOAD"] == "YCSB":
schema = "benchmarks/YCSB_schema.txt"
elif cfgs["WORKLOAD"] == "PPS":
schema = "benchmarks/PPS_schema.txt"
# NOTE: copy_files will fail if any (possibly) stray processes
# are still running one of the executables. Setting the 'kill'
# flag in environment.py to true to kill these processes. This
# is useful for running real experiments but dangerous when both
# of us are debugging...
# execute(copy_files,schema,output_exec_fname)
execute(copy_ifconfig)
if not env.batch_mode or last_exp and len(exps) > 0:
if env.batch_mode:
set_hosts(good_hosts[:batch_size])
puts("Deploying last batch...{}/{}".format(
batch_size, len(good_hosts)),
show_prefix=True)
else:
print("Deploying: {}".format(output_f))
if env.cluster != 'istc':
# Sync clocks before each experiment
print("Syncing Clocks...")
execute(sync_clocks)
if delay != '':
execute(set_delay, delay=delay)
with color():
puts("Starttime: {}".format(
datetime.datetime.now().strftime("%H:%M:%S")),
show_prefix=True)
execute(deploy, schema_path, nids, exps, runfiles, fmt)
with color():
puts("Endtime: {}".format(
datetime.datetime.now().strftime("%H:%M:%S")),
show_prefix=True)
if delay != '':
execute(reset_delay)
execute(get_results, outfiles, nids)
if not env.dry_run:
good_hosts = get_good_hosts()
set_hosts(good_hosts)
batch_size = 0
nids = {}
exps = {}
outfiles = {}
env.roledefs = None
else:
pids = []
print("Deploying: {}".format(output_f))
for n in range(ntotal):
if n < nnodes:
cmd = "./rundb -nid{}".format(n)
elif n < nnodes + nclnodes:
cmd = "./runcl -nid{}".format(n)
# elif n == nnodes+nclnodes:
# assert(CC_ALG == 'CALVIN')
# cmd = "./runsq -nid{}".format(n)
else:
assert (false)
print(cmd)
cmd = shlex.split(cmd)
ofile_n = "{}{}_{}.out".format(env.result_dir, n, output_f)
ofile = open(ofile_n, 'w')
p = subprocess.Popen(cmd, stdout=ofile, stderr=ofile)
pids.insert(0, p)
for n in range(ntotal):
pids[n].wait()
def run_exp_old(exps,network_test=False,delay=''):
if env.shmem:
schema_path = "/dev/shm/"
else:
schema_path = "{}/".format(env.rem_homedir)
good_hosts = []
if not network_test and EXECUTE_EXPS:
good_hosts = get_good_hosts()
with color():
puts("good host list =\n{}".format(pprint.pformat(good_hosts,depth=3)),show_prefix=True)
execute(copy_schema)
fmt,experiments = experiment_map[exps]()
batch_size = 0
nids = {}
outfiles = {}
exps = {}
runfiles = {}
for e in experiments:
print(e)
cfgs = get_cfgs(fmt,e)
output_fbase = get_outfile_name(cfgs,fmt,env.hosts)
output_exec_fname = get_execfile_name(cfgs,fmt,env.hosts)
output_f = output_fbase + STRNOW
last_exp = experiments.index(e) == len(experiments) - 1
skip_exp = False
# Check whether experiment has been already been run in this batch
if SKIP:
if len(glob.glob('{}*{}*.out'.format(env.result_dir,output_fbase))) > 0:
with color("warn"):
puts("experiment exists in results folder... skipping",show_prefix=True)
if last_exp:
skip_exp = True
else:
continue
global CC_ALG
CC_ALG = cfgs["CC_ALG"]
if EXECUTE_EXPS:
cfg_srcpath = "{}cfg".format(os.path.join("binaries",output_exec_fname))
cfg_destpath = "{}.cfg".format(os.path.join(env.result_dir,output_exec_fname+STRNOW))
local("cp {} {}".format(cfg_srcpath,cfg_destpath))
nnodes = cfgs["NODE_CNT"]
nclnodes = cfgs["CLIENT_NODE_CNT"]
try:
ntotal = nnodes + nclnodes
except TypeError:
nclnodes = cfgs[cfgs["CLIENT_NODE_CNT"]]
ntotal = nnodes + nclnodes
# if CC_ALG == 'CALVIN':
# ntotal += 1
if env.same_node:
ntotal = 1
if env.overlap:
ntotal = max(nnodes,nclnodes)
if env.cram:
ntotal = max(max(nnodes,nclnodes)/8,1)
if env.remote:
if not network_test:
set_hosts(good_hosts)
# if ntotal > len(env.hosts):
# msg = "Not enough nodes to run experiment!\n"
# msg += "\tRequired nodes: {}, ".format(ntotal)
# msg += "Actual nodes: {}".format(len(env.hosts))
# with color():
# puts(msg,show_prefix=True)
# cmd = "rm -f config.h {}".format(cfg_destpath)
# local(cmd)
# continue
if not skip_exp:
if env.batch_mode:
# If full, execute all exps in batch and reset everything
full = (batch_size + ntotal) > len(env.hosts)
if full:
if env.cluster != 'istc' and not env.dry_run:
# Sync clocks before each experiment
execute(sync_clocks)
with color():
puts("Batch is full, deploying batch...{}/{}".format(batch_size,len(good_hosts)),show_prefix=True)
with color("debug"):
puts(pprint.pformat(outfiles,depth=3),show_prefix=False)
set_hosts(env.hosts[:batch_size])
with color():
puts("Starttime: {}".format(datetime.datetime.now().strftime("%H:%M:%S")),show_prefix=True)
execute(deploy,schema_path,nids,exps,runfiles,fmt)
with color():
puts("Endtime: {}".format(datetime.datetime.now().strftime("%H:%M:%S")),show_prefix=True)
execute(get_results,outfiles,nids)
if not env.dry_run:
good_hosts = get_good_hosts()
env.roledefs = None
batch_size = 0
nids = {}
exps = {}
runfiles = {}
outfiles = {}
set_hosts(good_hosts)
else:
with color():
puts("Adding experiment to current batch: {}".format(output_f), show_prefix=True)
machines = env.hosts[batch_size : batch_size + ntotal]
batch_size += ntotal
else:
machines = env.hosts[:ntotal]
set_hosts(machines)
new_roles=execute(assign_roles,nnodes,nclnodes,append=env.batch_mode)[env.host]
new_nids,new_exps,new_runfiles = execute(write_ifconfig,new_roles,e,output_exec_fname)[env.host]
nids.update(new_nids)
exps.update(new_exps)
runfiles.update(new_runfiles)
for host,nid in new_nids.iteritems():
outfiles[host] = "{}.out".format(output_f)
# if env.same_node:
# outfiles[host] = "{}.out".format(output_f)
# else:
# outfiles[host] = "{}_{}.out".format(nid[0],output_f)
print(nids)
if cfgs["WORKLOAD"] == "TPCC":
schema = "benchmarks/TPCC_full_schema.txt"
# schema = "benchmarks/TPCC_short_schema.txt"
elif cfgs["WORKLOAD"] == "YCSB":
schema = "benchmarks/YCSB_schema.txt"
elif cfgs["WORKLOAD"] == "PPS":
schema = "benchmarks/PPS_schema.txt"
# NOTE: copy_files will fail if any (possibly) stray processes
# are still running one of the executables. Setting the 'kill'
# flag in environment.py to true to kill these processes. This
# is useful for running real experiments but dangerous when both
# of us are debugging...
# execute(copy_files,schema,output_exec_fname)
execute(copy_ifconfig)
if not env.batch_mode or last_exp and len(exps) > 0:
if env.batch_mode:
set_hosts(good_hosts[:batch_size])
puts("Deploying last batch...{}/{}".format(batch_size,len(good_hosts)),show_prefix=True)
else:
print("Deploying: {}".format(output_f))
if env.cluster != 'istc':
# Sync clocks before each experiment
print("Syncing Clocks...")
execute(sync_clocks)
if delay != '':
execute(set_delay,delay=delay)
with color():
puts("Starttime: {}".format(datetime.datetime.now().strftime("%H:%M:%S")),show_prefix=True)
execute(deploy,schema_path,nids,exps,runfiles,fmt)
with color():
puts("Endtime: {}".format(datetime.datetime.now().strftime("%H:%M:%S")),show_prefix=True)
if delay != '':
execute(reset_delay)
execute(get_results,outfiles,nids)
if not env.dry_run:
good_hosts = get_good_hosts()
set_hosts(good_hosts)
batch_size = 0
nids = {}
exps = {}
outfiles = {}
env.roledefs = None
else:
pids = []
print("Deploying: {}".format(output_f))
for n in range(ntotal):
if n < nnodes:
cmd = "./rundb -nid{}".format(n)
elif n < nnodes+nclnodes:
cmd = "./runcl -nid{}".format(n)
# elif n == nnodes+nclnodes:
# assert(CC_ALG == 'CALVIN')
# cmd = "./runsq -nid{}".format(n)
else:
assert(false)
print(cmd)
cmd = shlex.split(cmd)
ofile_n = "{}{}_{}.out".format(env.result_dir,n,output_f)
ofile = open(ofile_n,'w')
p = subprocess.Popen(cmd,stdout=ofile,stderr=ofile)
pids.insert(0,p)
for n in range(ntotal):
pids[n].wait()
def run_exp(exps, network_test=False, delay=''):
if env.shmem:
schema_path = "/dev/shm/"
else:
schema_path = "{}/".format(env.rem_homedir)
good_hosts = []
if not network_test and EXECUTE_EXPS:
good_hosts = get_good_hosts()
with color():
puts("good host list =\n{}".format(
pprint.pformat(good_hosts, depth=3)),
show_prefix=True)
fmt, experiments = experiment_map[exps]()
batch_size = 0
nids = {}
outfiles = {}
exps = {}
if SKIP:
for e in experiments[:]:
cfgs = get_cfgs(fmt, e)
output_fbase = get_outfile_name(cfgs, fmt, env.hosts)
if len(glob.glob('{}*{}*.out'.format(env.result_dir,
output_fbase))) > 0:
with color("warn"):
puts("experiment exists in results folder... skipping",
show_prefix=True)
experiments.remove(e)
experiments.sort(key=lambda x: x[fmt.index("NODE_CNT")] + x[fmt.index(
"CLIENT_NODE_CNT")],
reverse=True)
# Fill experiment pool
while len(experiments) > 0:
round_exps = []
batch_total = 0
for e in experiments[:]:
cfgs = get_cfgs(fmt, e)
nnodes = cfgs["NODE_CNT"]
nclnodes = cfgs["CLIENT_NODE_CNT"]
ccalg = cfgs["CC_ALG"]
ntotal = cfgs["NODE_CNT"] + cfgs["CLIENT_NODE_CNT"]
# if ccalg == 'CALVIN':
# ntotal += 1
if env.same_node:
ntotal = 1
if env.overlap:
ntotal = max(nnodes, nclnodes)
if env.cram:
ntotal = max(max(nnodes, nclnodes) / 8, 1)
if ntotal > len(env.hosts):
msg = "Not enough nodes to run experiment!\n"
msg += "\tRequired nodes: {}, ".format(ntotal)
msg += "Actual nodes: {}".format(len(env.hosts))
with color():
puts(msg, show_prefix=True)
experiments.remove(e)
continue
if (batch_total + ntotal) > len(env.hosts):
continue
batch_total += ntotal
round_exps.append(e)
experiments.remove(e)
if not EXECUTE_EXPS: continue
batch_size = 0
for e in round_exps:
set_hosts(good_hosts)
cfgs = get_cfgs(fmt, e)
global CC_ALG
nnodes = cfgs["NODE_CNT"]
nclnodes = cfgs["CLIENT_NODE_CNT"]
CC_ALG = cfgs["CC_ALG"]
ntotal = cfgs["NODE_CNT"] + cfgs["CLIENT_NODE_CNT"]
# if ccalg == 'CALVIN':
# ntotal += 1
if env.same_node:
ntotal = 1
if env.overlap:
ntotal = max(nnodes, nclnodes)
if env.cram:
ntotal = max(max(nnodes, nclnodes) / 8, 1)
output_fbase = get_outfile_name(cfgs, fmt, env.hosts)
output_exec_fname = get_execfile_name(cfgs, fmt, env.hosts)
output_f = output_fbase + STRNOW
cfg_srcpath = "{}cfg".format(
os.path.join("binaries", output_exec_fname))
cfg_destpath = "{}.cfg".format(
os.path.join(env.result_dir, output_exec_fname + STRNOW))
local("cp {} {}".format(cfg_srcpath, cfg_destpath))
with color():
puts("Adding experiment to current batch: {}".format(output_f),
show_prefix=True)
machines = env.hosts[batch_size:batch_size + ntotal]
batch_size += ntotal
set_hosts(machines)
new_roles = execute(assign_roles,
nnodes,
nclnodes,
append=env.batch_mode)[env.host]
new_nids, new_exps = execute(write_ifconfig, new_roles,
e)[env.host]
nids.update(new_nids)
exps.update(new_exps)
for host, nid in new_nids.iteritems():
outfiles[host] = "{}.out".format(output_f)
if cfgs["WORKLOAD"] == "TPCC":
schema = "benchmarks/TPCC_full_schema.txt"
# schema = "benchmarks/TPCC_short_schema.txt"
elif cfgs["WORKLOAD"] == "YCSB":
schema = "benchmarks/YCSB_schema.txt"
elif cfgs["WORKLOAD"] == "PPS":
schema = "benchmarks/PPS_schema.txt"
# NOTE: copy_files will fail if any (possibly) stray processes
# are still running one of the executables. Setting the 'kill'
# flag in environment.py to true to kill these processes. This
# is useful for running real experiments but dangerous when both
# of us are debugging...
# execute(copy_files,schema,output_exec_fname)
execute(copy_ifconfig)
if env.remote:
set_hosts(good_hosts[:batch_size])
if env.cluster != 'istc' and not env.dry_run:
# Sync clocks before each experiment
execute(sync_clocks)
with color():
puts("Batch is full, deploying batch...{}/{}".format(
batch_size, len(good_hosts)),
show_prefix=True)
with color("debug"):
puts(pprint.pformat(outfiles, depth=3), show_prefix=False)
with color():
puts("Starttime: {}".format(
datetime.datetime.now().strftime("%H:%M:%S")),
show_prefix=True)
execute(deploy, schema_path, nids, exps, runfiles, fmt)
with color():
puts("Endtime: {}".format(
datetime.datetime.now().strftime("%H:%M:%S")),
show_prefix=True)
execute(get_results, outfiles, nids)
good_hosts = get_good_hosts()
batch_size = 0
nids = {}
exps = {}
outfiles = {}
set_hosts(good_hosts)
env.roledefs = None
def run_exp(exps,network_test=False,delay=''):
if env.shmem:
schema_path = "/dev/shm/"
else:
schema_path = "{}/".format(env.rem_homedir)
good_hosts = []
if not network_test and EXECUTE_EXPS:
good_hosts = get_good_hosts()
with color():
puts("good host list =\n{}".format(pprint.pformat(good_hosts,depth=3)),show_prefix=True)
fmt,experiments = experiment_map[exps]()
batch_size = 0
nids = {}
outfiles = {}
exps = {}
if SKIP:
for e in experiments[:]:
cfgs = get_cfgs(fmt,e)
output_fbase = get_outfile_name(cfgs,fmt,env.hosts)
if len(glob.glob('{}*{}*.out'.format(env.result_dir,output_fbase))) > 0:
with color("warn"):
puts("experiment exists in results folder... skipping",show_prefix=True)
experiments.remove(e)
experiments.sort(key=lambda x: x[fmt.index("NODE_CNT")] + x[fmt.index("CLIENT_NODE_CNT")],reverse=True)
# Fill experiment pool
while len(experiments) > 0 :
round_exps = []
batch_total = 0
for e in experiments[:]:
cfgs = get_cfgs(fmt,e)
nnodes = cfgs["NODE_CNT"]
nclnodes = cfgs["CLIENT_NODE_CNT"]
ccalg = cfgs["CC_ALG"]
ntotal = cfgs["NODE_CNT"] + cfgs["CLIENT_NODE_CNT"]
# if ccalg == 'CALVIN':
# ntotal += 1
if env.same_node:
ntotal = 1
if env.overlap:
ntotal = max(nnodes,nclnodes)
if env.cram:
ntotal = max(max(nnodes,nclnodes)/8,1)
if ntotal > len(env.hosts):
msg = "Not enough nodes to run experiment!\n"
msg += "\tRequired nodes: {}, ".format(ntotal)
msg += "Actual nodes: {}".format(len(env.hosts))
with color():
puts(msg,show_prefix=True)
experiments.remove(e)
continue
if (batch_total + ntotal) > len(env.hosts):
continue
batch_total += ntotal
round_exps.append(e)
experiments.remove(e)
if not EXECUTE_EXPS: continue
batch_size = 0
for e in round_exps:
set_hosts(good_hosts)
cfgs = get_cfgs(fmt,e)
global CC_ALG
nnodes = cfgs["NODE_CNT"]
nclnodes = cfgs["CLIENT_NODE_CNT"]
CC_ALG = cfgs["CC_ALG"]
ntotal = cfgs["NODE_CNT"] + cfgs["CLIENT_NODE_CNT"]
# if ccalg == 'CALVIN':
# ntotal += 1
if env.same_node:
ntotal = 1
if env.overlap:
ntotal = max(nnodes,nclnodes)
if env.cram:
ntotal = max(max(nnodes,nclnodes)/8,1)
output_fbase = get_outfile_name(cfgs,fmt,env.hosts)
output_exec_fname = get_execfile_name(cfgs,fmt,env.hosts)
output_f = output_fbase + STRNOW
cfg_srcpath = "{}cfg".format(os.path.join("binaries",output_exec_fname))
cfg_destpath = "{}.cfg".format(os.path.join(env.result_dir,output_exec_fname+STRNOW))
local("cp {} {}".format(cfg_srcpath,cfg_destpath))
with color():
puts("Adding experiment to current batch: {}".format(output_f), show_prefix=True)
machines = env.hosts[batch_size : batch_size + ntotal]
batch_size += ntotal
set_hosts(machines)
new_roles=execute(assign_roles,nnodes,nclnodes,append=env.batch_mode)[env.host]
new_nids,new_exps = execute(write_ifconfig,new_roles,e)[env.host]
nids.update(new_nids)
exps.update(new_exps)
for host,nid in new_nids.iteritems():
outfiles[host] = "{}.out".format(output_f)
if cfgs["WORKLOAD"] == "TPCC":
schema = "benchmarks/TPCC_full_schema.txt"
# schema = "benchmarks/TPCC_short_schema.txt"
elif cfgs["WORKLOAD"] == "YCSB":
schema = "benchmarks/YCSB_schema.txt"
elif cfgs["WORKLOAD"] == "PPS":
schema = "benchmarks/PPS_schema.txt"
# NOTE: copy_files will fail if any (possibly) stray processes
# are still running one of the executables. Setting the 'kill'
# flag in environment.py to true to kill these processes. This
# is useful for running real experiments but dangerous when both
# of us are debugging...
# execute(copy_files,schema,output_exec_fname)
execute(copy_ifconfig)
if env.remote:
set_hosts(good_hosts[:batch_size])
if env.cluster != 'istc' and not env.dry_run:
# Sync clocks before each experiment
execute(sync_clocks)
with color():
puts("Batch is full, deploying batch...{}/{}".format(batch_size,len(good_hosts)),show_prefix=True)
with color("debug"):
puts(pprint.pformat(outfiles,depth=3),show_prefix=False)
with color():
puts("Starttime: {}".format(datetime.datetime.now().strftime("%H:%M:%S")),show_prefix=True)
execute(deploy,schema_path,nids,exps,runfiles,fmt)
with color():
puts("Endtime: {}".format(datetime.datetime.now().strftime("%H:%M:%S")),show_prefix=True)
execute(get_results,outfiles,nids)
good_hosts = get_good_hosts()
batch_size = 0
nids = {}
exps = {}
outfiles = {}
set_hosts(good_hosts)
env.roledefs = None