def postproc_model_logs(out_fldr):
""" These scripts are designed to be executed after results from different remote YCSBs are
being aggregated locally into one folder """
fname = sys._getframe().f_code.co_name
separator(fname)
old_dir = os.getcwd()
os.chdir(out_fldr)
try:
cmd = "model_rate_post_processor.r --src_folder {out_fldr}".format(
out_fldr=out_fldr)
logging.info("Executing [%s] \n cmd[%s]", fname, cmd)
proc = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
check_pid_output(proc)
# proc = subprocess.Popen(("Rscript " + ycsb_dir + "/models/model_rate_post_processor.r --f " + src_dir).split(), stdout=subprocess.PIPE)
except:
print_generic_exception()
finally:
os.chdir(old_dir)
def join_files(parsed_files, out_file):
logging.debug("Joining files: [%s] into [%s]", parsed_files, out_file)
for i, pf in enumerate(parsed_files):
if i == 0:
cmd = "cat %s > %s" % (pf, out_file)
else:
cmd = "cat %s >> %s" % (pf, out_file)
logging.debug("Joining [%s] into [%s]", pf, out_file)
proc = subprocess.Popen(cmd, shell=True)
check_pid_output(proc)
def split_file(ifile, n_parts):
try:
cmd = "split_file.sh {file} {n_parts} {base_file_name}".format(
file=ifile,
n_parts=n_parts,
base_file_name=os.path.basename(ifile))
logging.info("Splitting file [%s] into [%s] parts", ifile, n_parts)
proc = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
check_pid_output(proc)
except:
print_generic_exception("file splitting failed")
def postproc_plot_cdfs_per_experiment(out_fldr,
exp_id,
minutes2skip_from_ycsb_trace,
target_slo_us,
mask=YCSB_MASK):
separator(sys._getframe().f_code.co_name)
old_dir = os.getcwd()
os.chdir(out_fldr)
try:
files = get_files_matching_mask(where="./", mask="i%s*.ycsb" % exp_id)
print os.getcwd()
print "Files, ", files
cdf_files = []
for f in files:
out_file_name = "%s.%s" % (f, PARSED_FILE_EXTENSION)
cdf_files.append(out_file_name)
if not os.path.isfile(out_file_name):
cpp_parse_ycsb_file(f, out_file_name, valid_prefixes=["READ"])
cmd = "ycsb_plot_cdfs.py --src_files {src_files} --out_name {out_name} "\
"--skip_first_samples_min {skip} --target_latency_slo_us {slo} "\
"--consider_siblings --pre_parsed_input".format(
src_files=" ".join(cdf_files),
out_name="i%s_cdfs.png" % exp_id,
skip=minutes2skip_from_ycsb_trace,
slo=target_slo_us)
logging.info("Executing postproc_plot_cdfs_per_experiment \n cmd[%s]",
cmd)
proc = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
check_pid_output(proc)
except:
print_generic_exception()
finally:
os.chdir(old_dir)
def try_postprocessing(conf, workload_name, of_ycsb_out, mask=YCSB_MASK):
# if conf[workload_name]["p_workload"] == "com.yahoo.ycsb.workloads.TraceWorkload":
# postproc_traceworkload(conf, workload_name, of_ycsb_out)
if "only_stats" in conf and conf["only_stats"] == "1":
tmp_out = "%s.short" % of_ycsb_out
cmd = "head -n 5000 {inn} > {tmp_out}".format(inn=of_ycsb_out,
tmp_out=tmp_out)
print "Executing: ", cmd
proc = subprocess.Popen(cmd, shell=True)
check_pid_output(proc)
print "Executing: ", cmd
cmd = "tail -n 5000 {inn} >> {tmp_out}".format(inn=of_ycsb_out,
tmp_out=tmp_out)
print "Executing: ", cmd
proc = subprocess.Popen(cmd, shell=True)
check_pid_output(proc)
print "Executing: ", cmd
cmd = "mv {tmp_out} {inn}".format(inn=of_ycsb_out, tmp_out=tmp_out)
print "Executing: ", cmd
proc = subprocess.Popen(cmd, shell=True)
check_pid_output(proc)
pass
def postproc_plot_all_individual_cdfs(out_fldr, target_slo_us,
minutes2skip_from_ycsb_trace):
""" This function searches for ycsb files in the target directory and plots a one .png
file per one .ycsb file """
separator(sys._getframe().f_code.co_name)
old_dir = os.getcwd()
os.chdir(out_fldr)
try:
files = get_files_matching_mask(where="./", mask=YCSB_MASK)
print os.getcwd()
print "Files, ", files
for f in files:
parsed_ycsb_file = parse_ycsb_if_necessary(f)
cmd = "ycsb_plot_cdfs.py --src_files {src_files} --out_name {out_name} "\
"--skip_first_samples_min {skip} --target_latency_slo_us {slo} "\
"--consider_siblings --pre_parsed_input".format(
src_files=parsed_ycsb_file,
out_name="%s.png" % f,
skip=minutes2skip_from_ycsb_trace,
slo=target_slo_us)
logging.info(
"Executing postproc_plot_all_individual_cdfs \n cmd[%s]", cmd)
proc = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
check_pid_output(proc)
except:
print_generic_exception()
finally:
os.chdir(old_dir)
def postproc_plot_merged_cdf(out_fldr,
minutes2skip_from_ycsb_trace=0,
mask=YCSB_MASK,
out_name="merged_cdfs.png"):
separator(sys._getframe().f_code.co_name)
old_dir = os.getcwd()
os.chdir(out_fldr)
try:
files = get_files_matching_mask(where="./", mask=mask)
for f in files:
parse_ycsb_if_necessary(f)
merged_fname = "cdf.merged"
proc = subprocess.Popen("echo \"\" > %s" % merged_fname,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
check_pid_output(proc)
cdf_files = get_files_matching_mask(where="./", mask=YCSB_MASK)
for f in cdf_files:
cmd = "cat %s > %s" % (f, merged_fname)
logging.info("cmd[%s]", cmd)
proc = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
check_pid_output(proc)
cmd = "ycsb_plot_cdfs.py --src_files %s --out_name %s --skip_first_samples_min %s --consider_siblings --pre_parsed_input" % (
merged_fname, out_name, minutes2skip_from_ycsb_trace)
logging.info("Executing postproc_plot_merged_cdf \n cmd[%s]", cmd)
proc = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
check_pid_output(proc)
except:
print_generic_exception()
finally:
os.chdir(old_dir)
def postproc_visualise_traceworkload(conf,
out_fldr,
exp_id=None,
mask=YCSB_MASK,
warm_up_period_sec=0):
separator(sys._getframe().f_code.co_name)
def get_exp_and_workload_ids(ycsb_out_file_name):
# i1_wl1_ycsb.ycsb
tokens = ycsb_out_file_name.split("_")
exp_id = int(tokens[0][1:])
workload_id = int(tokens[1][2:])
return exp_id, workload_id
old_dir = os.getcwd()
os.chdir(out_fldr)
if exp_id != None:
mask = "i%s%s" % (exp_id, mask)
try:
files = get_files_matching_mask(where="./", mask=mask)
print os.getcwd()
print "Files", files
for f in files:
print "Processing file: ", f
exp_id, workload_id = get_exp_and_workload_ids(os.path.basename(f))
tmp_R_file = "%s.r" % f
cpp_parse_ycsb_file(f, tmp_R_file, valid_prefixes=["R"], pattern=2)
workload_name = "workload_%s" % workload_id
cmd = "visualize_ycsb_trace.R "\
"--intended_ts_file {trace_file} --actual_ts_file {tmp_R_file}"\
" --duration 1000 --interval 1 --id {id} --warmup {warmup} ".format(
trace_file=conf[workload_name]["p_tracefile"],
tmp_R_file=tmp_R_file, warmup=warm_up_period_sec,
id=exp_id)
logging.info(
"Executing post processing for workloadTrace 2/2 \n cmd[%s]",
cmd)
proc = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
check_pid_output(proc)
logging.info("Removing tmp file [%s]", tmp_R_file)
remove_files([tmp_R_file])
# exp_index = os.path.basename(of_ycsb_out)[:2] # taking iX from i1_ycsb.ycsb
# tmp_out_file = "%s_r.data" %exp_index
# # cmd = 'cat %s | grep " R " | awk "NF==3{print}{}" > %s' % (
# # of_ycsb_out, tmp_out_file)
# # logging.info("Executing post processing for WorkloadTrace 1/2 \n cmd[%s]", cmd)
# # proc = subprocess.Popen(cmd,
# # stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
# # check_pid_output(proc)
# cmd = "visualize_ycsb_trace.R "\
# "--intended_ts_file {trace_file} --actual_ts_file {tmp_out_file}"\
# " --duration 1000 --interval 1 --id {id}".format(
# trace_file=conf[workload_name]["p_tracefile"],
# tmp_out_file=tmp_out_file,
# id=exp_index)
# logging.info("Executing post processing for workloadTrace 2/2 \n cmd[%s]", cmd)
# proc = subprocess.Popen(cmd,
# stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
# check_pid_output(proc)
except:
print_generic_exception()
finally:
os.chdir(old_dir)