def __init__(self, fn_in, time_begin, time_end): #self.overloaded = overloaded # Unzip when the file is not there if not os.path.exists(fn_in): fn_zipped = "%s.bz2" % fn_in if not os.path.exists(fn_zipped): raise RuntimeError("Unexpected: %s" % fn_in) Util.RunSubp( "cd %s && bzip2 -dk %s > /dev/null" % (os.path.dirname(fn_zipped), os.path.basename(fn_zipped))) if not os.path.exists(fn_in): raise RuntimeError("Unexpected") #Cons.P(fn_in) mo_list = [] line_params = None line_run = None with open(fn_in) as fo: for line in fo: #Cons.P(line) # 2017-10-13 20:41:01:258 2 sec: 34 operations; 34 current ops/sec; est completion in 68 days 1 hours [READ: Count=28, Max=46943, Min=33, # Avg=32239.54, 90=45343, 99=46943, 99.9=46943, 99.99=46943] [INSERT: Count=8, Max=9343, Min=221, Avg=4660.88, 90=8695, 99=9343, 99.9=9343, # 99.99=9343] mo = re.match(r"\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d:\d\d\d (?P<rel_time>\d+) sec: \d+ operations; " \ "(?P<db_iops>(\d|\.)+) current ops\/sec; .*" \ "\[READ: Count=(?P<r_cnt>\d+), Max=(?P<r_max>\d+), Min=(?P<r_min>\d+), Avg=(?P<r_avg>(\d|\.)+)," \ " 90=(?P<r_90>\d+), 99=(?P<r_99>\d+), 99.9=(?P<r_999>\d+), 99.99=(?P<r_9999>\d+)\] " \ "\[INSERT: Count=(?P<w_cnt>\d+), Max=(?P<w_max>\d+), Min=(?P<w_min>\d+), Avg=(?P<w_avg>(\d|\.)+)," \ " 90=(?P<w_90>\d+), 99=(?P<w_99>\d+), 99.9=(?P<w_999>\d+), 99.99=(?P<w_9999>\d+)\] " \ , line) if mo is None: continue total_seconds = int(mo.group("rel_time")) s = total_seconds % 60 total_seconds -= s total_mins = total_seconds / 60 m = total_mins % 60 total_mins -= m h = total_mins / 60 rel_time = "%02d:%02d:%02d" % (h, m, s) if (time_begin <= rel_time) and (rel_time <= time_end): mo_list.append((rel_time, mo)) if len(mo_list) == 0: raise RuntimeError("Unexpected. Check file [%s]" % fn_in) cnt = 0 db_iops = [] r_cnt = 0 r_avg = 0.0 r_min = 0 r_max = 0 r_90 = 0 r_99 = 0 r_999 = 0 r_9999 = 0 w_cnt = 0 w_avg = 0.0 w_min = 0 w_max = 0 w_90 = 0 w_99 = 0 w_999 = 0 w_9999 = 0 for e in mo_list: rel_time = e[0] mo = e[1] db_iops.append(float(mo.group("db_iops"))) r_cnt += int(mo.group("r_cnt")) r_avg += float(mo.group("r_avg")) r_min += int(mo.group("r_min")) r_max += int(mo.group("r_max")) r_90 += int(mo.group("r_90")) r_99 += int(mo.group("r_99")) r_999 += int(mo.group("r_999")) r_9999 += int(mo.group("r_9999")) w_cnt += int(mo.group("w_cnt")) w_avg += float(mo.group("w_avg")) w_min += int(mo.group("w_min")) w_max += int(mo.group("w_max")) w_90 += int(mo.group("w_90")) w_99 += int(mo.group("w_99")) w_999 += int(mo.group("w_999")) w_9999 += int(mo.group("w_9999")) cnt += 1 self.db_iops_stat = Stat.Gen(db_iops) self.r_cnt = r_cnt self.r_avg = (float(r_avg) / cnt) self.r_min = (float(r_min) / cnt) self.r_max = (float(r_max) / cnt) self.r_90 = (float(r_90) / cnt) self.r_99 = (float(r_99) / cnt) self.r_999 = (float(r_999) / cnt) self.r_9999 = (float(r_9999) / cnt) self.w_cnt = (float(w_cnt) / cnt) self.w_avg = (float(w_avg) / cnt) self.w_min = (float(w_min) / cnt) self.w_max = (float(w_max) / cnt) self.w_90 = (float(w_90) / cnt) self.w_99 = (float(w_99) / cnt) self.w_999 = (float(w_999) / cnt) self.w_9999 = (float(w_9999) / cnt)
def WriteLat(self): if self.w_stat is not None: return self.w_stat with Cons.MT("Generating write latency stat ..."): self.w_stat = Stat.Gen(self.w_raw) return self.w_stat
def __init__(self, fn_in, exp_dt): self.fn_out = "%s/ycsb-by-time-%s" % (Conf.GetOutDir(), exp_dt) if os.path.isfile(self.fn_out): return with Cons.MT("Generating ycsb time-vs-metrics file for plot ..."): # Unzip when the file is not there if not os.path.exists(fn_in): fn_zipped = "%s.bz2" % fn_in if not os.path.exists(fn_zipped): raise RuntimeError("Unexpected: %s" % fn_in) Util.RunSubp("cd %s && bzip2 -dk %s > /dev/null" % (os.path.dirname(fn_zipped), os.path.basename(fn_zipped))) if not os.path.exists(fn_in): raise RuntimeError("Unexpected") #Cons.P(fn_in) mo_list = [] line_params = None line_run = None with open(fn_in) as fo: for line in fo: #Cons.P(line) # 2017-10-13 20:41:01:258 2 sec: 34 operations; 34 current ops/sec; est completion in 68 days 1 hours [READ: Count=28, Max=46943, Min=33, # Avg=32239.54, 90=45343, 99=46943, 99.9=46943, 99.99=46943] [INSERT: Count=8, Max=9343, Min=221, Avg=4660.88, 90=8695, 99=9343, 99.9=9343, # 99.99=9343] mo = re.match(r"\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d:\d\d\d (?P<rel_time>\d+) sec: \d+ operations; " \ "(?P<db_iops>(\d|\.)+) current ops\/sec; .*" \ "\[READ: Count=(?P<r_cnt>\d+), Max=(?P<r_max>\d+), Min=(?P<r_min>\d+), Avg=(?P<r_avg>(\d|\.)+)," \ " 90=(?P<r_90>\d+), 99=(?P<r_99>\d+), 99.9=(?P<r_999>\d+), 99.99=(?P<r_9999>\d+)\] " \ "\[INSERT: Count=(?P<w_cnt>\d+), Max=(?P<w_max>\d+), Min=(?P<w_min>\d+), Avg=(?P<w_avg>(\d|\.)+)," \ " 90=(?P<w_90>\d+), 99=(?P<w_99>\d+), 99.9=(?P<w_999>\d+), 99.99=(?P<w_9999>\d+)\] " \ , line) if mo is not None: total_seconds = int(mo.group("rel_time")) s = total_seconds % 60 total_seconds -= s total_mins = total_seconds / 60 m = total_mins % 60 total_mins -= m h = total_mins / 60 rel_time = "%02d:%02d:%02d" % (h, m, s) mo_list.append((rel_time, mo)) continue if line.startswith("params = {"): line_params = line continue if line.startswith("run = {"): line_run = line continue if len(mo_list) == 0: raise RuntimeError("Unexpected. Check file [%s]" % fn_in) cnt = 0 db_iops = [] r_cnt = 0 r_avg = 0.0 r_min = 0 r_max = 0 r_90 = 0 r_99 = 0 r_999 = 0 r_9999 = 0 w_cnt = 0 w_avg = 0.0 w_min = 0 w_max = 0 w_90 = 0 w_99 = 0 w_999 = 0 w_9999 = 0 for e in mo_list: rel_time = e[0] mo = e[1] db_iops.append(float(mo.group("db_iops"))) r_cnt += int(mo.group("r_cnt")) r_avg += float(mo.group("r_avg")) r_min += int(mo.group("r_min")) r_max += int(mo.group("r_max")) r_90 += int(mo.group("r_90")) r_99 += int(mo.group("r_99")) r_999 += int(mo.group("r_999")) r_9999 += int(mo.group("r_9999")) w_cnt += int(mo.group("w_cnt")) w_avg += float(mo.group("w_avg")) w_min += int(mo.group("w_min")) w_max += int(mo.group("w_max")) w_90 += int(mo.group("w_90")) w_99 += int(mo.group("w_99")) w_999 += int(mo.group("w_999")) w_9999 += int(mo.group("w_9999")) cnt += 1 db_iops_stat = Stat.Gen(db_iops) with open(self.fn_out, "w") as fo_out: fo_out.write("# %s" % line_params) fo_out.write("# %s" % line_run) fo_out.write("\n") fo_out.write("# db_iops.avg= %14f\n" % db_iops_stat.avg) fo_out.write("# db_iops.min= %14f\n" % db_iops_stat.min) fo_out.write("# db_iops.max= %14f\n" % db_iops_stat.max) fo_out.write("# db_iops._25= %14f\n" % db_iops_stat._25) fo_out.write("# db_iops._50= %14f\n" % db_iops_stat._50) fo_out.write("# db_iops._75= %14f\n" % db_iops_stat._75) fo_out.write("# r_cnt = %14f\n" % (float(r_cnt ) / cnt)) fo_out.write("# r_avg = %14f\n" % (float(r_avg ) / cnt)) fo_out.write("# r_min = %14f\n" % (float(r_min ) / cnt)) fo_out.write("# r_max = %14f\n" % (float(r_max ) / cnt)) fo_out.write("# r_90 = %14f\n" % (float(r_90 ) / cnt)) fo_out.write("# r_99 = %14f\n" % (float(r_99 ) / cnt)) fo_out.write("# r_999 = %14f\n" % (float(r_999 ) / cnt)) fo_out.write("# r_9999 = %14f\n" % (float(r_9999) / cnt)) fo_out.write("# w_cnt = %14f\n" % (float(w_cnt ) / cnt)) fo_out.write("# w_avg = %14f\n" % (float(w_avg ) / cnt)) fo_out.write("# w_min = %14f\n" % (float(w_min ) / cnt)) fo_out.write("# w_max = %14f\n" % (float(w_max ) / cnt)) fo_out.write("# w_90 = %14f\n" % (float(w_90 ) / cnt)) fo_out.write("# w_99 = %14f\n" % (float(w_99 ) / cnt)) fo_out.write("# w_999 = %14f\n" % (float(w_999 ) / cnt)) fo_out.write("# w_9999 = %14f\n" % (float(w_9999) / cnt)) fo_out.write("\n") fmt = "%8s" \ " %9.2f" \ " %6d %8.2f %3d %6d" \ " %6d %6d %6d %6d" \ " %6d %8.2f %3d %6d" \ " %6d %6d %6d %6d" header = Util.BuildHeader(fmt, "rel_time" \ " db_iops" \ " read_cnt read_lat_avg read_lat_min read_lat_max" \ " read_lat_90p read_lat_99p read_lat_99.9p read_lat_99.99p" \ " write_cnt write_lat_avg write_lat_min write_lat_max" \ " write_lat_90p write_lat_99p write_lat_99.9p write_lat_99.99p" \ ) i = 0 for e in mo_list: rel_time = e[0] mo = e[1] if i % 40 == 0: fo_out.write(header + "\n") fo_out.write((fmt + "\n") % ( rel_time , float(mo.group("db_iops")) , int(mo.group("r_cnt")) , float(mo.group("r_avg")) , int(mo.group("r_min")) , int(mo.group("r_max")) , int(mo.group("r_90")) , int(mo.group("r_99")) , int(mo.group("r_999")) , int(mo.group("r_9999")) , int(mo.group("w_cnt")) , float(mo.group("w_avg")) , int(mo.group("w_min")) , int(mo.group("w_max")) , int(mo.group("w_90")) , int(mo.group("w_99")) , int(mo.group("w_999")) , int(mo.group("w_9999")) )) i += 1 Cons.P("Created %s %d" % (self.fn_out, os.path.getsize(self.fn_out)))
def ReadLat(self): if self.r_stat is not None: return self.r_stat with Cons.MT("Generating read latency stat ..."): self.r_stat = Stat.Gen(self.r_raw) return self.r_stat