def __init__(self, fn_in, time_begin, time_end):
#self.overloaded = overloaded
# Unzip when the file is not there
if not os.path.exists(fn_in):
fn_zipped = "%s.bz2" % fn_in
if not os.path.exists(fn_zipped):
raise RuntimeError("Unexpected: %s" % fn_in)
Util.RunSubp(
"cd %s && bzip2 -dk %s > /dev/null" %
(os.path.dirname(fn_zipped), os.path.basename(fn_zipped)))
if not os.path.exists(fn_in):
raise RuntimeError("Unexpected")
#Cons.P(fn_in)
mo_list = []
line_params = None
line_run = None
with open(fn_in) as fo:
for line in fo:
#Cons.P(line)
# 2017-10-13 20:41:01:258 2 sec: 34 operations; 34 current ops/sec; est completion in 68 days 1 hours [READ: Count=28, Max=46943, Min=33,
# Avg=32239.54, 90=45343, 99=46943, 99.9=46943, 99.99=46943] [INSERT: Count=8, Max=9343, Min=221, Avg=4660.88, 90=8695, 99=9343, 99.9=9343,
# 99.99=9343]
mo = re.match(r"\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d:\d\d\d (?P<rel_time>\d+) sec: \d+ operations; " \
"(?P<db_iops>(\d|\.)+) current ops\/sec; .*" \
"\[READ: Count=(?P<r_cnt>\d+), Max=(?P<r_max>\d+), Min=(?P<r_min>\d+), Avg=(?P<r_avg>(\d|\.)+)," \
" 90=(?P<r_90>\d+), 99=(?P<r_99>\d+), 99.9=(?P<r_999>\d+), 99.99=(?P<r_9999>\d+)\] " \
"\[INSERT: Count=(?P<w_cnt>\d+), Max=(?P<w_max>\d+), Min=(?P<w_min>\d+), Avg=(?P<w_avg>(\d|\.)+)," \
" 90=(?P<w_90>\d+), 99=(?P<w_99>\d+), 99.9=(?P<w_999>\d+), 99.99=(?P<w_9999>\d+)\] " \
, line)
if mo is None:
continue
total_seconds = int(mo.group("rel_time"))
s = total_seconds % 60
total_seconds -= s
total_mins = total_seconds / 60
m = total_mins % 60
total_mins -= m
h = total_mins / 60
rel_time = "%02d:%02d:%02d" % (h, m, s)
if (time_begin <= rel_time) and (rel_time <= time_end):
mo_list.append((rel_time, mo))
if len(mo_list) == 0:
raise RuntimeError("Unexpected. Check file [%s]" % fn_in)
cnt = 0
db_iops = []
r_cnt = 0
r_avg = 0.0
r_min = 0
r_max = 0
r_90 = 0
r_99 = 0
r_999 = 0
r_9999 = 0
w_cnt = 0
w_avg = 0.0
w_min = 0
w_max = 0
w_90 = 0
w_99 = 0
w_999 = 0
w_9999 = 0
for e in mo_list:
rel_time = e[0]
mo = e[1]
db_iops.append(float(mo.group("db_iops")))
r_cnt += int(mo.group("r_cnt"))
r_avg += float(mo.group("r_avg"))
r_min += int(mo.group("r_min"))
r_max += int(mo.group("r_max"))
r_90 += int(mo.group("r_90"))
r_99 += int(mo.group("r_99"))
r_999 += int(mo.group("r_999"))
r_9999 += int(mo.group("r_9999"))
w_cnt += int(mo.group("w_cnt"))
w_avg += float(mo.group("w_avg"))
w_min += int(mo.group("w_min"))
w_max += int(mo.group("w_max"))
w_90 += int(mo.group("w_90"))
w_99 += int(mo.group("w_99"))
w_999 += int(mo.group("w_999"))
w_9999 += int(mo.group("w_9999"))
cnt += 1
self.db_iops_stat = Stat.Gen(db_iops)
self.r_cnt = r_cnt
self.r_avg = (float(r_avg) / cnt)
self.r_min = (float(r_min) / cnt)
self.r_max = (float(r_max) / cnt)
self.r_90 = (float(r_90) / cnt)
self.r_99 = (float(r_99) / cnt)
self.r_999 = (float(r_999) / cnt)
self.r_9999 = (float(r_9999) / cnt)
self.w_cnt = (float(w_cnt) / cnt)
self.w_avg = (float(w_avg) / cnt)
self.w_min = (float(w_min) / cnt)
self.w_max = (float(w_max) / cnt)
self.w_90 = (float(w_90) / cnt)
self.w_99 = (float(w_99) / cnt)
self.w_999 = (float(w_999) / cnt)
self.w_9999 = (float(w_9999) / cnt)
def WriteLat(self):
if self.w_stat is not None:
return self.w_stat
with Cons.MT("Generating write latency stat ..."):
self.w_stat = Stat.Gen(self.w_raw)
return self.w_stat
def __init__(self, fn_in, exp_dt):
self.fn_out = "%s/ycsb-by-time-%s" % (Conf.GetOutDir(), exp_dt)
if os.path.isfile(self.fn_out):
return
with Cons.MT("Generating ycsb time-vs-metrics file for plot ..."):
# Unzip when the file is not there
if not os.path.exists(fn_in):
fn_zipped = "%s.bz2" % fn_in
if not os.path.exists(fn_zipped):
raise RuntimeError("Unexpected: %s" % fn_in)
Util.RunSubp("cd %s && bzip2 -dk %s > /dev/null" % (os.path.dirname(fn_zipped), os.path.basename(fn_zipped)))
if not os.path.exists(fn_in):
raise RuntimeError("Unexpected")
#Cons.P(fn_in)
mo_list = []
line_params = None
line_run = None
with open(fn_in) as fo:
for line in fo:
#Cons.P(line)
# 2017-10-13 20:41:01:258 2 sec: 34 operations; 34 current ops/sec; est completion in 68 days 1 hours [READ: Count=28, Max=46943, Min=33,
# Avg=32239.54, 90=45343, 99=46943, 99.9=46943, 99.99=46943] [INSERT: Count=8, Max=9343, Min=221, Avg=4660.88, 90=8695, 99=9343, 99.9=9343,
# 99.99=9343]
mo = re.match(r"\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d:\d\d\d (?P<rel_time>\d+) sec: \d+ operations; " \
"(?P<db_iops>(\d|\.)+) current ops\/sec; .*" \
"\[READ: Count=(?P<r_cnt>\d+), Max=(?P<r_max>\d+), Min=(?P<r_min>\d+), Avg=(?P<r_avg>(\d|\.)+)," \
" 90=(?P<r_90>\d+), 99=(?P<r_99>\d+), 99.9=(?P<r_999>\d+), 99.99=(?P<r_9999>\d+)\] " \
"\[INSERT: Count=(?P<w_cnt>\d+), Max=(?P<w_max>\d+), Min=(?P<w_min>\d+), Avg=(?P<w_avg>(\d|\.)+)," \
" 90=(?P<w_90>\d+), 99=(?P<w_99>\d+), 99.9=(?P<w_999>\d+), 99.99=(?P<w_9999>\d+)\] " \
, line)
if mo is not None:
total_seconds = int(mo.group("rel_time"))
s = total_seconds % 60
total_seconds -= s
total_mins = total_seconds / 60
m = total_mins % 60
total_mins -= m
h = total_mins / 60
rel_time = "%02d:%02d:%02d" % (h, m, s)
mo_list.append((rel_time, mo))
continue
if line.startswith("params = {"):
line_params = line
continue
if line.startswith("run = {"):
line_run = line
continue
if len(mo_list) == 0:
raise RuntimeError("Unexpected. Check file [%s]" % fn_in)
cnt = 0
db_iops = []
r_cnt = 0
r_avg = 0.0
r_min = 0
r_max = 0
r_90 = 0
r_99 = 0
r_999 = 0
r_9999 = 0
w_cnt = 0
w_avg = 0.0
w_min = 0
w_max = 0
w_90 = 0
w_99 = 0
w_999 = 0
w_9999 = 0
for e in mo_list:
rel_time = e[0]
mo = e[1]
db_iops.append(float(mo.group("db_iops")))
r_cnt += int(mo.group("r_cnt"))
r_avg += float(mo.group("r_avg"))
r_min += int(mo.group("r_min"))
r_max += int(mo.group("r_max"))
r_90 += int(mo.group("r_90"))
r_99 += int(mo.group("r_99"))
r_999 += int(mo.group("r_999"))
r_9999 += int(mo.group("r_9999"))
w_cnt += int(mo.group("w_cnt"))
w_avg += float(mo.group("w_avg"))
w_min += int(mo.group("w_min"))
w_max += int(mo.group("w_max"))
w_90 += int(mo.group("w_90"))
w_99 += int(mo.group("w_99"))
w_999 += int(mo.group("w_999"))
w_9999 += int(mo.group("w_9999"))
cnt += 1
db_iops_stat = Stat.Gen(db_iops)
with open(self.fn_out, "w") as fo_out:
fo_out.write("# %s" % line_params)
fo_out.write("# %s" % line_run)
fo_out.write("\n")
fo_out.write("# db_iops.avg= %14f\n" % db_iops_stat.avg)
fo_out.write("# db_iops.min= %14f\n" % db_iops_stat.min)
fo_out.write("# db_iops.max= %14f\n" % db_iops_stat.max)
fo_out.write("# db_iops._25= %14f\n" % db_iops_stat._25)
fo_out.write("# db_iops._50= %14f\n" % db_iops_stat._50)
fo_out.write("# db_iops._75= %14f\n" % db_iops_stat._75)
fo_out.write("# r_cnt = %14f\n" % (float(r_cnt ) / cnt))
fo_out.write("# r_avg = %14f\n" % (float(r_avg ) / cnt))
fo_out.write("# r_min = %14f\n" % (float(r_min ) / cnt))
fo_out.write("# r_max = %14f\n" % (float(r_max ) / cnt))
fo_out.write("# r_90 = %14f\n" % (float(r_90 ) / cnt))
fo_out.write("# r_99 = %14f\n" % (float(r_99 ) / cnt))
fo_out.write("# r_999 = %14f\n" % (float(r_999 ) / cnt))
fo_out.write("# r_9999 = %14f\n" % (float(r_9999) / cnt))
fo_out.write("# w_cnt = %14f\n" % (float(w_cnt ) / cnt))
fo_out.write("# w_avg = %14f\n" % (float(w_avg ) / cnt))
fo_out.write("# w_min = %14f\n" % (float(w_min ) / cnt))
fo_out.write("# w_max = %14f\n" % (float(w_max ) / cnt))
fo_out.write("# w_90 = %14f\n" % (float(w_90 ) / cnt))
fo_out.write("# w_99 = %14f\n" % (float(w_99 ) / cnt))
fo_out.write("# w_999 = %14f\n" % (float(w_999 ) / cnt))
fo_out.write("# w_9999 = %14f\n" % (float(w_9999) / cnt))
fo_out.write("\n")
fmt = "%8s" \
" %9.2f" \
" %6d %8.2f %3d %6d" \
" %6d %6d %6d %6d" \
" %6d %8.2f %3d %6d" \
" %6d %6d %6d %6d"
header = Util.BuildHeader(fmt, "rel_time" \
" db_iops" \
" read_cnt read_lat_avg read_lat_min read_lat_max" \
" read_lat_90p read_lat_99p read_lat_99.9p read_lat_99.99p" \
" write_cnt write_lat_avg write_lat_min write_lat_max" \
" write_lat_90p write_lat_99p write_lat_99.9p write_lat_99.99p" \
)
i = 0
for e in mo_list:
rel_time = e[0]
mo = e[1]
if i % 40 == 0:
fo_out.write(header + "\n")
fo_out.write((fmt + "\n") % (
rel_time
, float(mo.group("db_iops"))
, int(mo.group("r_cnt")) , float(mo.group("r_avg")) , int(mo.group("r_min")) , int(mo.group("r_max"))
, int(mo.group("r_90")) , int(mo.group("r_99")) , int(mo.group("r_999")) , int(mo.group("r_9999"))
, int(mo.group("w_cnt")) , float(mo.group("w_avg")) , int(mo.group("w_min")) , int(mo.group("w_max"))
, int(mo.group("w_90")) , int(mo.group("w_99")) , int(mo.group("w_999")) , int(mo.group("w_9999"))
))
i += 1
Cons.P("Created %s %d" % (self.fn_out, os.path.getsize(self.fn_out)))
def ReadLat(self):
if self.r_stat is not None:
return self.r_stat
with Cons.MT("Generating read latency stat ..."):
self.r_stat = Stat.Gen(self.r_raw)
return self.r_stat
