def SstLevelInfo():
# Set Conf.ExpStartTime(), if not already set.
if Conf.ExpStartTime() is None:
MutantLogReader.Get()
fn = "%s/sst-info-by-time-by-levels-level-seps-%s" % (Conf.dn_result,
Conf.ExpStartTime())
if os.path.isfile(fn):
return fn
sst_y_cord_level_sep_highs = SstYCord.LevelSepHigh()
with Cons.MT(
"Generating Sst info by time by levels: level separators data file ..."
):
with open(fn, "w") as fo:
fmt = "%1d %10d %10s"
fo.write("%s\n" % Util.BuildHeader(
fmt, "level level_mid_for_labels level_low_for_separators"))
lh_prev = 0
for l, lh in sorted(sst_y_cord_level_sep_highs.iteritems()):
lm = (lh + lh_prev) / 2
fo.write((fmt + "\n") % (l, lm, lh_prev))
lh_prev = lh
Cons.P("Created %s %d" % (fn, os.path.getsize(fn)))
return fn
def _ReadStoredLog():
if Conf.ExpStartTime() is None:
return None
dn = "%s/work/mutant/misc/logs/cassandra" % os.path.expanduser("~")
fn = "%s/system-%s" % (dn, Conf.ExpStartTime())
if not os.path.isfile(fn):
# If there is a 7z file, uncompress it
fn_7z = "%s.7z" % fn
if os.path.isfile(fn_7z):
with Cons.MT("Found a 7z file. Uncompressing"):
Util.RunSubp("7z e -o%s %s" % (dn, fn_7z))
else:
return None
with Cons.MT("Reading the stored Cassandra Mutant log file %s" % fn, print_time=False):
lines = []
with open(fn) as fo:
for line in fo.readlines():
lines.append(line.strip())
# Stop after reading n lines for testing
if 0 < Conf.MaxCassLogLines():
if Conf.MaxCassLogLines() < len(lines):
break
#Cons.P(len(lines))
return lines
def SstInfo():
# Set Conf.ExpStartTime(), if not already set.
if Conf.ExpStartTime() is None:
MutantLogReader.Get()
fn = "%s/sst-info-by-time-by-levels-%s" % (Conf.dn_result,
Conf.ExpStartTime())
if os.path.isfile(fn):
return fn
(sst_lives, memt_lives) = MemtSstLife.Get()
with Cons.MT("Generating Sst info by time by levels data file ..."):
#with open(fn_m, "w") as fo:
# fo.write("%s\n" % Memt.Header())
# for addr, l in sorted(_memt_lives.iteritems()):
# fo.write("%s\n" % l)
#Cons.P("Created %s %d" % (fn_m, os.path.getsize(fn_m)))
with open(fn, "w") as fo:
fo.write("%s\n" % MemtSstLife.SstLife.Header())
for sst_gen, l in sorted(sst_lives.iteritems()):
fo.write("%s\n" % l)
Cons.P("Created %s %d" % (fn, os.path.getsize(fn)))
return fn
def SstHeatAtLastTime():
# Set Conf.ExpStartTime(), if not already set.
if Conf.ExpStartTime() is None:
MutantLogReader.Get()
fn_hlt = "%s/sst-heat-last-time-%s" % (Conf.dn_result, Conf.ExpStartTime())
if os.path.isfile(fn_hlt):
return fn_hlt
sst_lives = MemtSstLife.GetSstLives()
with Cons.MT("Generating Sst heats at the last time ..."):
# Gather heat info at n different times
num_times = Conf.heatmap_by_time_num_times
if Conf.ExpFinishTime() is None:
MemtSstLife.SetExpEndTimeFromSstLives()
min_sst_opened = None
for sst_gen, sl in sorted(sst_lives.iteritems()):
min_sst_opened = sl.Opened() if min_sst_opened is None else min(
min_sst_opened, sl.Opened())
# Start time is when the first Sstable is opened, not the experiment start
# time, when no SSTable exists yet.
# Exp start time: 160927-143257.395
# First Sstable open time: 160927-143411.273
st = datetime.datetime.strptime(min_sst_opened, "%y%m%d-%H%M%S.%f")
et = datetime.datetime.strptime(Conf.ExpFinishTime(),
"%y%m%d-%H%M%S.%f")
dur = (et - st).total_seconds()
sstgen_heat = []
t = st + datetime.timedelta(seconds=(float(dur) *
(num_times - 1) / num_times +
time_offset_in_sec))
for sst_gen, sl in sorted(sst_lives.iteritems()):
h = sl.HeatAtTime(t)
if h is None:
continue
sstgen_heat.append((sst_gen, h))
sstgen_heat.sort(key=lambda sh: sh[1], reverse=True)
# Note: Don't bother with the width proportional to the tablet size for now
fmt = "%4d %1d %8.3f"
with open(fn_hlt, "w") as fo:
# y0 is smaller than y1 (y0 is placed higher in the plot than y1).
fo.write("%s\n" % Util.BuildHeader(fmt, "sst_gen level heat"))
for sh in sstgen_heat:
sst_gen = sh[0]
heat = sh[1]
fo.write(
(fmt + "\n") % (sst_gen, sst_lives[sst_gen].level, heat))
Cons.P("Created %s %d" % (fn_hlt, os.path.getsize(fn_hlt)))
return fn_hlt
def LoadSstLivesFromPlotDataFiles():
global _sst_lives
if _sst_lives is not None:
return _sst_lives
fn_sst_info_by_time = "%s/sst-info-by-time-by-levels-%s" % (
Conf.dn_result, Conf.ExpStartTime())
if not os.path.isfile(fn_sst_info_by_time):
return None
fn_sst_heat_by_time = "%s/sst-heat-by-time-by-levels-%s" % (
Conf.dn_result, Conf.ExpStartTime())
if not os.path.isfile(fn_sst_heat_by_time):
return None
# This is not needed for the by-levels-with-heat plot
# "%s/sst-info-by-time-by-levels-%s" % (Conf.dn_result, Conf.ExpStartTime())
with Cons.MT("Loading Sst lives from (%s, %s) ..." %
(fn_sst_info_by_time, fn_sst_heat_by_time)):
_sst_lives = {}
with open(fn_sst_info_by_time) as fo:
for line in fo.readlines():
if len(line) == 0:
continue
if line[0] == "#":
continue
line = line.strip()
#Cons.P(line)
t = re.split(r" +", line)
sst_gen = int(t[0])
sl = SstLife(sst_gen)
sl.SetInfoFromPlotData(t)
_sst_lives[sst_gen] = sl
with open(fn_sst_heat_by_time) as fo:
sstgen_lines_tokens = {}
for line in fo.readlines():
if len(line) == 0:
continue
if line[0] == "#":
continue
line = line.strip()
if len(line) == 0:
continue
t = re.split(r" +", line)
if len(t) == 0:
continue
sst_gen = int(t[0])
if sst_gen not in sstgen_lines_tokens:
sstgen_lines_tokens[sst_gen] = []
sstgen_lines_tokens[sst_gen].append(t)
for sstgen, lt in sstgen_lines_tokens.iteritems():
_sst_lives[sstgen].SetHeatFromPlotData(lt)
return _sst_lives
def PlotSstAccDistAtSpecificTimes():
# At the time m sec after the n-th SSTable is created (time t). To get the
# max_plot_hgieht, all plot data files need to be generated before plotting
# the first one.
plot_data_fns_at_n = {}
with Cons.MT(
"Generating plot data for SSTables by levels with heat at specific times ..."
):
for (n, m) in Conf.times_sst_by_levels_with_heat:
(fn_in_boxes,
fn_in_level_seps) = SstByLevelsWithHeatAtSpecificTimes.Boxes(
n, m)
plot_data_fns_at_n[n] = (fn_in_boxes, fn_in_level_seps)
with Cons.MT(
"Plotting SSTables by levels with heat at specific times ..."):
dn = "%s/sst-by-level-by-ks-range-with-heat" % Conf.dn_result
for n, (fn_in_boxes,
fn_in_level_seps) in sorted(plot_data_fns_at_n.iteritems()):
env = os.environ.copy()
env["FN_IN_BOXES"] = fn_in_boxes
env["FN_IN_LEVEL_INFO"] = fn_in_level_seps
env["MAX_PLOT_HEIGHT"] = str(
SstByLevelsWithHeatAtSpecificTimes.max_plot_height)
fn_out = "%s/sst-by-level-by-ks-range-with-heat-%s-%s.pdf" % (
dn, Conf.ExpStartTime(), n)
env["FN_OUT"] = fn_out
Util.RunSubp(
"gnuplot %s/sst-by-level-by-ks-range-with-heat-at-specific-time.gnuplot"
% os.path.dirname(__file__),
env=env)
Cons.P("Created %s %d" % (fn_out, os.path.getsize(fn_out)))
def _ReadAndCacheCassLog():
with Cons.MT("Reading Cassandra log ..."):
lines = []
found_reset = False
lines1 = []
# WARN [main] 2016-09-20 02:20:39,250 MemSsTableAccessMon.java:115 - Mutant: ResetMon
pattern = re.compile(r"WARN \[(main|MigrationStage:\d+)\]" \
" (?P<datetime>\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d,\d\d\d)" \
" MemSsTableAccessMon.java:\d+ -" \
" Mutant: (?P<event>ResetMon)")
# Note: s0 only for now.
dn = "%s/work/mutant/log/%s/s0/cassandra" % (os.path.expanduser("~"), Util0.JobId())
fn = "%s/system.log" % dn
Cons.P("fn=%s" % fn)
with open(fn) as fo:
for line in fo.readlines():
line = line.strip()
mo = pattern.match(line)
if mo is not None:
found_reset = True
Conf.SetExpStartTime(Util0.ShortDateTime(mo.group("datetime")))
del lines[:]
lines.append(line)
# Keep reading zipped files like system.log.1.zip, until ResetMon is found
i = 1
while found_reset == False:
fn = "%s/system.log.%d.zip" % (dn, i)
Cons.P("ResetMon not found. Reading more from file %s ..." % fn)
with zipfile.ZipFile(fn, "r") as z:
for fn1 in z.namelist():
#Cons.P(fn1)
for line in z.read(fn1).split("\n"):
line = line.strip()
#Cons.P(line)
mo = pattern.match(line)
if mo is not None:
found_reset = True
Conf.SetExpStartTime(Util0.ShortDateTime(mo.group("datetime")))
del lines1[:]
lines1.append(line)
if len(lines1) != 0:
lines1.extend(lines)
lines = list(lines1)
del lines1[:]
i += 1
fn = "%s/work/mutant/misc/logs/cassandra/system-%s" \
% (os.path.expanduser("~"), Conf.ExpStartTime())
with open(fn, "w") as fo:
for line in lines:
fo.write("%s\n" % line)
Cons.P("Created a Cassandra log file %s %d" % (fn, os.path.getsize(fn)))
return lines
def PlotSstHeatAtLastTime():
# X-axis: Sstables
# Y-axis: Heat
# Sst heatmap boxes by time
env = os.environ.copy()
env["FN_IN"] = SstHeatmapByTime.SstHeatAtLastTime()
fn_out = "%s/sst-heat-at-last-time-%s.pdf" % (Conf.dn_result, Conf.ExpStartTime())
env["FN_OUT"] = fn_out
with Cons.MT("Plotting SSTable heatmap by time ..."):
Util.RunSubp("gnuplot %s/sst-heat-at-last-time.gnuplot" % os.path.dirname(__file__), env=env)
Cons.P("Created %s %d" % (fn_out, os.path.getsize(fn_out)))
def __init__(self):
if Conf.ExpStartTime() is None:
raise RuntimeError("Unexpected")
dn = "%s/work/mutant/misc/logs/cassandra" % os.path.expanduser("~")
self.fn_compaction_log = "%s/compaction-%s" % (dn, Conf.ExpStartTime())
# If not exist and there is a 7z file, uncompress it
if not os.path.isfile(self.fn_compaction_log):
fn_7z = "%s.7z" % self.fn_compaction_log
if os.path.isfile(fn_7z):
with Cons.MT("Found a 7z file. Uncompressing"):
Util.RunSubp("7z e -o%s %s" % (dn, fn_7z))
# If still not exist, Copy the current compaction log file
if not os.path.isfile(self.fn_compaction_log):
dn1 = "%s/work/mutant/log/%s/s0/cassandra" % (
os.path.expanduser("~"), Util0.JobId())
Util.RunSubp("cp %s/compaction.log %s" \
% (dn1, self.fn_compaction_log))
with Cons.MT("Reading compaction log %s" % self.fn_compaction_log,
print_time=False):
self.Read()
def PlotSstHeatmapByTime():
# X-axis: time
# Y-axis: total storage space (depending on the workload, it can grow as time
# goes by)
# Sst heatmap boxes by time
env = os.environ.copy()
(env["FN_IN_HEATMAP"], env["FN_IN_VERTICAL_LINES"], heat_max) = SstHeatmapByTime.Heatmap()
env["HEAT_MAX"] = str(heat_max)
fn_out = "%s/sst-heatmap-by-time-%s.pdf" % (Conf.dn_result, Conf.ExpStartTime())
env["FN_OUT"] = fn_out
with Cons.MT("Plotting SSTable heatmap by time ..."):
Util.RunSubp("gnuplot %s/sst-heatmap-by-time.gnuplot" % os.path.dirname(__file__), env=env)
Cons.P("Created %s %d" % (fn_out, os.path.getsize(fn_out)))
def PlotTabletByTimeByLevelWithHeat():
with Cons.MT("Plotting Memtable and SSTables by time by levels with heat ..."):
env = os.environ.copy()
# Memtables are not plotted for now
#env["FN_IN_MEMT"] = GetMemtByTimeData()
env["FN_IN_SST_INFO"] = SstInfoByTimeByLevel.SstInfo()
env["FN_IN_SST_LEVEL_INFO"] = SstInfoByTimeByLevel.SstLevelInfo()
env["FN_IN_SST_HEAT"] = SstInfoByTimeByLevel.SstHeatByTimeByLevel()
fn_out = "%s/sst-by-time-by-levels-with-heat-%s.pdf" % (Conf.dn_result, Conf.ExpStartTime())
env["FN_OUT"] = fn_out
with Cons.MT("Plotting ..."):
#Util.RunSubp("gnuplot %s/tablet-timeline.gnuplot" % os.path.dirname(__file__), env=env)
Util.RunSubp("gnuplot %s/sst-by-time-by-levels-with-heat.gnuplot" % os.path.dirname(__file__), env=env)
Cons.P("Created %s %d" % (fn_out, os.path.getsize(fn_out)))
def SstHeatByTimeByLevel():
# Set Conf.ExpStartTime(), if not already set.
if Conf.ExpStartTime() is None:
MutantLogReader.Get()
fn = "%s/sst-heat-by-time-by-levels-%s" % (Conf.dn_result, Conf.ExpStartTime())
if os.path.isfile(fn):
return fn
(sst_lives, memt_lives) = MemtSstLife.Get()
with Cons.MT("Generating SSTable by time by levels with heat data ..."):
with open(fn, "w") as fo:
fmt = "%4d %1d %17s %17s %8.1f %13d %13d %8.3f %8d %6s"
fo.write("%s\n" % Util.BuildHeader(fmt
, "sst_gen level time0 time1 age_in_sec y0 y1 heat(num_reads_per_sec_per_mb) heat_color heat_color_hex"))
num_heat_color_blocks_merged = 0
num_heat_color_blocks = 0
for sst_gen, sl in sorted(sst_lives.iteritems()):
# Some SSTables don't have any access stats at all, thus no heat info
if len(sl.heat_by_time) == 0:
continue
t0_begin = None
t0_prev = None
t1_prev = None
heat_prev = None
heat_color_prev = None
for t0, v in sl.heat_by_time.items():
t1 = v[0]
heat = v[1]
if t0_begin is None:
t0_begin = t0
v = heat / MemtSstLife.SstLife.max_heat
heat_color = TempColor.Get(v)
if heat_color == heat_color_prev:
# No change in t0_prev, heat_color_prev
t1_prev = t1
heat_prev = heat
num_heat_color_blocks_merged += 1
else:
if (t0_prev is not None) and (t1_prev is not None) \
and (heat_prev is not None) \
and (heat_color_prev is not None):
fo.write((fmt + "\n") \
% (sst_gen
, sst_lives[sst_gen].level
, t0_prev.strftime("%y%m%d-%H%M%S.%f")[:-3]
, t1_prev.strftime("%y%m%d-%H%M%S.%f")[:-3]
, (t1_prev - t0_begin).total_seconds()
, sst_lives[sst_gen].y_cord_low, sst_lives[sst_gen].YcordHigh()
, heat_prev
, heat_color_prev
, "%0.6X" % heat_color_prev
))
num_heat_color_blocks += 1
t0_prev = t0
t1_prev = t1
heat_prev = heat
heat_color_prev = heat_color
# The last one
if (t0_prev is not None) and (t1_prev is not None) \
and (heat_prev is not None) \
and (heat_color_prev is not None):
fo.write((fmt + "\n") \
% (sst_gen
, sst_lives[sst_gen].level
, t0_prev.strftime("%y%m%d-%H%M%S.%f")[:-3]
, t1_prev.strftime("%y%m%d-%H%M%S.%f")[:-3]
, (t1_prev - t0_begin).total_seconds()
, sst_lives[sst_gen].y_cord_low, sst_lives[sst_gen].YcordHigh()
, heat_prev
, heat_color_prev
, "%0.6X" % heat_color_prev
))
num_heat_color_blocks += 1
fo.write("\n")
Cons.P("num_heat_color_blocks_merged=%d" % num_heat_color_blocks_merged)
Cons.P("num_heat_color_blocks =%d" % num_heat_color_blocks)
Cons.P("Created %s %d" % (fn, os.path.getsize(fn)))
return fn
def _ReadAndCacheCassLogUntilDtCrossesStNotTested():
with Cons.MT("Reading Cassandra log ..."):
# Note: s0 only for now.
dn = "%s/work/mutant/log/%s/s0/cassandra" % (os.path.expanduser("~"), Util0.JobId())
# Start from system.log and keep reading system.log.n.zip where n >= 1,
# until you find two datetimes that cross the given exp start time
st = Conf.ExpStartTime()
ft = Conf.ExpFinishTime()
dt_prev = None
dt_crossed_st = False
# WARN [main] 2016-09-20 02:20:39,250 MemSsTableAccessMon.java:115 - Mutant: ...
pattern = re.compile(r".+" \
" (?P<datetime>\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d,\d\d\d)" \
" .+")
fn = "%s/system.log" % dn
lines = []
Cons.P("fn=%s" % fn)
with open(fn) as fo:
for line in fo.readlines():
line = line.strip()
mo = pattern.match(line)
if mo is None:
raise RuntimeError("Unexpected line=[%s]" % line)
dt = Util0.ShortDateTime(mo.group("datetime"))
if (st <= dt) and (dt <= ft):
lines.append(line)
if ft < dt:
break
if not dt_crossed_st:
if (dt_prev is not None) and (dt_prev < st) and (st < dt):
dt_crossed_st = True
dt_prev = dt
# Keep reading zipped files like system.log.1.zip, until ResetMon is found
i = 1
while dt_crossed_st == False:
fn = "%s/system.log.%d.zip" % (dn, i)
Cons.P("dt haven't crossed st yet. Reading more logs from file %s ..." % fn)
lines1 = []
with zipfile.ZipFile(fn, "r") as z:
dt_prev = None
for fn1 in z.namelist():
#Cons.P(fn1)
for line in z.read(fn1).split("\n"):
line = line.strip()
#Cons.P(line)
mo = pattern.match(line)
if mo is None:
raise RuntimeError("Unexpected line=[%s]" % line)
dt = Util0.ShortDateTime(mo.group("datetime"))
if (st <= dt) and (dt <= ft):
lines.append(line)
if ft < dt:
break
if not dt_crossed_st:
if (dt_prev is not None) and (dt_prev < st) and (st < dt):
dt_crossed_st = True
dt_prev = dt
lines1.append(line)
if len(lines1) != 0:
lines1.extend(lines)
lines = list(lines1)
del lines1[:]
i += 1
fn = "%s/work/mutant/misc/logs/cassandra/system-%s" \
% (os.path.expanduser("~"), Conf.ExpStartTime())
with open(fn, "w") as fo:
for line in lines:
fo.write("%s\n" % line)
Cons.P("Created a Cassandra log file %s %d" % (fn, os.path.getsize(fn)))
return lines
def Heatmap():
# Set Conf.ExpStartTime(), if not already set.
if Conf.ExpStartTime() is None:
MutantLogReader.Get()
fn_hm = "%s/sst-heatmap-by-time-%s" % (Conf.dn_result, Conf.ExpStartTime())
fn_vl = "%s/sst-heatmap-by-time-vertical-lines-%s" % (Conf.dn_result,
Conf.ExpStartTime())
if os.path.isfile(fn_hm) and os.path.isfile(fn_vl):
return (fn_hm, fn_vl, _MaxHeatFromHeatmapByTimeData(fn_hm))
sst_lives = MemtSstLife.GetSstLives()
with Cons.MT("Generating Sst heatmap by time ..."):
# Gather heat info at n different times
num_times = Conf.heatmap_by_time_num_times
if Conf.ExpFinishTime() is None:
MemtSstLife.SetExpEndTimeFromSstLives()
min_sst_opened = None
for sst_gen, sl in sorted(sst_lives.iteritems()):
min_sst_opened = sl.Opened() if min_sst_opened is None else min(
min_sst_opened, sl.Opened())
# Start time is when the first Sstable is opened, not the experiment start
# time, when no SSTable exists yet.
# Exp start time: 160927-143257.395
# First Sstable open time: 160927-143411.273
#st = datetime.datetime.strptime(Conf.ExpStartTime(), "%y%m%d-%H%M%S.%f")
st = datetime.datetime.strptime(min_sst_opened, "%y%m%d-%H%M%S.%f")
et = datetime.datetime.strptime(Conf.ExpFinishTime(),
"%y%m%d-%H%M%S.%f")
dur = (et - st).total_seconds()
# { t0: {HeatBoxes} }
time_heatboxes = {}
vertical_lines = []
for i in range(0, num_times):
t0 = st + datetime.timedelta(seconds=(float(dur) * i / num_times +
time_offset_in_sec))
t1 = st + datetime.timedelta(seconds=(float(dur) *
(i + 1) / num_times +
time_offset_in_sec))
vertical_lines.append(t1)
# Heat boxes are sorted by their heat and plotted with the heights
# proportional to the size.
boxes = []
for sst_gen, sl in sorted(sst_lives.iteritems()):
h = sl.HeatAtTime(t0)
if h is None:
continue
boxes.append(_Box(sl, t0, t1, h))
boxes.sort(key=lambda b: b.heat, reverse=True)
time_heatboxes[t0] = boxes
Cons.ClearLine()
Cons.Pnnl("%4d/%4d" % (i + 1, num_times))
print ""
del vertical_lines[-1]
# Set y-coordinate of each box
for t, boxes in sorted(time_heatboxes.iteritems()):
total_size = 0
for b in boxes:
total_size += b.sl.Size()
s = 0
for b in boxes:
b.y0 = float(s) / total_size
s += b.sl.Size()
b.y1 = float(s) / total_size
# Make leftmost time to 0.
t_first = None
t_base = datetime.datetime(2000, 1, 1)
for t, boxes in sorted(time_heatboxes.iteritems()):
if t_first is None:
t_first = t
for b in boxes:
b.t0 = t_base + (b.t0 - t_first)
b.t1 = t_base + (b.t1 - t_first)
for i in range(len(vertical_lines)):
vertical_lines[i] = t_base + (vertical_lines[i] - t_first)
fmt = "%4d %1d %17s %17s %6.4f %6.4f" \
" %8.3f %8d %6s"
with open(fn_hm, "w") as fo:
fo.write("# heat_max=%f\n" % MemtSstLife.SstLife.max_heat)
# y0 is smaller than y1 (y0 is placed higher in the plot than y1).
fo.write("%s\n" % Util.BuildHeader(fmt, \
"sst_gen level t0 t1 y0 y1" \
" heat heat_color heat_color_hex"))
for t, boxes in sorted(time_heatboxes.iteritems()):
for b in boxes:
fo.write((fmt + "\n") % ( \
b.sl.sst_gen, b.sl.level
, b.t0.strftime("%y%m%d-%H%M%S.%f")[:-3], b.t1.strftime("%y%m%d-%H%M%S.%f")[:-3]
, b.y0, b.y1
, b.heat, b.heat_color, ("%0.6X" % b.heat_color)
))
fo.write("\n")
Cons.P("Created %s %d" % (fn_hm, os.path.getsize(fn_hm)))
with open(fn_vl, "w") as fo:
for vl in vertical_lines:
fo.write("%s\n" % vl.strftime("%y%m%d-%H%M%S.%f")[:-3])
Cons.P("Created %s %d" % (fn_vl, os.path.getsize(fn_vl)))
return (fn_hm, fn_vl, MemtSstLife.SstLife.max_heat)
def Boxes(n, m):
if Conf.ExpStartTime() is None:
MutantLogReader.Get()
dn = "%s/sst-by-level-by-ks-range-with-heat" % Conf.dn_result
fn_boxes = "%s/%s-boxes-%d" % (dn, Conf.ExpStartTime(), n)
fn_level_info = "%s/%s-level-info-%d" % (dn, Conf.ExpStartTime(), n)
if os.path.isfile(fn_boxes) and os.path.isfile(fn_level_info):
_UpdateMaxPlotHeight(fn_boxes)
return (fn_boxes, fn_level_info)
sst_lives = MemtSstLife.GetSstLives()
with Cons.MT(
"Generating SSTables by level by ks range with heat at specific time n=%d m=%s ..."
% (n, m)):
Util.MkDirs(dn)
t = datetime.datetime.strptime(sst_lives[n].Opened(), "%y%m%d-%H%M%S.%f") \
+ datetime.timedelta(seconds=m)
# Boxes representing SSTables
boxes = []
for sst_gen, sl in sorted(sst_lives.iteritems()):
h = sl.HeatAtTime(t)
if h is None:
continue
boxes.append(_Box(sl, h))
# Sort boxes by level acendening, sst_gen decending. At the same level, the
# younger SSTables (with bigger sst_gen number) tend to be hotter.
boxes.sort(key=lambda b: b.sst_life.sst_gen, reverse=True)
boxes.sort(key=lambda b: b.sst_life.level)
# Box height: fixed or exponentially growing.
fixed_box_height = True
if fixed_box_height:
box_height = 1
else:
box_height = 0.5
level_labels_y = [box_height / 2.0]
level_separators_y = []
# Set y0 of each box
cur_y0_max = 0
prev_level = None
for b in boxes:
if fixed_box_height:
box_height = 1
else:
box_height = 0.5 * math.pow(2, b.sst_life.level)
if b.sst_life.level == 0:
b.y0 = 0
cur_y0_max = b.y0
else:
if b.sst_life.level == prev_level:
b.y0 = cur_y0_max
else:
if fixed_box_height:
prev_box_height = 1
else:
prev_box_height = 0.5 * math.pow(2, prev_level)
b.y0 = cur_y0_max + prev_box_height + 2 * y_spacing
sep = cur_y0_max + prev_box_height + y_spacing
level_separators_y.append(sep)
level_labels_y.append(sep + y_spacing + box_height / 2.0)
cur_y0_max = b.y0
b.y1 = b.y0 + box_height
prev_level = b.sst_life.level
global max_plot_height
max_plot_height = cur_y0_max + box_height + y_spacing if max_plot_height is None \
else max(max_plot_height, cur_y0_max + box_height + y_spacing)
fn_boxes = "%s/%s-boxes-%d" % (dn, Conf.ExpStartTime(), n)
with open(fn_boxes, "w") as fo:
fmt = "%3d %1d %9d %6.2f %8d %6s %20d %20d %5.1f %5.1f"
fo.write("%s\n" % Util.BuildHeader(fmt, "sst_gen level size heat(reads_per_sec_per_mb)" \
" heat_color heat_color_hex" \
" ks_first ks_last box_y0 box_y1"))
for b in boxes:
fo.write(
(fmt + "\n") % (b.sst_life.sst_gen, b.sst_life.level,
b.sst_life.Size(), b.heat, b.color,
"%0.6X" % b.color, b.sst_life.key_range[0],
b.sst_life.key_range[1], b.y0, b.y1))
Cons.P("Created %s %d" % (fn_boxes, os.path.getsize(fn_boxes)))
with open(fn_level_info, "w") as fo:
fmt = "%5s %5s"
fo.write("%s\n" %
Util.BuildHeader(fmt, "level_label_y level_separator_y"))
for i in range(max(len(level_labels_y), len(level_separators_y))):
l = "-"
if i < len(level_labels_y):
l = "%5.1f" % level_labels_y[i]
s = "-"
if i < len(level_separators_y):
s = "%5.1f" % level_separators_y[i]
fo.write((fmt + "\n") % (l, s))
Cons.P("Created %s %d" %
(fn_level_info, os.path.getsize(fn_level_info)))
return (fn_boxes, fn_level_info)
