def SstHeatAtLastTime():
# Set Conf.ExpStartTime(), if not already set.
if Conf.ExpStartTime() is None:
MutantLogReader.Get()
fn_hlt = "%s/sst-heat-last-time-%s" % (Conf.dn_result, Conf.ExpStartTime())
if os.path.isfile(fn_hlt):
return fn_hlt
sst_lives = MemtSstLife.GetSstLives()
with Cons.MT("Generating Sst heats at the last time ..."):
# Gather heat info at n different times
num_times = Conf.heatmap_by_time_num_times
if Conf.ExpFinishTime() is None:
MemtSstLife.SetExpEndTimeFromSstLives()
min_sst_opened = None
for sst_gen, sl in sorted(sst_lives.iteritems()):
min_sst_opened = sl.Opened() if min_sst_opened is None else min(
min_sst_opened, sl.Opened())
# Start time is when the first Sstable is opened, not the experiment start
# time, when no SSTable exists yet.
# Exp start time: 160927-143257.395
# First Sstable open time: 160927-143411.273
st = datetime.datetime.strptime(min_sst_opened, "%y%m%d-%H%M%S.%f")
et = datetime.datetime.strptime(Conf.ExpFinishTime(),
"%y%m%d-%H%M%S.%f")
dur = (et - st).total_seconds()
sstgen_heat = []
t = st + datetime.timedelta(seconds=(float(dur) *
(num_times - 1) / num_times +
time_offset_in_sec))
for sst_gen, sl in sorted(sst_lives.iteritems()):
h = sl.HeatAtTime(t)
if h is None:
continue
sstgen_heat.append((sst_gen, h))
sstgen_heat.sort(key=lambda sh: sh[1], reverse=True)
# Note: Don't bother with the width proportional to the tablet size for now
fmt = "%4d %1d %8.3f"
with open(fn_hlt, "w") as fo:
# y0 is smaller than y1 (y0 is placed higher in the plot than y1).
fo.write("%s\n" % Util.BuildHeader(fmt, "sst_gen level heat"))
for sh in sstgen_heat:
sst_gen = sh[0]
heat = sh[1]
fo.write(
(fmt + "\n") % (sst_gen, sst_lives[sst_gen].level, heat))
Cons.P("Created %s %d" % (fn_hlt, os.path.getsize(fn_hlt)))
return fn_hlt
def __str__(self):
try:
if self.ts_range is None:
tsr0 = None
tsr1 = None
else:
tsr0 = self.ts_range[0]
tsr1 = self.ts_range[1]
if self.key_range is None:
kr0 = None
kr1 = None
else:
kr0 = self.key_range[0]
kr1 = self.key_range[1]
return SstLife._fmt % (
self.sst_gen
, "-" if self.time_open_early is None else self.time_open_early
, self.time_open_normal
, self.Opened()
, (Conf.ExpFinishTime() if self.time_deleted is None else self.time_deleted)
, self.size
, "-" if self.level is None else self.level
, tsr0, tsr1
, kr0, kr1
, "-" if self.y_cord_low is None else self.y_cord_low
, str(self.sst_gen) \
+ str(CassCompactionLogReader.HowCreated(self.sst_gen)).replace("flushed", "F").replace("compacted", "C")
)
except TypeError as e:
Cons.P("TypeError: %d\n%s" %
(self.sst_gen, traceback.format_exc()))
sys.exit(1)
def __str__(self):
# When the size is not available, give some sensible value. Every
# Memtable has around 121 MiB.
#if self.size is None:
# size = 121000
# Cons.P("Hmm... Memtable doesn't have a size. Setting to %d" % size)
#else:
# size = self.size
if self.size is None:
raise RuntimeError("Unexpected")
return MemtLife.fmt % (self.addr, self.time_created,
(Conf.ExpFinishTime() if self.time_discarded is
None else self.time_discarded), self.size)
def _ReadAndCacheCassLogUntilDtCrossesStNotTested():
with Cons.MT("Reading Cassandra log ..."):
# Note: s0 only for now.
dn = "%s/work/mutant/log/%s/s0/cassandra" % (os.path.expanduser("~"), Util0.JobId())
# Start from system.log and keep reading system.log.n.zip where n >= 1,
# until you find two datetimes that cross the given exp start time
st = Conf.ExpStartTime()
ft = Conf.ExpFinishTime()
dt_prev = None
dt_crossed_st = False
# WARN [main] 2016-09-20 02:20:39,250 MemSsTableAccessMon.java:115 - Mutant: ...
pattern = re.compile(r".+" \
" (?P<datetime>\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d,\d\d\d)" \
" .+")
fn = "%s/system.log" % dn
lines = []
Cons.P("fn=%s" % fn)
with open(fn) as fo:
for line in fo.readlines():
line = line.strip()
mo = pattern.match(line)
if mo is None:
raise RuntimeError("Unexpected line=[%s]" % line)
dt = Util0.ShortDateTime(mo.group("datetime"))
if (st <= dt) and (dt <= ft):
lines.append(line)
if ft < dt:
break
if not dt_crossed_st:
if (dt_prev is not None) and (dt_prev < st) and (st < dt):
dt_crossed_st = True
dt_prev = dt
# Keep reading zipped files like system.log.1.zip, until ResetMon is found
i = 1
while dt_crossed_st == False:
fn = "%s/system.log.%d.zip" % (dn, i)
Cons.P("dt haven't crossed st yet. Reading more logs from file %s ..." % fn)
lines1 = []
with zipfile.ZipFile(fn, "r") as z:
dt_prev = None
for fn1 in z.namelist():
#Cons.P(fn1)
for line in z.read(fn1).split("\n"):
line = line.strip()
#Cons.P(line)
mo = pattern.match(line)
if mo is None:
raise RuntimeError("Unexpected line=[%s]" % line)
dt = Util0.ShortDateTime(mo.group("datetime"))
if (st <= dt) and (dt <= ft):
lines.append(line)
if ft < dt:
break
if not dt_crossed_st:
if (dt_prev is not None) and (dt_prev < st) and (st < dt):
dt_crossed_st = True
dt_prev = dt
lines1.append(line)
if len(lines1) != 0:
lines1.extend(lines)
lines = list(lines1)
del lines1[:]
i += 1
fn = "%s/work/mutant/misc/logs/cassandra/system-%s" \
% (os.path.expanduser("~"), Conf.ExpStartTime())
with open(fn, "w") as fo:
for line in lines:
fo.write("%s\n" % line)
Cons.P("Created a Cassandra log file %s %d" % (fn, os.path.getsize(fn)))
return lines
def Heatmap():
# Set Conf.ExpStartTime(), if not already set.
if Conf.ExpStartTime() is None:
MutantLogReader.Get()
fn_hm = "%s/sst-heatmap-by-time-%s" % (Conf.dn_result, Conf.ExpStartTime())
fn_vl = "%s/sst-heatmap-by-time-vertical-lines-%s" % (Conf.dn_result,
Conf.ExpStartTime())
if os.path.isfile(fn_hm) and os.path.isfile(fn_vl):
return (fn_hm, fn_vl, _MaxHeatFromHeatmapByTimeData(fn_hm))
sst_lives = MemtSstLife.GetSstLives()
with Cons.MT("Generating Sst heatmap by time ..."):
# Gather heat info at n different times
num_times = Conf.heatmap_by_time_num_times
if Conf.ExpFinishTime() is None:
MemtSstLife.SetExpEndTimeFromSstLives()
min_sst_opened = None
for sst_gen, sl in sorted(sst_lives.iteritems()):
min_sst_opened = sl.Opened() if min_sst_opened is None else min(
min_sst_opened, sl.Opened())
# Start time is when the first Sstable is opened, not the experiment start
# time, when no SSTable exists yet.
# Exp start time: 160927-143257.395
# First Sstable open time: 160927-143411.273
#st = datetime.datetime.strptime(Conf.ExpStartTime(), "%y%m%d-%H%M%S.%f")
st = datetime.datetime.strptime(min_sst_opened, "%y%m%d-%H%M%S.%f")
et = datetime.datetime.strptime(Conf.ExpFinishTime(),
"%y%m%d-%H%M%S.%f")
dur = (et - st).total_seconds()
# { t0: {HeatBoxes} }
time_heatboxes = {}
vertical_lines = []
for i in range(0, num_times):
t0 = st + datetime.timedelta(seconds=(float(dur) * i / num_times +
time_offset_in_sec))
t1 = st + datetime.timedelta(seconds=(float(dur) *
(i + 1) / num_times +
time_offset_in_sec))
vertical_lines.append(t1)
# Heat boxes are sorted by their heat and plotted with the heights
# proportional to the size.
boxes = []
for sst_gen, sl in sorted(sst_lives.iteritems()):
h = sl.HeatAtTime(t0)
if h is None:
continue
boxes.append(_Box(sl, t0, t1, h))
boxes.sort(key=lambda b: b.heat, reverse=True)
time_heatboxes[t0] = boxes
Cons.ClearLine()
Cons.Pnnl("%4d/%4d" % (i + 1, num_times))
print ""
del vertical_lines[-1]
# Set y-coordinate of each box
for t, boxes in sorted(time_heatboxes.iteritems()):
total_size = 0
for b in boxes:
total_size += b.sl.Size()
s = 0
for b in boxes:
b.y0 = float(s) / total_size
s += b.sl.Size()
b.y1 = float(s) / total_size
# Make leftmost time to 0.
t_first = None
t_base = datetime.datetime(2000, 1, 1)
for t, boxes in sorted(time_heatboxes.iteritems()):
if t_first is None:
t_first = t
for b in boxes:
b.t0 = t_base + (b.t0 - t_first)
b.t1 = t_base + (b.t1 - t_first)
for i in range(len(vertical_lines)):
vertical_lines[i] = t_base + (vertical_lines[i] - t_first)
fmt = "%4d %1d %17s %17s %6.4f %6.4f" \
" %8.3f %8d %6s"
with open(fn_hm, "w") as fo:
fo.write("# heat_max=%f\n" % MemtSstLife.SstLife.max_heat)
# y0 is smaller than y1 (y0 is placed higher in the plot than y1).
fo.write("%s\n" % Util.BuildHeader(fmt, \
"sst_gen level t0 t1 y0 y1" \
" heat heat_color heat_color_hex"))
for t, boxes in sorted(time_heatboxes.iteritems()):
for b in boxes:
fo.write((fmt + "\n") % ( \
b.sl.sst_gen, b.sl.level
, b.t0.strftime("%y%m%d-%H%M%S.%f")[:-3], b.t1.strftime("%y%m%d-%H%M%S.%f")[:-3]
, b.y0, b.y1
, b.heat, b.heat_color, ("%0.6X" % b.heat_color)
))
fo.write("\n")
Cons.P("Created %s %d" % (fn_hm, os.path.getsize(fn_hm)))
with open(fn_vl, "w") as fo:
for vl in vertical_lines:
fo.write("%s\n" % vl.strftime("%y%m%d-%H%M%S.%f")[:-3])
Cons.P("Created %s %d" % (fn_vl, os.path.getsize(fn_vl)))
return (fn_hm, fn_vl, MemtSstLife.SstLife.max_heat)
