def GetSstAccFreqAtSpecificTime(at_simulated_time):
fn = "%s/sst-accfreq-%s-at-%.04f" % (Conf.Get("dn_result"),
Conf.Get("simulation_time_begin"),
at_simulated_time)
if os.path.isfile(fn):
return fn
# t0: a specific time when we take the snapshot
t0 = SimTime.SimulatedTimeAt(at_simulated_time)
Cons.P("t0 (time of snapshot): %s" % t0)
sst_lives = GetSstLives()
with open(fn, "w") as fo:
fmt = "%4d %13s %13s %7.3f %1d %5.2f"
fo.write("# t0 (time of snapshot): %s\n" % t0)
fo.write("%s\n" % Util.BuildHeader(
fmt,
"sst_id ts_before ts_after reads_per_64MB_per_sec level age_in_day"
))
for sst_id, sl in sst_lives.iteritems():
if t0 < SimTime.ToSimulatedTime(sl.ts_created):
continue
if (sl.ts_deleted is not None) and (SimTime.ToSimulatedTime(
sl.ts_deleted) < t0):
continue
ts_prev = None
for ts, v in sorted(sl.ts_acccnt.iteritems()):
ts_simulated = SimTime.ToSimulatedTime(ts)
if ts_simulated < t0:
ts_prev = ts_simulated
continue
#Cons.P("ts_simulated: %s" % ts_simulated)
if ts_prev is not None:
#cnt = v[0]
cnt_per_64MB_per_sec = v[1]
#temp = v[2]
fo.write(
(fmt + "\n") %
(sst_id, ts_prev.strftime("%y%d%m-%H%M%S"),
ts_simulated.strftime("%y%d%m-%H%M%S"),
cnt_per_64MB_per_sec, sl.Level(),
((t0 - SimTime.ToSimulatedTime(
sl.TsCreated())).total_seconds() / 3600.0 / 24)))
break
Cons.P("Created %s %d" % (fn, os.path.getsize(fn)))
return fn
def PrepareData(self):
simulation_time_begin_year = SimTime.SimulationTimeBegin().strftime(
"%y")
self.row = []
for h, i in sorted(_header_idx.iteritems()):
if h.startswith("dsk/xvd") \
or h.startswith("io/xvd") \
or h.startswith("total_cpu_usage:"):
self.row.append(float(self.raw_row[i]))
elif h.startswith("system:time"):
# It doesn't have year. Use simulation_time_begin_year.
# 27-12 15:34:18
# 01234567890123
simulation_time = datetime.datetime.strptime(
(simulation_time_begin_year + self.raw_row[i]),
"%y%d-%m %H:%M:%S")
self.row.append(
SimTime.ToSimulatedTime(simulation_time).strftime(
"%m%d-%H%M%S"))
elif h.startswith("memory_usage:") \
or h.startswith("net/total:") \
or h.startswith("system:"):
self.row.append(float(self.raw_row[i]) / 1024.0)
else:
self.row.append(self.raw_row[i])
def __init__(self, ts, event, sst_id):
self.ts = ts
if ts is not None:
self.ts = SimTime.ToSimulatedTime(self.ts)
# "C"reated or "D"eleted
self.event = event
self.sst_id = sst_id
if self.ts is not None:
if SimTime.SimulatedTimeEnd() < self.ts:
# This happens. Tolerate ts no bigger than SimTime.SimulatedTimeEnd()
# by 5 minutes, which is small compared with the 14 day time
# interval.
if SimTime.SimulatedTimeEnd() < self.ts:
Cons.P("SimTime.SimulatedTimeEnd() %s < self.ts %s. event=%s. It's okay. Adjust to the former" \
% (SimTime.SimulatedTimeEnd(), self.ts, event))
self.ts = SimTime.SimulatedTimeEnd()
if self.event == "D" and self.ts is None:
self.ts = SimTime.SimulatedTimeEnd()
if self.ts is None:
raise RuntimeError("Unexpected")
def GetData():
fn_hm = "%s/sst-heatmap-by-time-%s" % (Conf.dn_result, Conf.Get("simulation_time_begin"))
fn_vl = "%s/sst-heatmap-by-time-vertical-lines-%s" % (Conf.dn_result, Conf.Get("simulation_time_begin"))
if os.path.isfile(fn_hm) and os.path.isfile(fn_vl):
return (fn_hm, fn_vl, _MaxHeatFromHeatmapByTimeData(fn_hm))
# {sst_id, SstLife()}
sst_lives = RocksdbLogReader.GetSstLives()
with Cons.MT("Generating Sst heatmap by time ..."):
# Gather temperature info at n different times
num_times = Conf.heatmap_by_time_num_times
# Set start and end times
# Start time is when the first Sstable is created, not the simulation_time_begin, when no SSTable exists yet.
# Exp start time: 160927-143257.395
# First Sstable open time: 160927-143411.273
min_sst_opened = None
for sst_gen, sl in sorted(sst_lives.iteritems()):
min_sst_opened = sl.TsCreated() if min_sst_opened is None else min(min_sst_opened, sl.TsCreated())
st = min_sst_opened
et = SimTime.SimulationTimeEnd()
dur = (et - st).total_seconds()
# { t0: {HeatBoxes} }
time_heatboxes = {}
max_temperature = None
for i in range(0, num_times):
t0 = st + datetime.timedelta(seconds=(float(dur) * i / num_times + time_offset_in_sec))
t1 = st + datetime.timedelta(seconds=(float(dur) * (i+1) / num_times + time_offset_in_sec))
# Sort boxes by their temperature. Heights are proportional to their sizes.
boxes = []
for sst_gen, sl in sorted(sst_lives.iteritems()):
temp = sl.TempAtTime(t0)
if max_temperature is None:
max_temperature = temp
else:
max_temperature = max(max_temperature, temp)
if temp is None:
continue
boxes.append(_Box(sl, t0, t1, temp))
boxes.sort(key=lambda b: b.temp, reverse=True)
time_heatboxes[t0] = boxes
Cons.ClearLine()
Cons.Pnnl("%4d/%4d" % (i + 1, num_times))
print ""
for t, boxes in sorted(time_heatboxes.iteritems()):
for b in boxes:
b.SetTempColor(max_temperature)
# Set y-coordinate of each box
for t, boxes in sorted(time_heatboxes.iteritems()):
total_size = 0
for b in boxes:
total_size += b.sl.Size()
s = 0
for b in boxes:
b.y0 = float(s) / total_size
s += b.sl.Size()
b.y1 = float(s) / total_size
# Convert to simulated time
# { t0: {HeatBoxes} }
time_heatboxes1 = {}
for t, boxes in sorted(time_heatboxes.iteritems()):
t1 = SimTime.ToSimulatedTime(t)
for b in boxes:
b.t0 = SimTime.ToSimulatedTime(b.t0)
b.t1 = SimTime.ToSimulatedTime(b.t1)
time_heatboxes1[t1] = boxes
# Make leftmost time to 0.
t_first = None
#t_base = datetime.datetime(2000, 1, 1)
vertical_lines = []
for t, boxes in sorted(time_heatboxes1.iteritems()):
if t_first is None:
t_first = t
vl = None
for b in boxes:
#b.t0 = t_base + (b.t0 - t_first)
#b.t1 = t_base + (b.t1 - t_first)
b.t0 = (b.t0 - t_first).total_seconds()
b.t1 = (b.t1 - t_first).total_seconds()
vl = b.t1
vertical_lines.append(vl)
del vertical_lines[-1]
fmt = "%4d %1d" \
" %8d %8d" \
" %6.4f %6.4f" \
" %8.3f %11.6f %8d %6s"
with open(fn_hm, "w") as fo:
fo.write("# max_temperature=%f\n" % max_temperature)
# y0 is smaller than y1 (y0 is placed higher in the plot than y1).
fo.write("%s\n" % Util.BuildHeader(fmt, \
"sst_gen level t0 t1 y0 y1" \
" temp temp_relative temp_color heat_color_hex"))
for t, boxes in sorted(time_heatboxes1.iteritems()):
for b in boxes:
fo.write((fmt + "\n") % ( \
b.sl.Id(), b.sl.Level()
#, b.t0.strftime("%y%m%d-%H%M%S.%f")[:-3], b.t1.strftime("%y%m%d-%H%M%S.%f")[:-3]
, b.t0, b.t1
, b.y0, b.y1
, b.temp, (float(b.temp) / max_temperature)
, b.temp_color, ("%0.6X" % b.temp_color)
))
fo.write("\n")
Cons.P("Created %s %d" % (fn_hm, os.path.getsize(fn_hm)))
with open(fn_vl, "w") as fo:
for vl in vertical_lines:
#fo.write("%s\n" % vl.strftime("%y%m%d-%H%M%S.%f")[:-3])
fo.write("%8d\n" % vl)
Cons.P("Created %s %d" % (fn_vl, os.path.getsize(fn_vl)))
return (fn_hm, fn_vl, max_temperature)