def Created(line):
mo = re.match(r"(?P<ts>(\d|\/|-|:|\.)+) .+EVENT_LOG_v1 (?P<json>.+)",
line)
if mo is None:
raise RuntimeError("Unexpected: [%s]" % line)
# Fix the illegal json format
j = mo.group("json").replace("kFlush", "\"kFlush\"").replace(
"kCompaction", "\"kCompaction\"").replace("kRecovery",
"\"kRecovery\"")
try:
j1 = json.loads(j)
except ValueError as e:
Cons.P("%s [%s]" % (e, line))
sys.exit(1)
sst_size = int(j1["file_size"])
sst_id = int(j1["file_number"])
ts1 = SstEvents._GetRelTs(mo.group("ts"))
SstEvents.sstid_size[sst_id] = sst_size
SstEvents.cur_sstsize += sst_size
SstEvents.cur_numssts += 1
SstEvents.ts_sstsize[ts1] = SstEvents.cur_sstsize
SstEvents.ts_numssts[ts1] = SstEvents.cur_numssts
SstEvents.sstid_createts[sst_id] = ts1
SstEvents.createts_sstid[ts1] = sst_id
hc = HowCreated.Add(sst_id, j1)
if SstEvents.migrate_sstables:
if hc.Reason()[0] == "C":
CompInfo.AddOutSstInfo(j1)
def Write(fn):
fmt = "%12s %12s %7.3f %4d %4d %12d %4s %4s %1s %1s %1s"
with open(fn, "w") as fo:
fo.write(Util.BuildHeader(fmt, "rel_ts_HHMMSS_begin" \
" rel_ts_HHMMSS_end" \
" ts_dur" \
" num_sstables_begin" \
" num_sstables_end" \
" sstable_size_sum_end" \
" end_sst_id" \
" end_sst_creation_jobid" \
" end_sst_creation_reason" \
" end_sst_temp_triggered_single_migr" \
" end_sst_migration_direction") + "\n")
ts_prev = datetime.timedelta(0)
ts_str_prev = "00:00:00.000"
num_ssts_prev = 0
total_sst_size_prev = 0
for ts, num_ssts in sorted(SstEvents.ts_numssts.iteritems()):
ts_str = _ToStr(ts)
sst_id = "-"
job_id = "-"
creation_reason = "-"
# Temperature-triggered single-sstable migration
temp_triggered_migr = "-"
migr_dirc = "-"
if ts in SstEvents.createts_sstid:
sst_id = SstEvents.createts_sstid[ts]
hc = HowCreated.Get(sst_id)
job_id = hc.JobId()
creation_reason = hc.Reason()
temp_triggered_migr = "T" if CompInfo.TempTriggeredSingleSstMigr(
job_id) else "-"
if SstEvents.migrate_sstables:
if creation_reason == "C":
migr_dirc = CompInfo.MigrDirc(job_id, sst_id)
fo.write(
(fmt + "\n") %
(ts_str_prev, ts_str,
(ts.total_seconds() - ts_prev.total_seconds()),
num_ssts_prev, num_ssts, total_sst_size_prev, sst_id,
job_id, creation_reason, temp_triggered_migr, migr_dirc))
ts_str_prev = ts_str
ts_prev = ts
num_ssts_prev = num_ssts
total_sst_size_prev = SstEvents.ts_sstsize[ts]
Cons.P("Created %s %d" % (fn, os.path.getsize(fn)))
def _CalcMigrDirections(self):
#Cons.P(self)
# In storage 0 and 1: fast and slow
total_in_sst_sizes = [0, 0]
for in_sst_id in self.in_sst_ids:
hc = HowCreated.Get(in_sst_id)
total_in_sst_sizes[hc.PathId()] += hc.Size()
in_sst0_size_ratio = float(
total_in_sst_sizes[0]) / sum(total_in_sst_sizes)
#Cons.P("total_in_sst_sizes=%s in_sst0_size_ratio=%d" % (total_in_sst_sizes, in_sst0_size_ratio))
# self.out_ssts={13: how_created=None migr_dirc=None path_id=1 sst_id=13}
l = len(self.out_ssts)
i = 0
for out_sst_id, v in sorted(self.out_ssts.iteritems()):
i += 1
# path_id_before: what the output SSTable's path_id would have been. It is for assigning the migration direction.
# It is calculated based on the input SSTables' path_ids.
if (i / l) <= in_sst0_size_ratio:
path_id_before = 0
else:
path_id_before = 1
v.SetMigrDirc(path_id_before)
def __init__(self, dn_log_job, exp_dt):
self.fn_out = "%s/rocksdb-by-time-%s" % (Conf.GetOutDir(), exp_dt)
if os.path.isfile(self.fn_out):
return
# These classes rely on the global data structures, thus not thread-safe. Fine for now.
SstEvents.Init(exp_dt)
HowCreated.Init()
CompInfo.Init()
with Cons.MT("Generating rocksdb time-vs-metrics file for plot ..."):
fn_log_rocksdb = "%s/rocksdb/%s" % (dn_log_job, exp_dt)
if not os.path.exists(fn_log_rocksdb):
fn_zipped = "%s.bz2" % fn_log_rocksdb
if not os.path.exists(fn_zipped):
raise RuntimeError("Unexpected: %s" % fn_log_rocksdb)
Util.RunSubp(
"cd %s && bzip2 -dk %s > /dev/null" %
(os.path.dirname(fn_zipped), os.path.basename(fn_zipped)))
if not os.path.exists(fn_log_rocksdb):
raise RuntimeError("Unexpected")
Cons.P(fn_log_rocksdb)
with open(fn_log_rocksdb) as fo:
for line in fo:
#Cons.P(line)
# 2018/01/05-08:10:30.085011 7f40083d8700 migrate_sstables: 0
if " migrate_sstables: " in line:
mo = re.match(
r"(?P<ts>(\d|\/|-|:|\.)+) .* migrate_sstables: (?P<v>\d).*",
line)
if mo is None:
raise RuntimeError("Unexpected: [%s]" % line)
self.migrate_sstables = (mo.group("v") == "1")
SstEvents.migrate_sstables = self.migrate_sstables
HowCreated.migrate_sstables = self.migrate_sstables
# 2017/10/13-20:41:54.872056 7f604a7e4700 EVENT_LOG_v1 {"time_micros": 1507927314871238, "cf_name": "usertable", "job": 3, "event":
# "table_file_creation", "file_number": 706, "file_size": 258459599, "path_id": 0, "table_properties": {"data_size": 256772973, "index_size": 1685779,
# "filter_size": 0, "raw_key_size": 6767934, "raw_average_key_size": 30, "raw_value_size": 249858360, "raw_average_value_size": 1140,
# "num_data_blocks": 54794, "num_entries": 219174, "filter_policy_name": "", "reason": kFlush, "kDeletedKeys": "0", "kMergeOperands": "0"}}
elif "\"event\": \"table_file_creation\"" in line:
SstEvents.Created(line)
# 2018/01/01-05:33:49.183505 7f97d0ff1700 EVENT_LOG_v1 {"time_micros": 1514784829183496, "job": 6, "event": "table_file_deletion", "file_number": 21}
elif "\"event\": \"table_file_deletion\"" in line:
SstEvents.Deleted(line)
# 2018/01/23-00:13:13.593310 7fa321da9700 EVENT_LOG_v1 {"time_micros": 1516666393593302, "mutant_calc_out_sst_path_id": {"in_sst": "(sst_id=16
# temp=57.189 level=0 path_id=0 size=258425431 age=30)", "in_sst_temp": "57.188590", "sst_ott": 3176.66, "out_sst_path_id": 1,
# "temp_triggered_single_sst_compaction": 1}}
elif "\"mutant_calc_out_sst_path_id\"" in line:
CompInfo.SetTempTriggeredSingleSstMigr(line)
# Figure out how an SSTable is created
# (a) start building CompInfo
# 2018/01/05-08:40:21.078219 7fd13ffff700 EVENT_LOG_v1 {"time_micros": 1515141621078214, "mutant_sst_compaction_migration": {"in_sst": "(sst_id=16
# temp=57.345 level=0 path_id=0 size=258423184 age=30) (sst_id=13 temp=3738.166 level=0 path_id=1 size=118885 age=440)", "out_sst_path_id": 1,
# "temp_triggered_single_sst_compaction": 1}}
#
# Some SSTables are created without this log, meaning not going through _CalcOutputPathId(). Use the other one.
#elif "mutant_sst_compaction_migration" in line:
# if not self.migrate_sstables:
# continue
# CompInfo.Add(line)
# Build CompInfo
# We manually parse this just because there are multiple keys with "files_L0" and json would probably not be able to parse it
# 2018/01/05-08:40:21.078303 7fd13ffff700 EVENT_LOG_v1 {"time_micros": 1515141621078294, "job": 5, "event": "compaction_started", "files_L0": [16,
# 13], "files_L0": [16, 13], "score": 0, "input_data_size": 517084138}
elif "compaction_started" in line:
CompInfo.SetCompStarted(line)
# (c) Assign out_sst_info to CompInfo using job_id. It is done when parsing table_file_creation above.
CompInfo.CalcMigrDirections()
SstEvents.Write(self.fn_out)