def gen_read_op_df(rops):
#import pdb; pdb.set_trace();
ops_diclist = utils.gen_v_list(rops)
enqueue_time = utils.gen_element_in_diclist(ops_diclist, "enqueue_time")
opts = [utils.timestamp(x) for x in enqueue_time]
opid = utils.gen_element_in_diclist(ops_diclist, "opid")
before_enqueue_lat = utils.gen_element_in_diclist(ops_diclist,
"before_enqueue_lat")
reply_time = utils.gen_element_in_diclist(ops_diclist, "reply_time")
eop_lat = [
utils.time_substract(reply_time[idx], enqueue_time[idx])
for idx in range(len(enqueue_time))
]
op_lat = [
float(eop_lat[idx]) + float(before_enqueue_lat[idx])
for idx in range(len(eop_lat))
]
print op_lat
queue_lat = utils.gen_element_in_diclist(ops_diclist, "queue_latency")
dequeue_lat = utils.gen_element_in_diclist(ops_diclist, "dequeue_latency")
dequeue_count = utils.gen_element_in_diclist(ops_diclist, "dequeue_count")
all_data = {
"opid": opid,
"opts": opts,
"op_lat": op_lat,
"enqueue_time": enqueue_time,
"before_enqueue_lat": before_enqueue_lat,
"queue_lat": queue_lat,
"dequeue_lat": dequeue_lat,
"dequeue_count": dequeue_count
}
return pd.DataFrame(all_data)
def process_op_commit(words, wops):
#print "op_commit", words
op_commit = {}
op_commit["time"] = words[0] + "-" + words[1]
for key in osdtpl.op_commit_key2idx.keys():
idx = osdtpl.op_commit_key2idx[key]
op_commit[key] = words[idx]
for op in wops.values():
if op.has_key("rep_tid"):
if op["rep_tid"] == op_commit["rep_tid"]:
op["bluestore_lat"] = utils.time_substract(
op_commit["time"], op["eval_repop_time"])
break
def gen_write_op_df(ops):
ops_diclist = utils.gen_v_list(ops)
enqueue_time = utils.gen_element_in_diclist(ops_diclist, "enqueue_time")
opts = [utils.timestamp(x) for x in enqueue_time]
reply_time = utils.gen_element_in_diclist(ops_diclist, "reply_time")
opid = utils.gen_element_in_diclist(ops_diclist, "opid")
opref = utils.gen_element_in_diclist(ops_diclist, "opref")
before_enqueue_lat = utils.gen_element_in_diclist(ops_diclist,
"before_enqueue_lat")
## currently just use reply_time - enqueue_time
op_lat = [
utils.time_substract(reply_time[idx], enqueue_time[idx])
for idx in range(len(enqueue_time))
]
## construct subop
for op in ops_diclist:
if not op.has_key("subop"):
pass
#print "no subop", op
else:
pass
#print "has subop", op
subops = utils.gen_element_in_diclist(ops_diclist, "subop")
subop_str = []
for subs in subops:
cstr = gen_subop_str(subs)
subop_str.append(cstr)
queue_lat = utils.gen_element_in_diclist(ops_diclist, "queue_latency")
dequeue_lat = utils.gen_element_in_diclist(ops_diclist, "dequeue_latency")
dequeue_count = utils.gen_element_in_diclist(ops_diclist, "dequeue_count")
bluestore_lat = utils.gen_element_in_diclist(ops_diclist, "bluestore_lat")
color = ["red" for x in bluestore_lat]
all_data = {
"opts": opts,
"enqueue_time": enqueue_time,
"op_lat": op_lat,
"before_enqueue_lat": before_enqueue_lat,
"opid": opid,
"subop_str": subop_str,
"queue_lat": queue_lat,
"dequeue_lat": dequeue_lat,
"dequeue_count": dequeue_count,
"bluestore_lat": bluestore_lat,
"color": color
}
df = pd.DataFrame(all_data)
return df
def process_subop_reply(words, wops):
subop_reply = {}
subop_reply["time"] = words[0] + "-" + words[1]
for k in osdtpl.subop_reply_key2idx.keys():
v = osdtpl.subop_reply_key2idx[k]
subop_reply[k] = words[v]
opid = repop_reply2op(subop_reply["subop_reply_id"])
if wops.has_key(opid):
# Since this is a subop reply, subop sent must already been processed
assert wops[opid].has_key("subop")
for subop in wops[opid]["subop"]:
if subop["dst"] == subop_reply["src"]:
subop["latency"] = utils.time_substract(
subop_reply["time"], subop["time"])
subop["subosd"] = subop_reply["subosd"]
def process_dequeue_op_start(words, wops, rops):
dequeue = {}
dequeue["time"] = words[0] + "-" + words[1]
for key in osdtpl.dequeue_op_start_key2idx.keys():
idx = osdtpl.dequeue_op_start_key2idx[key]
dequeue[key] = words[idx]
for ops in [wops, rops]:
if ops.has_key(dequeue["opid"]):
op = ops[dequeue["opid"]]
op["dequeue_time"] = dequeue["time"]
op["queue_latency"] = str(
utils.time_substract(dequeue["time"], op["enqueue_time"]))
if op.has_key("dequeue_count"):
op["dequeue_count"] += 1
else:
op["dequeue_count"] = 1
op["dequeue_thread"] = dequeue["thread"]
def process_dequeue_op_end(words, wops, rops):
dequeue_end = {}
dequeue_end["time"] = words[0] + "-" + words[1]
for key in osdtpl.dequeue_op_end_key2idx.keys():
idx = osdtpl.dequeue_op_end_key2idx[key]
dequeue_end[key] = words[idx]
for ops in [wops, rops]:
for op in ops.values():
if op.has_key("dequeue_thread"
): ## means it is still in the osd dequeue thread
if op["dequeue_thread"] == dequeue_end["thread"]:
op["dequeue_latency"] = str(
utils.time_substract(dequeue_end["time"],
op["dequeue_time"]))
if op["op_type"] == "read":
## for read, it is also the end of a read op
op["reply_time"] = dequeue_end["time"]
del op["dequeue_thread"]
def process_bs_write_end(words, begin, stats):
bs_we = {}
time = words[0] + "-" + words[1]
bs_we["time"] = time
stat = {}
for key in osdtpl.bs_write_end_key2idx.keys():
idx = osdtpl.bs_write_end_key2idx[key]
bs_we[key] = words[idx]
if begin.has_key(bs_we["thread"]):
bs_wb = begin[bs_we["thread"]]
stat["start_time"] = bs_wb["time"]
# Use microseconds for latency
stat["latency"] = int(
utils.time_substract(bs_we["time"], bs_wb["time"]) * 1000000)
stat["aligned"] = utils.is_aligned(bs_we["offlen"], 4096)
stat["len"] = utils.get_len(bs_we["offlen"])
stat["offset"] = utils.get_offset(bs_we["offlen"])
stat["type"] = "bs_write"
stats.append(stat)
del begin[bs_we["thread"]]
