def __init__(self):
# 创建日志文件夹
if not os.path.exists(self.__log_path):
os.mkdir(self.__log_path)
# 初始化
self.__logger = logging.getLogger(__name__)
self.__logger.setLevel(logging.DEBUG)
# 设置log文件输出 级别.路径.格式化
if self.__handler == "1":
# 创建,路径
handler = logging.FileHandler(filename=self.__log_path +
timestamp() + "--" +
self.__handler_name + ".log",
encoding='utf-8')
# 级别
handler.setLevel(self.__handler_level)
# 格式化
formatter = logging.Formatter(self.__output_format)
handler.setFormatter(formatter)
# 添加
self.__logger.addHandler(handler)
# 设置控制台日志 级别格式化
if self.__console == "1":
# 创建
console = logging.StreamHandler()
# 级别
console.setLevel(self.__console_level)
# 添加
self.__logger.addHandler(console)
def gen_read_op_df(rops):
#import pdb; pdb.set_trace();
ops_diclist = utils.gen_v_list(rops)
enqueue_time = utils.gen_element_in_diclist(ops_diclist, "enqueue_time")
opts = [utils.timestamp(x) for x in enqueue_time]
opid = utils.gen_element_in_diclist(ops_diclist, "opid")
before_enqueue_lat = utils.gen_element_in_diclist(ops_diclist,
"before_enqueue_lat")
reply_time = utils.gen_element_in_diclist(ops_diclist, "reply_time")
eop_lat = [
utils.time_substract(reply_time[idx], enqueue_time[idx])
for idx in range(len(enqueue_time))
]
op_lat = [
float(eop_lat[idx]) + float(before_enqueue_lat[idx])
for idx in range(len(eop_lat))
]
print op_lat
queue_lat = utils.gen_element_in_diclist(ops_diclist, "queue_latency")
dequeue_lat = utils.gen_element_in_diclist(ops_diclist, "dequeue_latency")
dequeue_count = utils.gen_element_in_diclist(ops_diclist, "dequeue_count")
all_data = {
"opid": opid,
"opts": opts,
"op_lat": op_lat,
"enqueue_time": enqueue_time,
"before_enqueue_lat": before_enqueue_lat,
"queue_lat": queue_lat,
"dequeue_lat": dequeue_lat,
"dequeue_count": dequeue_count
}
return pd.DataFrame(all_data)
def print_mcs_bokeh():
TOOLS = "crosshair,pan,wheel_zoom,box_zoom,reset,hover,save"
time = utils.gen_element_in_diclist(mds_cache_stats, "time")
timestamp = [utils.timestamp(x) for x in time]
xtime = [(x - timestamp[0]).total_seconds() for x in timestamp]
#print xtime
xinodes = utils.gen_element_in_diclist(mds_cache_stats, "inodes")
inodes = [int(x) for x in xinodes]
#inodes[0] = 10000
xcaps = utils.gen_element_in_diclist(mds_cache_stats, "caps")
caps = [int(x) for x in xcaps]
print caps
TOOLTIPS = [("(x, y)", "x:$x y:$y"), ("time", "time:@time"),
("caps", "caps:@caps"), ("inodes", "inodes:@inodes")]
all_data = {"xtime": xtime, "time": time, "inodes": inodes, "caps": caps}
hover = HoverTool(tooltips=TOOLTIPS)
pd.options.display.max_rows = 300000000
df = pd.DataFrame(all_data)
source = ColumnDataSource(df)
#source = ColumnDataSource(data = dict(x = xtime, y = inodes))
output_file("inodes.html")
#p = figure(plot_width=800, plot_height=600, x_axis_label='timestamp', y_axis_label='inode number')
p = figure(plot_width=800,
plot_height=600,
tools=TOOLS,
active_scroll='wheel_zoom',
tooltips=TOOLTIPS,
x_axis_label='timestamp',
y_axis_label='inode number')
p.add_tools(hover)
p.circle('xtime',
'inodes',
source=source,
legend_label="inodes",
size=3,
color="red")
p.circle(x="xtime",
y="caps",
source=source,
legend_label="caps",
size=3,
color="black")
#p.line('x', 'y', source = source, legend_label=inodes, line_color="blue", line_dash="4 4")
show(p)
print "bokeh output"
def gen_write_op_df(ops):
ops_diclist = utils.gen_v_list(ops)
enqueue_time = utils.gen_element_in_diclist(ops_diclist, "enqueue_time")
opts = [utils.timestamp(x) for x in enqueue_time]
reply_time = utils.gen_element_in_diclist(ops_diclist, "reply_time")
opid = utils.gen_element_in_diclist(ops_diclist, "opid")
opref = utils.gen_element_in_diclist(ops_diclist, "opref")
before_enqueue_lat = utils.gen_element_in_diclist(ops_diclist,
"before_enqueue_lat")
## currently just use reply_time - enqueue_time
op_lat = [
utils.time_substract(reply_time[idx], enqueue_time[idx])
for idx in range(len(enqueue_time))
]
## construct subop
for op in ops_diclist:
if not op.has_key("subop"):
pass
#print "no subop", op
else:
pass
#print "has subop", op
subops = utils.gen_element_in_diclist(ops_diclist, "subop")
subop_str = []
for subs in subops:
cstr = gen_subop_str(subs)
subop_str.append(cstr)
queue_lat = utils.gen_element_in_diclist(ops_diclist, "queue_latency")
dequeue_lat = utils.gen_element_in_diclist(ops_diclist, "dequeue_latency")
dequeue_count = utils.gen_element_in_diclist(ops_diclist, "dequeue_count")
bluestore_lat = utils.gen_element_in_diclist(ops_diclist, "bluestore_lat")
color = ["red" for x in bluestore_lat]
all_data = {
"opts": opts,
"enqueue_time": enqueue_time,
"op_lat": op_lat,
"before_enqueue_lat": before_enqueue_lat,
"opid": opid,
"subop_str": subop_str,
"queue_lat": queue_lat,
"dequeue_lat": dequeue_lat,
"dequeue_count": dequeue_count,
"bluestore_lat": bluestore_lat,
"color": color
}
df = pd.DataFrame(all_data)
return df
def gen_bs_data_frame(bs_stats):
bs_stime = utils.gen_element_in_diclist(bs_stats, "start_time")
bs_lat = utils.gen_element_in_diclist(bs_stats, "latency")
bs_len = utils.gen_element_in_diclist(bs_stats, "len")
bs_align = utils.gen_element_in_diclist(bs_stats, "aligned")
bs_ts = [utils.timestamp(x) for x in bs_stime]
bs_color = ["green" for x in bs_stime]
all_data = {
"ts": bs_ts,
"align": bs_align,
"length": bs_len,
"lat": bs_lat,
"color": bs_color
}
df = pd.DataFrame(all_data)
return df
def test_to_dict(self):
self.assertEqual(TextThought.objects.count(), 0)
thought = TextThought()
thought.author_id = self.user.id
thought.author = self.user.username
thought.content = 'test content'
thought.title = 'test title'
thought.tags = {'tag1', 'tag2', 'tag3'}
thought.save()
d = thought.to_dict()
self.assertEqual(len(d), 4)
self.assertEqual(set(d['tags']), {'tag1', 'tag2', 'tag3'})
self.assertEqual(d['content'], 'test content')
self.assertEqual(d['title'], 'test title')
now = timestamp(utcnow())
just_now = now - 5
self.assertTrue(now >= d['pub_date'] >= just_now)
def to_dict(self):
return {
'title': self.title,
'pub_date': timestamp(self.id.generation_time),
'tags': list(self.tags),
}
def run(self): with open(self.path + timestamp() + '.html', 'wb') as fp: # 定义测试报告 r = HTMLTestRunner(stream=fp, title=self.title, description=self.description) r.run(self.discover)
def get_start_timestamp(fp):
first_line = fp.readline()
words = utils.log_split(first_line)
return utils.timestamp(words[0] + "-" + words[1])
def get_delta_hour(time):
ts = utils.timestamp(time)
delta_hour = (ts - start_ts).total_seconds() / 3600
return int(delta_hour)