def run(self, accessor, scale, threshold=None):
result = {}
if threshold.has_key("DiskQueueDiagnosis"):
threshold_val = threshold["DiskQueueDiagnosis"][accessor["name"]]
else:
threshold_val = accessor["threshold"]
for bucket, stats_info in stats_buffer.buckets.iteritems():
#print bucket, stats_info
disk_queue_avg_error = []
disk_queue_avg_warn = []
res = []
values = stats_info[scale][accessor["counter"][0]]
curr_values = stats_info[scale][accessor["counter"][1]]
cmdset_values = stats_info[scale][accessor["counter"][2]]
nodeStats = values["nodeStats"]
samplesCount = values["samplesCount"]
timestamps = values["timestamp"]
total = []
for node, vals in nodeStats.iteritems():
curr_vals = curr_values["nodeStats"][node]
cmdset_vals = cmdset_values["nodeStats"][node]
if samplesCount > 0:
node_avg_dwq = sum(vals) / samplesCount
node_avg_curr = sum(curr_vals) / samplesCount
node_avg_cmdset = sum(cmdset_vals) / samplesCount
else:
node_avg_curr = 0
node_avg_cmdest = 0
abnormal_segs = util.abnormal_extract(vals, threshold_val["disk_write_queue"]["low"])
abnormal_vals = []
for seg in abnormal_segs:
begin_index = seg[0]
seg_total = seg[1]
if seg_total < threshold_val["recurrence"]:
continue
end_index = begin_index + seg_total
cmdset_avg = sum(cmdset_vals[begin_index : end_index]) / seg_total
curr_avg = sum(curr_vals[begin_index : end_index]) / seg_total
dwq_avg = sum(vals[begin_index : end_index]) / seg_total
if curr_avg > node_avg_curr and cmdset_avg > node_avg_cmdset:
symptom = accessor["symptom"] % (util.pretty_datetime(timestamps[begin_index]),
util.pretty_datetime(timestamps[end_index-1]),
util.number_label(int(cmdset_avg)),
util.number_label(int(curr_avg)),
util.number_label(dwq_avg))
abnormal_vals.append(dwq_avg)
if dwq_avg > threshold_val["disk_write_queue"]["high"]:
disk_queue_avg_error.append({"node":node, "value":symptom})
else:
disk_queue_avg_warn.append({"node":node, "level":"yellow", "value":symptom})
if len(abnormal_vals) > 0:
res.append((node, {"value":util.number_label(dwq_avg), "raw":abnormal_vals}))
total.append(node_avg_dwq)
if len(disk_queue_avg_error) > 0:
res.append(("error", disk_queue_avg_error))
if len(disk_queue_avg_warn) > 0:
res.append(("warn", disk_queue_avg_warn))
if len(nodeStats) > 0:
rate = sum(total) / len(nodeStats)
res.append(("total", {"value" : util.number_label(rate),
"raw" : total}))
result[bucket] = res
return result
def run(self, accessor, scale, threshold=None):
result = {}
thresholdval = accessor["threshold"]
if threshold.has_key("PerformanceDiagnosis_diskread"):
thresholdval = threshold["PerformanceDiagnosis_diskread"]
for bucket, stats_info in stats_buffer.buckets.iteritems():
if stats_info[scale].get(accessor["counter"][0], None) is None:
return result
diskRead_values = stats_info[scale][accessor["counter"][0]]
cacheMissRate_values = stats_info[scale][accessor["counter"][1]]
arr_values = stats_info[scale][accessor["counter"][2]]
memUsed_values = stats_info[scale][accessor["counter"][3]]
curr_values = stats_info[scale][accessor["counter"][4]]
cmdSet_values = stats_info[scale][accessor["counter"][5]]
timestamps = diskRead_values["timestamp"]
samplesCount = diskRead_values["samplesCount"]
trend = []
num_warn = []
for node, vals in diskRead_values["nodeStats"].iteritems():
diskRead_vals = diskRead_values["nodeStats"][node]
cacheMissRate_vals = cacheMissRate_values["nodeStats"][node]
arr_vals = arr_values["nodeStats"][node]
memUsed_vals = memUsed_values["nodeStats"][node]
curr_vals = curr_values["nodeStats"][node]
cmdSet_vals = cmdSet_values["nodeStats"][node]
if samplesCount > 0:
node_avg_mem = sum(memUsed_vals) / samplesCount
node_avg_curr = sum(curr_vals) / samplesCount
node_avg_cmdset = sum(cmdSet_vals) / samplesCount
else:
node_avg_curr = 0
# Fine grained analysis
abnormal_segs = util.abnormal_extract(diskRead_vals, thresholdval["ep_bg_fetched"])
abnormal_vals = []
for seg in abnormal_segs:
begin_index = seg[0]
seg_total = seg[1]
if seg_total < thresholdval["recurrence"]:
continue
end_index = begin_index + seg_total
diskread_avg = sum(diskRead_vals[begin_index : end_index]) / seg_total
cmr_avg = sum(cacheMissRate_vals[begin_index : end_index]) / seg_total
arr_avg = sum(arr_vals[begin_index : end_index]) / seg_total
mem_avg = sum(memUsed_vals[begin_index : end_index]) / seg_total
curr_avg = sum(curr_vals[begin_index : end_index]) / seg_total
cmdSet_avg = sum(cmdSet_vals[begin_index : end_index]) / seg_total
if cmr_avg > thresholdval["ep_cache_miss_rate"] and \
arr_avg < thresholdval["vb_active_resident_items_ratio"] and \
mem_avg > node_avg_mem and \
curr_avg > node_avg_curr and \
cmdSet_avg > node_avg_cmdset:
symptom = accessor["symptom"] % (util.pretty_datetime(timestamps[begin_index]),
util.pretty_datetime(timestamps[end_index-1]),
util.number_label(int(curr_avg)),
util.size_label(int(mem_avg)),
util.pretty_float(cmr_avg),
util.pretty_float(arr_avg),
util.number_label(int(diskread_avg)))
num_warn.append({"node":node, "value":symptom})
abnormal_vals.append(diskread_avg)
if len(abnormal_vals) > 0:
trend.append((node, {"value" : util.pretty_float(sum(abnormal_vals)/len(abnormal_vals)),
"raw" : abnormal_vals}
))
if len(num_warn) > 0:
trend.append(("warn", num_warn))
result[bucket] = trend
return result