def run(self, accessor, scale, threshold=None):
result = {}
for node, values in stats_buffer.nodes.iteritems():
if values["status"] != "healthy":
continue
if values["StorageInfo"].has_key("hdd"):
result[node] = {
"ip": values["host"],
"port": values["port"],
"type" : "hdd",
"free": util.size_label(values["StorageInfo"]["hdd"]["free"]),
"quotaTotal" : util.size_label(values["StorageInfo"]["hdd"]["quotaTotal"]),
"used" : util.size_label(values["StorageInfo"]["hdd"]["used"]),
"usedByData" : util.size_label(values["StorageInfo"]["hdd"]["usedByData"]),
"total" : util.size_label(values["StorageInfo"]["hdd"]["total"])}
if values["StorageInfo"].has_key("ram"):
result[node] = {
"ip": values["host"],
"port": values["port"],
"type" : "ram",
"quotaTotal" : util.size_label(values["StorageInfo"]["ram"]["quotaTotal"]),
"used" : util.size_label(values["StorageInfo"]["ram"]["used"]),
"usedByData" : util.size_label(values["StorageInfo"]["ram"]["usedByData"]),
"total" : util.size_label(values["StorageInfo"]["ram"]["total"])}
return result
def run(self, accessor):
result = []
for node, values in stats_buffer.nodes.iteritems():
result.append({"ip": values["host"],
"port": values["port"],
"cpuUtilization" :util.pretty_float(values["systemStats"]["cpu_utilization_rate"]),
"swapTotal": util.size_label(values["systemStats"]["swap_total"]),
"swapUsed" : util.size_label(values["systemStats"]["swap_used"]),
"currentItems" : values["systemStats"]["currentItems"],
"currentItemsTotal" : values["systemStats"]["currentItemsTotal"],
"replicaCurrentItems" : values["systemStats"]["replicaCurrentItems"]})
return result
def run(self, accessor, scale, threshold=None):
result = {}
for node, values in stats_buffer.nodes.iteritems():
if values["status"] != "healthy":
continue
result[node] = {
"ip": values["host"],
"port": values["port"],
"cpuUtilization" :util.pretty_float(values["systemStats"]["cpu_utilization_rate"]),
"swapTotal": util.size_label(values["systemStats"]["swap_total"]),
"swapUsed" : util.size_label(values["systemStats"]["swap_used"]),
"currentItems" : values["systemStats"]["currentItems"],
"currentItemsTotal" : values["systemStats"]["currentItemsTotal"],
"replicaCurrentItems" : values["systemStats"]["replicaCurrentItems"]}
return result
def run(self, accessor, threshold=None):
result = {}
if threshold.has_key("MemoryFragmentation"):
threshold_val = threshold["MemoryFragmentation"][accessor["name"]]
else:
threshold_val = accessor["threshold"]
for bucket, bucket_stats in stats_buffer.node_stats.iteritems():
num_error = []
for node, stats_info in bucket_stats.iteritems():
for key, value in stats_info.iteritems():
if key.find(accessor["counter"]) >= 0:
if accessor.has_key("threshold"):
if int(value) > threshold_val:
if accessor.has_key("unit"):
if accessor["unit"] == "time":
symptom = accessor["symptom"].format(util.time_label(value), util.time_label(threshold_val))
num_error.append({"node":node, "value": symptom})
elif accessor["unit"] == "size":
symptom = accessor["symptom"].format(util.size_label(value), util.size_label(threshold_val))
num_error.append({"node":node, "value": symptom})
else:
symptom = accessor["symptom"].format(value, threshold_val)
num_error.append({"node":node, "value": symptom})
else:
symptom = accessor["symptom"].format(value, threshold_val)
num_error.append({"node":node, "value": symptom})
if len(num_error) > 0:
result[bucket] = {"error" : num_error}
return result
def run(self, accessor, scale, threshold=None):
result = {}
if threshold.has_key(accessor["name"]):
threshold_val = threshold[accessor["name"]]
elif accessor.has_key("threshold"):
threshold_val = accessor["threshold"]
else:
threshold_val = None
for bucket, bucket_stats in stats_buffer.node_stats.iteritems():
stats = []
for node, stats_info in bucket_stats.iteritems():
if accessor["counter"] not in stats_info.keys():
stats.append((node, "N/A"))
continue
for key, value in stats_info.iteritems():
if isinstance(value, dict):
continue
if key.find(accessor["counter"]) >= 0:
if accessor.has_key("threshold"):
if int(value) > threshold_val:
stats.append((node, (key, value)))
else:
if accessor.has_key("unit"):
if accessor["unit"] == "time":
stats.append((node, util.time_label(value)))
elif accessor["unit"] == "size":
stats.append((node, util.size_label(int(value))))
else:
stats.append((node, (key,value)))
result[bucket] = stats
return result
def run(self, accessor, scale, threshold=None):
result = {}
for bucket, stats_info in stats_buffer.buckets.iteritems():
values = stats_info[scale][accessor["counter"]]
timestamps = values["timestamp"]
timestamps = [x - timestamps[0] for x in timestamps]
nodeStats = values["nodeStats"]
for node, vals in nodeStats.iteritems():
if len(vals):
avg = vals[-1]
else:
avg = 0
if accessor.has_key("unit"):
if accessor["unit"] == "size":
avg = util.size_label(avg)
elif accessor["unit"] == "number":
avg = util.number_label(avg)
elif accessor["unit"] == "time":
avg = util.time_label(avg)
else:
avg = util.pretty_float(avg)
if result.has_key(node):
result[node].append((bucket, avg))
else:
result[node] = [(bucket, avg)]
return result
def run(self, accessor):
result = []
total = 0
for node, nodeinfo in stats_buffer.nodes.iteritems():
if nodeinfo["StorageInfo"].has_key("hdd"):
total += nodeinfo['StorageInfo']['hdd']['free']
result.append(util.size_label(total))
return result
def run(self, accessor, threshold=None):
total = 0
for node, nodeinfo in stats_buffer.nodes.iteritems():
if nodeinfo["status"] != "healthy":
continue
if nodeinfo["StorageInfo"].has_key("hdd"):
total += nodeinfo['StorageInfo']['hdd']['usedByData']
return util.size_label(total)
def run(self, accessor, scale, threshold=None):
result = {}
sizing = {}
if threshold.has_key(accessor["name"]):
threshold_val = threshold[accessor["name"]]
elif accessor.has_key("threshold"):
threshold_val = accessor["threshold"]
else:
threshold_val = None
for bucket, bucket_stats in stats_buffer.node_stats.iteritems():
stats = []
for node, stats_info in bucket_stats.iteritems():
if not sizing.has_key(node):
sizing[node] = []
if accessor["counter"][0] not in stats_info.keys():
stats.append((node, "N/A"))
sizing[node].append((bucket, "N/A"))
continue
for key, value in stats_info.iteritems():
if isinstance(value, dict):
continue
if key.find(accessor["counter"][0]) >= 0:
if accessor["counter"][1] in stats_info and \
accessor["counter"][2] in stats_info:
total_item_resident = int(stats_info[accessor["counter"][1]]) -\
int(stats_info[accessor["counter"][2]])
if total_item_resident:
value = float(value) / total_item_resident
else:
value = 0
if accessor.has_key("unit"):
if accessor["unit"] == "time":
stats.append((node, util.time_label(value)))
sizing[node].append((bucket, util.time_label(value)))
elif accessor["unit"] == "size":
stats.append((node, util.size_label(int(value))))
sizing[node].append((bucket, util.size_label(int(value))))
else:
stats.append((node, (key,value)))
result[bucket] = stats
result["_sizing"] = sizing
return result
def run(self, accessor):
result = []
for node, values in stats_buffer.nodes.iteritems():
if values["StorageInfo"].has_key("hdd"):
result.append({
"ip":
values["host"],
"port":
values["port"],
"type":
"hdd",
"free":
util.size_label(values["StorageInfo"]["hdd"]["free"]),
"quotaTotal":
util.size_label(
values["StorageInfo"]["hdd"]["quotaTotal"]),
"used":
util.size_label(values["StorageInfo"]["hdd"]["used"]),
"usedByData":
util.size_label(
values["StorageInfo"]["hdd"]["usedByData"]),
"total":
util.size_label(values["StorageInfo"]["hdd"]["total"])
})
if values["StorageInfo"].has_key("ram"):
result.append({
"ip":
values["host"],
"port":
values["port"],
"type":
"ram",
"quotaTotal":
util.size_label(
values["StorageInfo"]["ram"]["quotaTotal"]),
"used":
util.size_label(values["StorageInfo"]["ram"]["used"]),
"usedByData":
util.size_label(
values["StorageInfo"]["ram"]["usedByData"]),
"total":
util.size_label(values["StorageInfo"]["ram"]["total"])
})
return result
def run(self, accessor):
result = []
for node, values in stats_buffer.nodes.iteritems():
result.append({
"ip":
values["host"],
"port":
values["port"],
"cpuUtilization":
util.pretty_float(
values["systemStats"]["cpu_utilization_rate"]),
"swapTotal":
util.size_label(values["systemStats"]["swap_total"]),
"swapUsed":
util.size_label(values["systemStats"]["swap_used"]),
"currentItems":
values["systemStats"]["currentItems"],
"currentItemsTotal":
values["systemStats"]["currentItemsTotal"],
"replicaCurrentItems":
values["systemStats"]["replicaCurrentItems"]
})
return result
def run(self, accessor):
result = []
for node, values in stats_buffer.nodes.iteritems():
if values["StorageInfo"].has_key("hdd"):
result.append({"ip": values["host"],
"port": values["port"],
"type" : "hdd",
"free": util.size_label(values["StorageInfo"]["hdd"]["free"]),
"quotaTotal" : util.size_label(values["StorageInfo"]["hdd"]["quotaTotal"]),
"used" : util.size_label(values["StorageInfo"]["hdd"]["used"]),
"usedByData" : util.size_label(values["StorageInfo"]["hdd"]["usedByData"]),
"total" : util.size_label(values["StorageInfo"]["hdd"]["total"])})
if values["StorageInfo"].has_key("ram"):
result.append({"ip": values["host"],
"port": values["port"],
"type" : "ram",
"quotaTotal" : util.size_label(values["StorageInfo"]["ram"]["quotaTotal"]),
"used" : util.size_label(values["StorageInfo"]["ram"]["used"]),
"usedByData" : util.size_label(values["StorageInfo"]["ram"]["usedByData"]),
"total" : util.size_label(values["StorageInfo"]["ram"]["total"])})
return result
def run(self, accessor, scale, threshold=None):
result = {}
for node, values in stats_buffer.nodes.iteritems():
if values["status"] != "healthy":
continue
val = None
if isinstance(accessor["counter"], list):
val = values[accessor["countergroup"]][accessor["counter"][0]] - values[accessor["countergroup"]][accessor["counter"][1]]
else:
if accessor.has_key("suffix"):
val = values[accessor["countergroup"]][accessor["counter"]][accessor["suffix"]]
else:
val = values[accessor["countergroup"]][accessor["counter"]]
if val:
result[node] = val
if accessor.has_key("unit"):
if accessor["unit"] == "size":
result[node] = util.size_label(val)
return result
def run(self, accessor):
result = {}
cluster = 0
for bucket, stats_info in stats_buffer.buckets.iteritems():
values = stats_info[accessor["scale"]][accessor["counter"]]
timestamps = values["timestamp"]
timestamps = [x - timestamps[0] for x in timestamps]
nodeStats = values["nodeStats"]
samplesCount = values["samplesCount"]
trend = []
total = 0
data = []
for node, vals in nodeStats.iteritems():
avg = sum(vals) / samplesCount
trend.append((node, util.size_label(avg)))
data.append(avg)
#print data
trend.append(("variance", util.two_pass_variance(data)))
result[bucket] = trend
return result
def run(self, accessor):
result = {}
for bucket, bucket_stats in stats_buffer.node_stats.iteritems():
stats = []
for node, stats_info in bucket_stats.iteritems():
for key, value in stats_info.iteritems():
if key.find(accessor["counter"]) >= 0:
if accessor.has_key("threshold"):
if int(value) > accessor["threshold"]:
stats.append((node, (key, value)))
else:
if accessor.has_key("unit"):
if accessor["unit"] == "time":
stats.append((node, util.time_label(value)))
elif accessor["unit"] == "size":
stats.append((node, util.size_label(int(value))))
else:
stats.append((node, (key,value)))
result[bucket] = stats
return result
def run(self, accessor):
result = {}
for bucket, bucket_stats in stats_buffer.node_stats.iteritems():
stats = []
for node, stats_info in bucket_stats.iteritems():
for key, value in stats_info.iteritems():
if key.find(accessor["counter"]) >= 0:
if accessor.has_key("threshold"):
if int(value) > accessor["threshold"]:
stats.append((node, (key, value)))
else:
if accessor.has_key("unit"):
if accessor["unit"] == "time":
stats.append(
(node, util.time_label(value)))
elif accessor["unit"] == "size":
stats.append(
(node, util.size_label(int(value))))
else:
stats.append((node, (key, value)))
result[bucket] = stats
return result
def run(self, accessor, scale, threshold=None):
result = {}
thresholdval = accessor["threshold"]
if threshold.has_key("PerformanceDiagnosis_diskread"):
thresholdval = threshold["PerformanceDiagnosis_diskread"]
for bucket, stats_info in stats_buffer.buckets.iteritems():
if stats_info[scale].get(accessor["counter"][0], None) is None:
return result
diskRead_values = stats_info[scale][accessor["counter"][0]]
cacheMissRate_values = stats_info[scale][accessor["counter"][1]]
arr_values = stats_info[scale][accessor["counter"][2]]
memUsed_values = stats_info[scale][accessor["counter"][3]]
curr_values = stats_info[scale][accessor["counter"][4]]
cmdSet_values = stats_info[scale][accessor["counter"][5]]
timestamps = diskRead_values["timestamp"]
samplesCount = diskRead_values["samplesCount"]
trend = []
num_warn = []
for node, vals in diskRead_values["nodeStats"].iteritems():
diskRead_vals = diskRead_values["nodeStats"][node]
cacheMissRate_vals = cacheMissRate_values["nodeStats"][node]
arr_vals = arr_values["nodeStats"][node]
memUsed_vals = memUsed_values["nodeStats"][node]
curr_vals = curr_values["nodeStats"][node]
cmdSet_vals = cmdSet_values["nodeStats"][node]
if samplesCount > 0:
node_avg_mem = sum(memUsed_vals) / samplesCount
node_avg_curr = sum(curr_vals) / samplesCount
node_avg_cmdset = sum(cmdSet_vals) / samplesCount
else:
node_avg_curr = 0
# Fine grained analysis
abnormal_segs = util.abnormal_extract(diskRead_vals, thresholdval["ep_bg_fetched"])
abnormal_vals = []
for seg in abnormal_segs:
begin_index = seg[0]
seg_total = seg[1]
if seg_total < thresholdval["recurrence"]:
continue
end_index = begin_index + seg_total
diskread_avg = sum(diskRead_vals[begin_index : end_index]) / seg_total
cmr_avg = sum(cacheMissRate_vals[begin_index : end_index]) / seg_total
arr_avg = sum(arr_vals[begin_index : end_index]) / seg_total
mem_avg = sum(memUsed_vals[begin_index : end_index]) / seg_total
curr_avg = sum(curr_vals[begin_index : end_index]) / seg_total
cmdSet_avg = sum(cmdSet_vals[begin_index : end_index]) / seg_total
if cmr_avg > thresholdval["ep_cache_miss_rate"] and \
arr_avg < thresholdval["vb_active_resident_items_ratio"] and \
mem_avg > node_avg_mem and \
curr_avg > node_avg_curr and \
cmdSet_avg > node_avg_cmdset:
symptom = accessor["symptom"] % (util.pretty_datetime(timestamps[begin_index]),
util.pretty_datetime(timestamps[end_index-1]),
util.number_label(int(curr_avg)),
util.size_label(int(mem_avg)),
util.pretty_float(cmr_avg),
util.pretty_float(arr_avg),
util.number_label(int(diskread_avg)))
num_warn.append({"node":node, "value":symptom})
abnormal_vals.append(diskread_avg)
if len(abnormal_vals) > 0:
trend.append((node, {"value" : util.pretty_float(sum(abnormal_vals)/len(abnormal_vals)),
"raw" : abnormal_vals}
))
if len(num_warn) > 0:
trend.append(("warn", num_warn))
result[bucket] = trend
return result
def run(self, accessor):
result = {}
for bucket, bucket_stats in stats_buffer.node_stats.iteritems():
num_error = []
for node, stats_info in bucket_stats.iteritems():
for key, value in stats_info.iteritems():
if key.find(accessor["counter"]) >= 0:
if accessor.has_key("threshold"):
if int(value) > accessor["threshold"]:
if accessor.has_key("unit"):
if accessor["unit"] == "time":
num_error.append({"node":node, "value": (key, util.time_label(value))})
elif accessor["unit"] == "size":
num_error.append({"node":node, "value": (key, util.size_label(value))})
else:
num_error.append({"node":node, "value": (key, value)})
else:
num_error.append({"node":node, "value": (key, value)})
if len(num_error) > 0:
result[bucket] = {"error" : num_error}
return result
