def main2():
tsa = datalogger.load_tsa_raw(datestring)
print(len(tsa.keys()))
print(tsa.keys()[0])
print(tsa[tsa.keys()[0]])
tsastat = TimeseriesArrayStats(tsa)
print(tsastat[tsastat.keys()[0]])
def main3():
tsa = datalogger[datestring]
print(len(tsa.keys()))
print(tsa.keys()[0])
print(tsa[tsa.keys()[0]])
tsastat = TimeseriesArrayStats(tsa)
print(tsastat[tsastat.keys()[0]])
print(tsa[("ucsfib-sr1-1-mgmt0","port-channel1304","propVirtual")]["ifOutOctets"])
print(tsastat[("ucsfib-sr1-1-mgmt0","port-channel1304","propVirtual")])
def get_total_stats(datalogger, tsa, keyfunc):
wikitext = ""
wikitext += "---+ Total-Data Statistics\n"
for value_key in tsa.get_value_keys():
wikitext += "---++ %s Total Statistics of %s\n" % (datalogger.tablename, value_key)
wikitext += "This table is a summation of all data for field using %s\n" % "lambda a: sum(a)"
grouped_tsa = tsa.get_group_by_tsa((), group_func=lambda a: sum(a))
tsa_stats = TimeseriesArrayStats(grouped_tsa)
wikitext += get_wiki_dict_table(tsa_stats.get_stats(value_key), keyfunc)
return(wikitext)
def get_ungrouped_stats(datalogger, tsa, keyfunc):
wikitext = ""
wikitext += "---+ Ungrouped-Data Statistics\n"
wikitext += "Here are some statistical breakdowns for every index combination\n"
for value_key in tsa.get_value_keys():
wikitext += "---++ %s Statistics of %s\n" % (datalogger.tablename, value_key)
tsa_stats = TimeseriesArrayStats(tsa)
stat_dict = tsa_stats.get_stats(value_key)
wikitext += get_wiki_dict_table(stat_dict, keyfunc)
return(wikitext)
def get_grouped_stats(datalogger, tsa, keyfunc):
wikitext = ""
wikitext += "---+ Grouped-Data Statistics\n"
wikitext += "These statistics are grouped by some index_key, in sql something like select sum(value_key) from ... group by index_key\n"
for subkey in tsa.get_index_keys():
wikitext += "---++ Grouped by %s\n" % subkey
for value_key in tsa.get_value_keys():
wikitext += "---+++ %s Statistics of %s grouped by %s\n" % (datalogger.tablename, value_key, subkey)
wikitext += "This table is grouped by %s field using %s\n" % (subkey, "lambda a: sum(a)")
grouped_tsa = tsa.get_group_by_tsa((subkey,), group_func=lambda a: sum(a))
tsa_stats = TimeseriesArrayStats(grouped_tsa)
wikitext += get_wiki_dict_table(tsa_stats.get_stats(value_key), keyfunc)
return(wikitext)
def groupby(tsastat, index_keys):
"""
group tsastat by index_keys, which are a subset of the original index_keys
the grouping functions are predefined, it makes no sense to make this variable
parameters:
tsastat <TimeseriesArrayStats>
index_keys <tuple>
returns:
<TimeseriesArrayStats>
"""
group_funcs = {
"sum" : lambda a, b: a + b,
"avg" : lambda a, b: (a + b) / 2,
"min" : min,
"max" : max,
"count" : lambda a, b: a + b,
"std" : lambda a, b: (a + b) / 2,
"median" : lambda a, b: (a + b) / 2,
"mean" : lambda a, b: (a + b) / 2,
"last" : lambda a, b: (a + b) / 2,
"first" : lambda a, b: (a + b) / 2,
}
try:
assert all(index_key in tsastat.index_keys for index_key in index_keys)
except AssertionError:
logging.error("All given index_keys have to be in tsastat.index_keys")
return
# intermediate data
data = {}
for key in tsastat.keys():
key_dict = dict(zip(tsastat.index_keys, key))
group_key = tuple((key_dict[key] for key in index_keys))
if group_key not in data:
data[group_key] = tsastat[key].stats
else:
# there is something to group
for value_key in tsastat[key].keys():
for stat_func, value in tsastat[key][value_key].items():
# group values by function
grouped_value = group_funcs[stat_func](value, data[group_key][value_key][stat_func])
# store
data[group_key][value_key][stat_func] = grouped_value
# get to same format as TimeseriesArrayStats.to_json returns
outdata = [tsastat.index_keys, tsastat.value_keys, ]
outdata.append([(key, json.dumps(value)) for key, value in data.items()])
# use TimeseriesArrayStats.from_json to get to TimeseriesArrayStats
# object
new_tsastat = TimeseriesArrayStats.from_json(json.dumps(outdata))
return new_tsastat
def sr_report_unused_cpu(datestring):
special_reports_dir = os.path.join(datalogger.global_cachedir, datestring, "__special_reports")
if not os.path.exists(special_reports_dir):
os.mkdir(special_reports_dir)
tsastat = datalogger.load_tsastats(datestring)
# destination tsastat, like original but extended index_keynames
tsastats_new = TimeseriesArrayStats.__new__(TimeseriesArrayStats)
tsastats_new.index_keys = tsastat.index_keys + ("esxhost", "cluster")
tsastats_new.value_keys = tsastat.value_keys
tsastats_new.stats = {}
for key, stats in tsastat.stats.items():
key_dict = dict(zip(tsastat.index_keys, key))
vm_moref = webapp.get_vm_moref_by_name(key_dict["hostname"])
host_moref = webapp.get_host_by_vm(vm_moref)
# try to get ESX Server
try:
key_dict["esxhost"] = webapp.get_name_by_moref(host_moref[0])
except KeyError:
key_dict["esxhost"] = "not found"
# try to get cluster
try:
key_dict["cluster"] = webapp.get_cluster_by_vm(vm_moref)
except KeyError:
key_dict["cluster"] = "no cluster"
new_index_key = tuple((key_dict[key1] for key1 in tsastats_new.index_keys))
tsastats_new.stats[new_index_key] = stats
# group by hostname
tsastat_g = datalogger.tsastat_group_by(tsastats_new, ("hostname", "cluster", "esxhost"))
data = [tsastat_g.index_keynames + tsastat_g.value_keynames]
for key in tsastat_g.keys():
data.append(key + ("%0.2f" % tsastat_g[key]["cpu.idle.summation"]["min"], "%0.2f" % tsastat_g[key]["cpu.used.summation"]["avg"], "%0.2f" % tsastat_g[key]["cpu.used.summation"]["max"]))
json.dump(data, open(os.path.join(special_reports_dir, "sr_unused_cpu_by_hostname.json"), "w"))
# group by cluster
tsastat_g = datalogger.tsastat_group_by(tsastats_new, ("cluster", ))
data = [tsastat_g.index_keynames + tsastat_g.value_keynames]
for key in tsastat_g.keys():
data.append(key + ("%0.2f" % tsastat_g[key]["cpu.idle.summation"]["min"], "%0.2f" % tsastat_g[key]["cpu.used.summation"]["avg"], "%0.2f" % tsastat_g[key]["cpu.used.summation"]["max"]))
json.dump(data, open(os.path.join(special_reports_dir, "sr_unused_cpu_by_cluster.json"), "w"))
# group by esxhost
tsastat_g = datalogger.tsastat_group_by(tsastats_new, ("esxhost", "cluster"))
data = [tsastat_g.index_keynames + tsastat_g.value_keynames]
for key in tsastat_g.keys():
data.append(key + ("%0.2f" % tsastat_g[key]["cpu.idle.summation"]["min"], "%0.2f" % tsastat_g[key]["cpu.used.summation"]["avg"], "%0.2f" % tsastat_g[key]["cpu.used.summation"]["max"]))
json.dump(data, open(os.path.join(special_reports_dir, "sr_unused_cpu_by_esxhost.json"), "w"))
def get_tsastats(self, project, tablename, datestring):
"""
get TimeseriesStatsArray object for this particular project/tablename/datestring combination
parameters:
project <str>
tablename <str>
datestring <str>
returns:
<TimeseriesStatsArray>
"""
uri_params = {
"project" : project,
"tablename" : tablename,
"datestring" : datestring,
}
query_params = {}
data = self.__get_raw_data("get_tsastats", uri_params, query_params)
tsastats = TimeseriesArrayStats.from_json(data)
return tsastats