def gen_caches(project, tablename, datestring):
datalogger = DataLogger(basedir, project, tablename)
caches = datalogger.get_caches(datestring)
suffix = "%s/%s/%s\t" % (datestring, project, tablename)
data = None
if caches["tsa"]["raw"] is None:
if len(caches["tsa"]["keys"]) == 0:
logging.info("%s RAW Data not availabale maybe archived, tsa exists already", suffix)
else:
logging.debug("%s RAW Data is missing, no tsa archive exists", suffix)
else:
if len(caches["tsa"]["keys"]) == 0:
logging.info("%s TSA Archive missing, calling get_tsa and load_tsastats", suffix)
data = datalogger.load_tsa(datestring)
else:
if len(caches["tsastat"]["keys"]) == 0:
logging.info("%s TSASTAT Archive missing, calling load_tsastats", suffix)
data = datalogger.load_tsastats(datestring)
else:
if len(caches["ts"]["keys"]) == 0:
logging.info("%s there are no ts archives, something went wrong, or tsa is completely empty, calling load_tsastats", suffix)
data = datalogger.load_tsastats(datestring)
else:
logging.debug("%s All fine", suffix)
if caches["quantile"]["exists"] is not True:
logging.info("%s Quantile archive is missing, calling load_quantile", suffix)
data = datalogger.load_quantile(datestring)
del data
del caches
del datalogger
def get_scatter_data(args):
"""
gets scatter plot data of two value_keys of the same tablename
ex: Datalogger/{projectname}/{tablename}/{datestring}/{value_keyname1}/{value_keyname2}/{stat function name}
value_keyname{1/2} has to be one of get_value_keynames
stat function name has to be one of get_stat_func_names
returns:
json(highgraph data)
"""
assert len(args) == 6
project, tablename, datestring, value_key1, value_key2, stat_func_name = args
logging.info("project : %s", project)
logging.info("tablename : %s", tablename)
logging.info("datestring : %s", datestring)
logging.info("value_key1 : %s", value_key1)
logging.info("value_key2 : %s", value_key2)
datalogger = DataLogger(basedir, project, tablename)
tsastats = datalogger.load_tsastats(datestring)
hc_scatter_data = []
for key, tsstat in tsastats.items():
hc_scatter_data.append({
"name" : str(key),
"data" : ((tsstat[value_key1]["avg"], tsstat[value_key2]["avg"]), )
})
return json.dumps(hc_scatter_data)
def sr_hrstorage_unused(args):
"""
special report to get a report of unused SNMP Host Storage
works only with snmp/hrStorageTable
"""
datestring, storage_type = args[:2]
datalogger = DataLogger(basedir, "snmp", "hrStorageTable")
tsastat = datalogger.load_tsastats(datestring)
data = []
data.append(("hostname", "hrStorageDescr", "hrStorageSizeKb", "hrStorageUsedKb", "hrStorageNotUsedKbMin", "hrStorageNotUsedPct"))
for index_key in tsastat.keys():
# (u'srvcacdbp1.tilak.cc', u'Physical Memory',
# u'HOST-RESOURCES-TYPES::hrStorageRam')
if (u"HOST-RESOURCES-TYPES::%s" % storage_type) not in index_key:
del tsastat[index_key]
if index_key[1][:4] in (u"/run", u"/dev", u"/sys"):
del tsastat[index_key]
for key, tsstat in tsastat.items():
sizekb = tsstat["hrStorageSize"]["min"] * tsstat["hrStorageAllocationUnits"]["max"] / 1024
usedkb = tsstat["hrStorageUsed"]["max"] * tsstat["hrStorageAllocationUnits"]["max"] / 1024
notused = sizekb - usedkb
notused_pct = 0.0
try:
notused_pct = 100.0 * notused / sizekb
except ZeroDivisionError:
pass
data.append((key[0], key[1], "%0.2f" % sizekb, "%0.2f" % usedkb, "%0.2f" % notused, "%0.2f" % notused_pct))
return json.dumps(data)
def get_monthly_ts(self, project, tablename, monthstring, args):
"""
get monthly statistical values
TODO: should be combined with get_lt_ts
"""
index_key_enc = None
value_keyname = None
stat_func_name = "avg"
if len(args) == 2:
index_key_enc, value_keyname = args
else:
index_key_enc, value_keyname, stat_func_name = args
if len(monthstring) != 7:
web.internalerror()
return "monthstring, has to be in YYYY-MM format"
# key_str should be a tuple string, convert to unicode tuple
index_key = tuple([unicode(key_value) for key_value in eval(base64.b64decode(index_key_enc))])
logging.info("index_key : %s", index_key)
logging.info("value_keyname : %s", value_keyname)
logging.info("stat_func_name: %s", stat_func_name)
datalogger = DataLogger(basedir, project, tablename)
filterkeys = dict(zip(datalogger.index_keynames, index_key))
ret_data = []
for datestring in datalogger.monthwalker(monthstring):
logging.debug("getting tsatstats for %s", monthstring)
try:
tsastats = datalogger.load_tsastats(datestring, filterkeys=filterkeys)
ret_data.append([datestring, tsastats[index_key][value_keyname][stat_func_name]])
except DataLoggerRawFileMissing as exc:
logging.error("No Input File for datestring %s found, skipping this date", datestring)
except DataLoggerLiveDataError as exc:
logging.error("Reading from live data is not allowed, skipping this data, and ending loop")
break
return json.dumps(ret_data)
def get_tsastats_func(self, args):
"""
return json data to render html table from it
parameters:
<b>project</b> project string
<b>tablename</b> tablename string
<b>datestring</b> datestring in YYYY-MM-DD form
<b>stat_func_name</b> statistical function
"""
project, tablename, datestring, stat_func_name = args
datalogger = DataLogger(basedir, project, tablename)
tsastats = datalogger.load_tsastats(datestring)
return json.dumps(tsastats.to_csv(stat_func_name))
def sr_vicenter_unused_mem(args):
"""
special resport to find virtual machine which are not used their ram entirely
on this machines there is a possibility to save some virtual memory
works only for VMware machine, in special virtualMachineMemoryStats
"""
datestring = args[0]
datalogger = DataLogger(basedir, "vicenter", "virtualMachineMemoryStats")
tsastat = datalogger.load_tsastats(datestring)
tsastat_g = datalogger.tsastat_group_by(tsastat, ("hostname", ))
data = []
data.append(("hostname", "avg_active_max", "avg_granted_min", "avg_notused_min"))
for key in tsastat_g.keys():
not_used = tsastat_g[key]["mem.granted.average"]["min"] - tsastat_g[key]["mem.active.average"]["max"]
data.append((key[0], "%0.2f" % tsastat_g[key]["mem.active.average"]["max"], "%0.3f" % tsastat_g[key]["mem.granted.average"]["min"], "%0.2f" % not_used))
return json.dumps(data)
def sr_vicenter_unused_cpu_cores(args):
"""
special report to find virtual machine which re not used their virtual core entirely
on this machine there is a possibility to save some virtual cores
works only for VMware machines, in special virtualMachineCpuStats
"""
datestring = args[0]
datalogger = DataLogger(basedir, "vicenter", "virtualMachineCpuStats")
tsastat = datalogger.load_tsastats(datestring)
tsastat_g = datalogger.tsastat_group_by(tsastat, ("hostname", ))
data = []
data.append(("hostname", "avg_idle_min", "avg_used_avg", "avg_used_max"))
for key in tsastat_g.keys():
num_cpu = sum([key[0] in index_key for index_key in tsastat.keys()])
if num_cpu < 3:
continue
data.append((key[0], "%0.2f" % tsastat_g[key]["cpu.idle.summation"]["min"], "%0.2f" % tsastat_g[key]["cpu.used.summation"]["avg"], "%0.2f" % tsastat_g[key]["cpu.used.summation"]["max"]))
return json.dumps(data)
def get_tsastats(self, args):
"""
return exported TimeseriesArrayStats json formatted
[
list of index_keys,
list of value_keys,
list of [
index_key : tsstat_dictionary
]
]
returns:
json(tsastats_dict)
"""
project, tablename, datestring = args[:3]
datalogger = DataLogger(basedir, project, tablename)
tsastats = datalogger.load_tsastats(datestring)
return tsastats.to_json()
def get_tsastats_table(self, args):
"""
return html renderer table from tsatstats data
"""
def csv_to_table(csvdata, keys):
outbuffer = []
outbuffer.append("<thead><tr>")
[outbuffer.append("<th>%s</th>" % header) for header in csvdata[0]]
outbuffer.append("</tr></thead><tbody>")
for values in csvdata[1:]:
outbuffer.append("<tr>")
[outbuffer.append("<td >%s</td>" % value) for value in values[0:keys]]
[outbuffer.append("<td type=numeric>%0.2f</td>" % value) for value in values[keys:]]
outbuffer.append("</tr>")
outbuffer.append("</tbody>")
return outbuffer
project, tablename, datestring, stat_func_name = args
datalogger = DataLogger(basedir, project, tablename)
tsastats = datalogger.load_tsastats(datestring)
return json.dumps("\n".join(csv_to_table(tsastats.to_csv(stat_func_name), len(tsastats.index_keys))))
def get_lt_ts(self, project, tablename, args):
"""
get longtime statistical values
"""
# datestringStart + "/" + datestringStop + "/" + Base64.encode(indexKey) + "/" + valueKeyname + "/" + statFuncName
start, stop, index_key_enc, value_keyname, stat_func_name = args
index_key = tuple([unicode(key_value) for key_value in eval(base64.b64decode(index_key_enc))])
datalogger = DataLogger(basedir, project, tablename)
filterkeys = dict(zip(datalogger.index_keynames, index_key))
ret_data = []
for datestring in datalogger.datewalker(start, stop):
try:
tsastats = datalogger.load_tsastats(datestring, filterkeys=filterkeys)
ret_data.append([datestring, tsastats[index_key][value_keyname][stat_func_name]])
except DataLoggerRawFileMissing as exc:
logging.error("No Input File for datestring %s found, skipping this date", datestring)
except DataLoggerLiveDataError as exc:
logging.error("Reading from live data is not allowed, skipping this data, and ending loop")
break
return json.dumps(ret_data)
def get_tsastats(self, project, tablename, datestring, args):
"""
return exported TimeseriesArrayStats json formatted
[
list of index_keys,
list of value_keys,
list of [
index_key : tsstat_dictionary
]
]
if optional args is given, only one specific statistical function is returned
returns:
json(tsastats_dict)
"""
logging.info("optional arguments received: %s", args)
datalogger = DataLogger(basedir, project, tablename)
tsastats = datalogger.load_tsastats(datestring)
if len(args) > 0:
return json.dumps(tsastats.to_csv(args[0]))
return tsastats.to_json()
def sr_hrstorageram_unused(args):
"""
special report to find servers which are not using their ram entirely
specially on virtual machines are is a huge saving potential
works only for snmp data especially hrStorageTable
"""
datestring = args[0]
datalogger = DataLogger(basedir, "snmp", "hrStorageTable")
tsastat = datalogger.load_tsastats(datestring)
data = []
data.append(("hostname", "hrStorageSizeKb", "hrStorageUsedKb", "hrStorageNotUsedKbMin", "hrStorageNotUsedPct"))
for index_key in tsastat.keys():
# (u'srvcacdbp1.tilak.cc', u'Physical Memory',
# u'HOST-RESOURCES-TYPES::hrStorageRam')
if u'HOST-RESOURCES-TYPES::hrStorageRam' not in index_key:
del tsastat[index_key]
for key, tsstat in datalogger.tsastat_group_by(tsastat, ("hostname", )).items():
sizekb = tsstat["hrStorageSize"]["min"] * tsstat["hrStorageAllocationUnits"]["max"] / 1024
usedkb = tsstat["hrStorageUsed"]["max"] * tsstat["hrStorageAllocationUnits"]["max"] / 1024
notused = sizekb - usedkb
notused_pct = 100.0 * notused / sizekb
data.append((key[0], "%0.2f" % sizekb, "%0.2f" % usedkb, "%0.2f" % notused, "%0.2f" % notused_pct))
return json.dumps(data)
def get_tsstat(self, project, tablename, datestring, args):
"""
return exported TimeseriesStats data
[
list of index_keys,
list of value_keys,
list of [
index_key : tsstat_dictionary
]
]
if optional args is given, only one specific statistical function is returned
returns:
json(tsastats_dict)
"""
logging.info("optional arguments received: %s", args)
if len(args) > 0:
key_str = args[0]
key = tuple([unicode(key_value) for key_value in eval(base64.b64decode(key_str))])
datalogger = DataLogger(basedir, project, tablename)
tsastats = datalogger.load_tsastats(datestring)
return json.dumps(tsastats[key].stats)
class Test(unittest.TestCase):
def setUp(self):
self.basedir = "/var/rrd"
self.datestring = "2015-11-30"
self.project = DataLogger.get_projects(self.basedir)[0]
self.tablename = DataLogger.get_tablenames(self.basedir, self.project)[0]
self.datalogger = DataLogger(self.basedir, self.project, self.tablename)
def test_simple(self):
self.assertTrue(self.datalogger.project == self.project)
self.assertTrue(self.datalogger.tablename == self.tablename)
self.assertTrue(isinstance(self.datalogger.delimiter, basestring))
self.assertTrue(isinstance(self.datalogger.ts_keyname, basestring))
self.assertTrue(isinstance(self.datalogger.headers, tuple))
self.assertTrue(isinstance(self.datalogger.value_keynames, tuple))
self.assertTrue(all((keyname in self.datalogger.headers for keyname in self.datalogger.value_keynames)))
self.assertTrue(isinstance(self.datalogger.index_keynames, tuple))
self.assertTrue(all((keyname in self.datalogger.headers for keyname in self.datalogger.index_keynames)))
self.assertTrue(isinstance(self.datalogger.blacklist, tuple))
self.assertTrue(all((keyname in self.datalogger.headers for keyname in self.datalogger.blacklist)))
self.assertTrue(isinstance(self.datalogger.raw_basedir, basestring))
self.assertTrue(os.path.exists(self.datalogger.raw_basedir))
self.assertTrue(os.path.isdir(self.datalogger.raw_basedir))
self.assertTrue(isinstance(self.datalogger.global_cachedir, basestring))
self.assertTrue(os.path.exists(self.datalogger.global_cachedir))
self.assertTrue(os.path.isdir(self.datalogger.global_cachedir))
# meta is something like this
# {u'ts_keyname': u'ts',
# 'stat_func_names': [u'count', ... ],
# u'interval': 300,
# u'blacklist': [],
# u'headers': [u'ts', u'http_host', ... ],
# u'delimiter': u'\t',
# u'value_keynames': {
# u'actconn': u'asis',
# u'hits': u'asis',
# ...
# },
# u'index_keynames': [u'http_host']}
self.assertTrue(self.datalogger.meta["headers"] == list(self.datalogger.headers))
self.assertTrue(self.datalogger.meta["value_keynames"].keys() == list(self.datalogger.value_keynames))
self.assertTrue(self.datalogger.meta["index_keynames"] == list(self.datalogger.index_keynames))
self.assertTrue(self.datalogger.meta["blacklist"] == list(self.datalogger.blacklist))
self.assertTrue(self.datalogger.meta["delimiter"] == self.datalogger.delimiter)
self.assertTrue(self.datalogger.meta["ts_keyname"] == self.datalogger.ts_keyname)
self.assertTrue(isinstance(self.datalogger.meta["stat_func_names"], list))
def test_statics(self):
self.assertTrue(isinstance(DataLogger.get_user(self.basedir), basestring))
self.assertTrue(isinstance(DataLogger.get_group(self.basedir), basestring))
self.assertTrue(isinstance(DataLogger.get_yesterday_datestring(), basestring))
lbd = DataLogger.get_last_business_day_datestring()
self.assertTrue(isinstance(DataLogger.get_last_business_day_datestring(), basestring))
self.assertTrue(isinstance(DataLogger.datestring_to_date(lbd), datetime.date))
for datestring in DataLogger.datewalker("2016-01-01", "2016-02-29"):
self.assertTrue(isinstance(datestring, basestring))
for datestring in DataLogger.monthwalker("2016-02"):
self.assertTrue(isinstance(datestring, basestring))
self.assertEqual(list(DataLogger.monthwalker("2016-02"))[-1], "2016-02-29")
self.assertTrue(isinstance(DataLogger.get_ts_for_datestring("2016-01-01"), tuple))
self.assertTrue(isinstance(DataLogger.get_ts_for_datestring("2016-01-01")[0], float))
self.assertTrue(isinstance(DataLogger.get_ts_for_datestring("2016-01-01")[1], float))
def test_data(self):
self.datalogger.load_tsa(self.datestring)
self.datalogger.load_tsastats(self.datestring)
self.datalogger.load_correlationmatrix(self.datestring)
self.datalogger.load_quantile(self.datestring)
