def tsa_group_by(tsa, datestring, index_keynames, group_func, interval):
"""
group given tsa by subkeys, and use group_func to aggregate data
the individual timeseries are automatically grouped by timestamp and interval
defined in configuration
parameters:
tsa <TimeseriesArray>
datestring <str> datestring to use to aggregate data TODO: get this from tsa
subkey <tuple> could also be empty, to aggregate everything
group_func <func> like lambda a, b : (a + b) / 2 to get averages
interval <int> interval in seconds the timeseries values should appear
returns:
<TimeseriesArray>
"""
# intermediated tsa
tsa2 = TimeseriesArray(index_keynames=index_keynames,
value_keynames=tsa.value_keynames,
ts_key=tsa.ts_key,
datatypes=tsa.datatypes)
start_ts, _ = DataLogger.get_ts_for_datestring(datestring)
ts_keyname = tsa.ts_key
for data in tsa.export():
# align timestamp
nearest_slot = round((data[ts_keyname] - start_ts) / interval)
data[ts_keyname] = int(start_ts + nearest_slot * interval)
tsa2.add(data, group_func)
return tsa2
def test_export(self):
print("testing export, add")
tsa = TimeseriesArray(meta["index_keynames"],
meta["value_keynames"],
"ts",
datatypes=meta["value_keynames"])
for entry in self.app.export():
tsa.add(entry)
assert tsa == self.app
def test_load(self):
print("testing load, get_ts_filename, filtermatch, get_dumpfilename")
tsa = TimeseriesArray.load("testdata/fcIfC3AccountingTable",
meta2["index_keynames"],
datatypes=meta2["value_keynames"],
filterkeys=None,
index_pattern=None,
matchtype="and")
#for key in tsa.keys():
# print(key)
# loading only Timeseries matching this index key
filterkeys = {"hostname": "fca-sr2-8gb-21", "ifDescr": None}
tsa = TimeseriesArray.load("testdata/fcIfC3AccountingTable",
meta2["index_keynames"],
datatypes=meta2["value_keynames"],
filterkeys=filterkeys,
index_pattern=None,
matchtype="and")
for key in tsa.keys():
assert key[0] == "fca-sr2-8gb-21"
# loading only Timeseries matching this index key
filterkeys = {"hostname": None, "ifDescr": "port-channel 1"}
tsa = TimeseriesArray.load("testdata/fcIfC3AccountingTable",
meta2["index_keynames"],
datatypes=meta2["value_keynames"],
filterkeys=filterkeys,
index_pattern=None,
matchtype="and")
for key in tsa.keys():
assert key[1] == "port-channel 1"
# loading only Timeseries matching this index key using or
filterkeys = {
"hostname": "fca-sr2-8gb-21",
"ifDescr": "port-channel 1"
}
tsa = TimeseriesArray.load("testdata/fcIfC3AccountingTable",
meta2["index_keynames"],
datatypes=meta2["value_keynames"],
filterkeys=filterkeys,
index_pattern=None,
matchtype="or")
for key in tsa.keys():
assert key[1] == "port-channel 1" or key[0] == "fca-sr2-8gb-21"
# using regular expression to filter some index_keys
tsa = TimeseriesArray.load("testdata/fcIfC3AccountingTable",
meta2["index_keynames"],
datatypes=meta2["value_keynames"],
filterkeys=None,
index_pattern="(.*)fca-(.*)",
matchtype="and")
for key in tsa.keys():
assert "fca-" in str(key)
def setUp(self):
self.basedir = "/var/rrd"
self.datestring = "2015-11-30"
self.testfile = "testdata/fcIfC3AccountingTable"
index_keys = (u"hostname", ) # the unicode is important
self.app = TimeseriesArray.load("testdata/",
meta["index_keynames"],
datatypes=meta["value_keynames"])
def load_tsa(self, filterkeys=None, index_pattern=None):
"""
caching version to load_tsa_raw
if never called, get ts from load_tsa_raw, and afterwards dump_tsa
on every consecutive call read from cached version
parameters:
datestring <str>
filterkeys <tuple> or None default None
index_pattern <str> or None default None
HINT:
use delete_caches to delete all precalculated files
use setup to define some sort of timedelta to use
returns
<TimeseriesArray> object read from cachefile or from raw data
"""
cachefilename = os.path.join(self.cachedir, TimeseriesArray.get_dumpfilename(self.index_keynames))
def fallback():
"""
fallback method to use, if reading from cache data is not possible
"""
tsa = self.load_tsa_raw()
tsa.dump(self.cachedir) # save full data
# read the data afterwards to make sure there is no problem,
# without setting tsa.cache = true this is only the tsa structure no timeseries data
tsa = TimeseriesArray.load(self.cachedir, self.index_keynames, filterkeys=filterkeys, index_pattern=index_pattern, datatypes=self.datatypes)
return tsa
if not os.path.isfile(cachefilename):
logging.info("cachefile %s does not exist, fallback read from raw data file", cachefilename)
return fallback()
logging.debug("loading stored TimeseriesArray object file %s", cachefilename)
try:
tsa = TimeseriesArray.load(self.cachedir, self.index_keynames, filterkeys=filterkeys, index_pattern=index_pattern, datatypes=self.datatypes)
return tsa
except IOError:
logging.error("IOError while reading from %s, using fallback", cachefilename)
os.unlink(cachefilename)
return fallback()
except EOFError:
logging.error("EOFError while reading from %s, using fallback", cachefilename)
os.unlink(cachefilename)
return fallback()
def test_dump(self):
print("testing dump, get_ts_dumpfilename, __eq__")
testdir = "testdata/tsa_testdump"
if not os.path.isdir(testdir):
os.mkdir(testdir)
tsa = TimeseriesArray.load("testdata/fcIfC3AccountingTable",
meta2["index_keynames"],
datatypes=meta2["value_keynames"],
filterkeys=None,
index_pattern=None,
matchtype="and")
tsa.dump(testdir, overwrite=True)
tsa1 = TimeseriesArray.load(testdir,
meta2["index_keynames"],
datatypes=meta2["value_keynames"],
filterkeys=None,
index_pattern=None,
matchtype="and")
assert tsa == tsa1
def test_add_calc_col_single(self):
print("testing load, cache, add_calc_col_single, slice")
tsa = TimeseriesArray.load("testdata/",
meta["index_keynames"],
datatypes=meta["value_keynames"])
tsa.cache = True
tsa.add_calc_col_single("bytes_received", "kbytes_received",
lambda a: a / 8)
tsa1 = tsa.slice(("bytes_received", "kbytes_received"))
print(tsa1[('nagios.tilak.cc', )])
def fallback():
"""
fallback method to use, if reading from cache data is not possible
"""
tsa = self.load_tsa_raw()
tsa.dump(self.cachedir) # save full data
# read the data afterwards to make sure there is no problem,
# without setting tsa.cache = true this is only the tsa structure no timeseries data
tsa = TimeseriesArray.load(self.cachedir, self.index_keynames, filterkeys=filterkeys, index_pattern=index_pattern, datatypes=self.datatypes)
return tsa
def test_remove_col(self):
print("testing remove_col, load, cache")
tsa = TimeseriesArray.load("testdata/",
meta["index_keynames"],
datatypes=meta["value_keynames"])
tsa.cache = True
tsa.remove_col("uptime")
assert "uptime" not in tsa.value_keynames
try:
tsa.remove_col("unknown")
except KeyError:
# should raise this exception
pass
def load_tsa_raw(self):
"""
read data from raw input files and return TimeseriesArray object
parameters:
datestring <str> isodate representation of date like 2015-12-31
timedelta <int> amount second to correct raw input timestamps
returns:
<TimeseriesArray> object wich holds all data of this day
"""
tsa = TimeseriesArray(self.index_keynames, self.value_keynames, datatypes=self.datatypes)
for rowdict in self.__read_raw_dict():
try:
tsa.add(rowdict)
except ValueError as exc:
logging.exception(exc)
logging.error("ValueError by adding this data to TimeseriesArray: %s", rowdict)
raise exc
except AssertionError as exc:
logging.exception(exc)
logging.error("AssertionError by adding this data to TimeseriesArray: %s", rowdict)
raise exc
return tsa
def test_convert(self):
print("testing load, cache, convert, slice")
tsa = TimeseriesArray.load("testdata/",
meta["index_keynames"],
datatypes=meta["value_keynames"])
tsa.cache = True # thats crucial, otherwise every timeseries will alwys be read from disk
tsa.convert(colname="uptime",
datatype="persecond",
newcolname="uptime_persecond")
tsa.convert("uptime", "derive", "uptime_derive")
tsa.convert("uptime", "percent", "uptime_percent")
tsa.convert("com_select", "counter32", "com_select_counter32")
tsa.convert("com_select", "counter64", "com_select_counter64")
tsa.convert("com_select", "gauge32", "com_select_gauge32")
tsa.convert("com_select", "counterreset", "com_select_counterreset")
assert all(newcol in tsa.value_keynames
for newcol in ("uptime_persecond", "uptime_derive",
"uptime_percent", "com_select_counter32",
"com_select_counter64", "com_select_gauge32",
"com_select_counterreset"))
tsa1 = tsa.slice(("uptime", "uptime_persecond", "com_select_gauge32"))
print(tsa1[('nagios.tilak.cc', )])