def test_load(self):
print("testing load, get_ts_filename, filtermatch, get_dumpfilename")
tsa = TimeseriesArray.load("testdata/fcIfC3AccountingTable",
meta2["index_keynames"],
datatypes=meta2["value_keynames"],
filterkeys=None,
index_pattern=None,
matchtype="and")
#for key in tsa.keys():
# print(key)
# loading only Timeseries matching this index key
filterkeys = {"hostname": "fca-sr2-8gb-21", "ifDescr": None}
tsa = TimeseriesArray.load("testdata/fcIfC3AccountingTable",
meta2["index_keynames"],
datatypes=meta2["value_keynames"],
filterkeys=filterkeys,
index_pattern=None,
matchtype="and")
for key in tsa.keys():
assert key[0] == "fca-sr2-8gb-21"
# loading only Timeseries matching this index key
filterkeys = {"hostname": None, "ifDescr": "port-channel 1"}
tsa = TimeseriesArray.load("testdata/fcIfC3AccountingTable",
meta2["index_keynames"],
datatypes=meta2["value_keynames"],
filterkeys=filterkeys,
index_pattern=None,
matchtype="and")
for key in tsa.keys():
assert key[1] == "port-channel 1"
# loading only Timeseries matching this index key using or
filterkeys = {
"hostname": "fca-sr2-8gb-21",
"ifDescr": "port-channel 1"
}
tsa = TimeseriesArray.load("testdata/fcIfC3AccountingTable",
meta2["index_keynames"],
datatypes=meta2["value_keynames"],
filterkeys=filterkeys,
index_pattern=None,
matchtype="or")
for key in tsa.keys():
assert key[1] == "port-channel 1" or key[0] == "fca-sr2-8gb-21"
# using regular expression to filter some index_keys
tsa = TimeseriesArray.load("testdata/fcIfC3AccountingTable",
meta2["index_keynames"],
datatypes=meta2["value_keynames"],
filterkeys=None,
index_pattern="(.*)fca-(.*)",
matchtype="and")
for key in tsa.keys():
assert "fca-" in str(key)
def setUp(self):
self.basedir = "/var/rrd"
self.datestring = "2015-11-30"
self.testfile = "testdata/fcIfC3AccountingTable"
index_keys = (u"hostname", ) # the unicode is important
self.app = TimeseriesArray.load("testdata/",
meta["index_keynames"],
datatypes=meta["value_keynames"])
def test_dump(self):
print("testing dump, get_ts_dumpfilename, __eq__")
testdir = "testdata/tsa_testdump"
if not os.path.isdir(testdir):
os.mkdir(testdir)
tsa = TimeseriesArray.load("testdata/fcIfC3AccountingTable",
meta2["index_keynames"],
datatypes=meta2["value_keynames"],
filterkeys=None,
index_pattern=None,
matchtype="and")
tsa.dump(testdir, overwrite=True)
tsa1 = TimeseriesArray.load(testdir,
meta2["index_keynames"],
datatypes=meta2["value_keynames"],
filterkeys=None,
index_pattern=None,
matchtype="and")
assert tsa == tsa1
def test_add_calc_col_single(self):
print("testing load, cache, add_calc_col_single, slice")
tsa = TimeseriesArray.load("testdata/",
meta["index_keynames"],
datatypes=meta["value_keynames"])
tsa.cache = True
tsa.add_calc_col_single("bytes_received", "kbytes_received",
lambda a: a / 8)
tsa1 = tsa.slice(("bytes_received", "kbytes_received"))
print(tsa1[('nagios.tilak.cc', )])
def fallback():
"""
fallback method to use, if reading from cache data is not possible
"""
tsa = self.load_tsa_raw()
tsa.dump(self.cachedir) # save full data
# read the data afterwards to make sure there is no problem,
# without setting tsa.cache = true this is only the tsa structure no timeseries data
tsa = TimeseriesArray.load(self.cachedir, self.index_keynames, filterkeys=filterkeys, index_pattern=index_pattern, datatypes=self.datatypes)
return tsa
def test_remove_col(self):
print("testing remove_col, load, cache")
tsa = TimeseriesArray.load("testdata/",
meta["index_keynames"],
datatypes=meta["value_keynames"])
tsa.cache = True
tsa.remove_col("uptime")
assert "uptime" not in tsa.value_keynames
try:
tsa.remove_col("unknown")
except KeyError:
# should raise this exception
pass
def load_tsa(self, filterkeys=None, index_pattern=None):
"""
caching version to load_tsa_raw
if never called, get ts from load_tsa_raw, and afterwards dump_tsa
on every consecutive call read from cached version
parameters:
datestring <str>
filterkeys <tuple> or None default None
index_pattern <str> or None default None
HINT:
use delete_caches to delete all precalculated files
use setup to define some sort of timedelta to use
returns
<TimeseriesArray> object read from cachefile or from raw data
"""
cachefilename = os.path.join(self.cachedir, TimeseriesArray.get_dumpfilename(self.index_keynames))
def fallback():
"""
fallback method to use, if reading from cache data is not possible
"""
tsa = self.load_tsa_raw()
tsa.dump(self.cachedir) # save full data
# read the data afterwards to make sure there is no problem,
# without setting tsa.cache = true this is only the tsa structure no timeseries data
tsa = TimeseriesArray.load(self.cachedir, self.index_keynames, filterkeys=filterkeys, index_pattern=index_pattern, datatypes=self.datatypes)
return tsa
if not os.path.isfile(cachefilename):
logging.info("cachefile %s does not exist, fallback read from raw data file", cachefilename)
return fallback()
logging.debug("loading stored TimeseriesArray object file %s", cachefilename)
try:
tsa = TimeseriesArray.load(self.cachedir, self.index_keynames, filterkeys=filterkeys, index_pattern=index_pattern, datatypes=self.datatypes)
return tsa
except IOError:
logging.error("IOError while reading from %s, using fallback", cachefilename)
os.unlink(cachefilename)
return fallback()
except EOFError:
logging.error("EOFError while reading from %s, using fallback", cachefilename)
os.unlink(cachefilename)
return fallback()
def test_convert(self):
print("testing load, cache, convert, slice")
tsa = TimeseriesArray.load("testdata/",
meta["index_keynames"],
datatypes=meta["value_keynames"])
tsa.cache = True # thats crucial, otherwise every timeseries will alwys be read from disk
tsa.convert(colname="uptime",
datatype="persecond",
newcolname="uptime_persecond")
tsa.convert("uptime", "derive", "uptime_derive")
tsa.convert("uptime", "percent", "uptime_percent")
tsa.convert("com_select", "counter32", "com_select_counter32")
tsa.convert("com_select", "counter64", "com_select_counter64")
tsa.convert("com_select", "gauge32", "com_select_gauge32")
tsa.convert("com_select", "counterreset", "com_select_counterreset")
assert all(newcol in tsa.value_keynames
for newcol in ("uptime_persecond", "uptime_derive",
"uptime_percent", "com_select_counter32",
"com_select_counter64", "com_select_gauge32",
"com_select_counterreset"))
tsa1 = tsa.slice(("uptime", "uptime_persecond", "com_select_gauge32"))
print(tsa1[('nagios.tilak.cc', )])