def tsa_group_by(tsa, datestring, index_keynames, group_func, interval): """ group given tsa by subkeys, and use group_func to aggregate data the individual timeseries are automatically grouped by timestamp and interval defined in configuration parameters: tsa <TimeseriesArray> datestring <str> datestring to use to aggregate data TODO: get this from tsa subkey <tuple> could also be empty, to aggregate everything group_func <func> like lambda a, b : (a + b) / 2 to get averages interval <int> interval in seconds the timeseries values should appear returns: <TimeseriesArray> """ # intermediated tsa tsa2 = TimeseriesArray(index_keynames=index_keynames, value_keynames=tsa.value_keynames, ts_key=tsa.ts_key, datatypes=tsa.datatypes) start_ts, _ = DataLogger.get_ts_for_datestring(datestring) ts_keyname = tsa.ts_key for data in tsa.export(): # align timestamp nearest_slot = round((data[ts_keyname] - start_ts) / interval) data[ts_keyname] = int(start_ts + nearest_slot * interval) tsa2.add(data, group_func) return tsa2
def test_export(self): print("testing export, add") tsa = TimeseriesArray(meta["index_keynames"], meta["value_keynames"], "ts", datatypes=meta["value_keynames"]) for entry in self.app.export(): tsa.add(entry) assert tsa == self.app
def test_load(self): print("testing load, get_ts_filename, filtermatch, get_dumpfilename") tsa = TimeseriesArray.load("testdata/fcIfC3AccountingTable", meta2["index_keynames"], datatypes=meta2["value_keynames"], filterkeys=None, index_pattern=None, matchtype="and") #for key in tsa.keys(): # print(key) # loading only Timeseries matching this index key filterkeys = {"hostname": "fca-sr2-8gb-21", "ifDescr": None} tsa = TimeseriesArray.load("testdata/fcIfC3AccountingTable", meta2["index_keynames"], datatypes=meta2["value_keynames"], filterkeys=filterkeys, index_pattern=None, matchtype="and") for key in tsa.keys(): assert key[0] == "fca-sr2-8gb-21" # loading only Timeseries matching this index key filterkeys = {"hostname": None, "ifDescr": "port-channel 1"} tsa = TimeseriesArray.load("testdata/fcIfC3AccountingTable", meta2["index_keynames"], datatypes=meta2["value_keynames"], filterkeys=filterkeys, index_pattern=None, matchtype="and") for key in tsa.keys(): assert key[1] == "port-channel 1" # loading only Timeseries matching this index key using or filterkeys = { "hostname": "fca-sr2-8gb-21", "ifDescr": "port-channel 1" } tsa = TimeseriesArray.load("testdata/fcIfC3AccountingTable", meta2["index_keynames"], datatypes=meta2["value_keynames"], filterkeys=filterkeys, index_pattern=None, matchtype="or") for key in tsa.keys(): assert key[1] == "port-channel 1" or key[0] == "fca-sr2-8gb-21" # using regular expression to filter some index_keys tsa = TimeseriesArray.load("testdata/fcIfC3AccountingTable", meta2["index_keynames"], datatypes=meta2["value_keynames"], filterkeys=None, index_pattern="(.*)fca-(.*)", matchtype="and") for key in tsa.keys(): assert "fca-" in str(key)
def setUp(self): self.basedir = "/var/rrd" self.datestring = "2015-11-30" self.testfile = "testdata/fcIfC3AccountingTable" index_keys = (u"hostname", ) # the unicode is important self.app = TimeseriesArray.load("testdata/", meta["index_keynames"], datatypes=meta["value_keynames"])
def load_tsa(self, filterkeys=None, index_pattern=None): """ caching version to load_tsa_raw if never called, get ts from load_tsa_raw, and afterwards dump_tsa on every consecutive call read from cached version parameters: datestring <str> filterkeys <tuple> or None default None index_pattern <str> or None default None HINT: use delete_caches to delete all precalculated files use setup to define some sort of timedelta to use returns <TimeseriesArray> object read from cachefile or from raw data """ cachefilename = os.path.join(self.cachedir, TimeseriesArray.get_dumpfilename(self.index_keynames)) def fallback(): """ fallback method to use, if reading from cache data is not possible """ tsa = self.load_tsa_raw() tsa.dump(self.cachedir) # save full data # read the data afterwards to make sure there is no problem, # without setting tsa.cache = true this is only the tsa structure no timeseries data tsa = TimeseriesArray.load(self.cachedir, self.index_keynames, filterkeys=filterkeys, index_pattern=index_pattern, datatypes=self.datatypes) return tsa if not os.path.isfile(cachefilename): logging.info("cachefile %s does not exist, fallback read from raw data file", cachefilename) return fallback() logging.debug("loading stored TimeseriesArray object file %s", cachefilename) try: tsa = TimeseriesArray.load(self.cachedir, self.index_keynames, filterkeys=filterkeys, index_pattern=index_pattern, datatypes=self.datatypes) return tsa except IOError: logging.error("IOError while reading from %s, using fallback", cachefilename) os.unlink(cachefilename) return fallback() except EOFError: logging.error("EOFError while reading from %s, using fallback", cachefilename) os.unlink(cachefilename) return fallback()
def test_dump(self): print("testing dump, get_ts_dumpfilename, __eq__") testdir = "testdata/tsa_testdump" if not os.path.isdir(testdir): os.mkdir(testdir) tsa = TimeseriesArray.load("testdata/fcIfC3AccountingTable", meta2["index_keynames"], datatypes=meta2["value_keynames"], filterkeys=None, index_pattern=None, matchtype="and") tsa.dump(testdir, overwrite=True) tsa1 = TimeseriesArray.load(testdir, meta2["index_keynames"], datatypes=meta2["value_keynames"], filterkeys=None, index_pattern=None, matchtype="and") assert tsa == tsa1
def test_add_calc_col_single(self): print("testing load, cache, add_calc_col_single, slice") tsa = TimeseriesArray.load("testdata/", meta["index_keynames"], datatypes=meta["value_keynames"]) tsa.cache = True tsa.add_calc_col_single("bytes_received", "kbytes_received", lambda a: a / 8) tsa1 = tsa.slice(("bytes_received", "kbytes_received")) print(tsa1[('nagios.tilak.cc', )])
def fallback(): """ fallback method to use, if reading from cache data is not possible """ tsa = self.load_tsa_raw() tsa.dump(self.cachedir) # save full data # read the data afterwards to make sure there is no problem, # without setting tsa.cache = true this is only the tsa structure no timeseries data tsa = TimeseriesArray.load(self.cachedir, self.index_keynames, filterkeys=filterkeys, index_pattern=index_pattern, datatypes=self.datatypes) return tsa
def test_remove_col(self): print("testing remove_col, load, cache") tsa = TimeseriesArray.load("testdata/", meta["index_keynames"], datatypes=meta["value_keynames"]) tsa.cache = True tsa.remove_col("uptime") assert "uptime" not in tsa.value_keynames try: tsa.remove_col("unknown") except KeyError: # should raise this exception pass
def load_tsa_raw(self): """ read data from raw input files and return TimeseriesArray object parameters: datestring <str> isodate representation of date like 2015-12-31 timedelta <int> amount second to correct raw input timestamps returns: <TimeseriesArray> object wich holds all data of this day """ tsa = TimeseriesArray(self.index_keynames, self.value_keynames, datatypes=self.datatypes) for rowdict in self.__read_raw_dict(): try: tsa.add(rowdict) except ValueError as exc: logging.exception(exc) logging.error("ValueError by adding this data to TimeseriesArray: %s", rowdict) raise exc except AssertionError as exc: logging.exception(exc) logging.error("AssertionError by adding this data to TimeseriesArray: %s", rowdict) raise exc return tsa
def test_convert(self): print("testing load, cache, convert, slice") tsa = TimeseriesArray.load("testdata/", meta["index_keynames"], datatypes=meta["value_keynames"]) tsa.cache = True # thats crucial, otherwise every timeseries will alwys be read from disk tsa.convert(colname="uptime", datatype="persecond", newcolname="uptime_persecond") tsa.convert("uptime", "derive", "uptime_derive") tsa.convert("uptime", "percent", "uptime_percent") tsa.convert("com_select", "counter32", "com_select_counter32") tsa.convert("com_select", "counter64", "com_select_counter64") tsa.convert("com_select", "gauge32", "com_select_gauge32") tsa.convert("com_select", "counterreset", "com_select_counterreset") assert all(newcol in tsa.value_keynames for newcol in ("uptime_persecond", "uptime_derive", "uptime_percent", "com_select_counter32", "com_select_counter64", "com_select_gauge32", "com_select_counterreset")) tsa1 = tsa.slice(("uptime", "uptime_persecond", "com_select_gauge32")) print(tsa1[('nagios.tilak.cc', )])