def get_quantile_web(self, args): """ return exported QuantileArray json formatted, special version for use in webpages to render with tablesorter in difference to get_quantile the value_keyname has to be given ex: Datalogger/get_quantile/{projectname}/{tablename}/{datestring} [ dict of index_keys : dict of quantile, list of index_keys, list of value_names, ] returns: json(quantile_dict) """ project, tablename, datestring, value_keyname = args[:4] datalogger = DataLogger(basedir, project, tablename) qa = datalogger.load_quantile(datestring) ret_data = [] # build header ret_data.append(list(datalogger.index_keynames) + ["Q0", "Q1", "Q2", "Q3", "Q4"]) # data part for k, v in qa[value_keyname].quantile.items(): ret_data.append(list(k) + v.values()) return json.dumps(ret_data)
def get_quantile(self, project, tablename, datestring, args): """ return exported QuantileArray json formatted ex: Datalogger/get_quantile/{projectname}/{tablename}/{datestring} [ dict of index_keys : dict of quantile, list of index_keys, list of value_names, ] returns: json(quantile_dict) """ logging.info("optional arguments received: %s", args) datalogger = DataLogger(basedir, project, tablename) quantile = datalogger.load_quantile(datestring) if len(args) > 0: value_keyname = args[0] ret_data = [] # build header ret_data.append(list(datalogger.index_keynames) + ["Q0", "Q1", "Q2", "Q3", "Q4"]) # data part for k, v in quantile[value_keyname].quantile.items(): ret_data.append(list(k) + v.values()) return json.dumps(ret_data) return quantile.to_json()
def gen_caches(project, tablename, datestring): datalogger = DataLogger(basedir, project, tablename) caches = datalogger.get_caches(datestring) suffix = "%s/%s/%s\t" % (datestring, project, tablename) data = None if caches["tsa"]["raw"] is None: if len(caches["tsa"]["keys"]) == 0: logging.info("%s RAW Data not availabale maybe archived, tsa exists already", suffix) else: logging.debug("%s RAW Data is missing, no tsa archive exists", suffix) else: if len(caches["tsa"]["keys"]) == 0: logging.info("%s TSA Archive missing, calling get_tsa and load_tsastats", suffix) data = datalogger.load_tsa(datestring) else: if len(caches["tsastat"]["keys"]) == 0: logging.info("%s TSASTAT Archive missing, calling load_tsastats", suffix) data = datalogger.load_tsastats(datestring) else: if len(caches["ts"]["keys"]) == 0: logging.info("%s there are no ts archives, something went wrong, or tsa is completely empty, calling load_tsastats", suffix) data = datalogger.load_tsastats(datestring) else: logging.debug("%s All fine", suffix) if caches["quantile"]["exists"] is not True: logging.info("%s Quantile archive is missing, calling load_quantile", suffix) data = datalogger.load_quantile(datestring) del data del caches del datalogger
def get_quantile(self, args): """ return exported QuantileArray json formatted ex: Datalogger/get_quantile/{projectname}/{tablename}/{datestring} [ dict of index_keys : dict of quantile, list of index_keys, list of value_names, ] returns: json(quantile_dict) """ project, tablename, datestring = args[:3] datalogger = DataLogger(basedir, project, tablename) quantile = datalogger.load_quantile(datestring) return quantile.to_json()
class Test(unittest.TestCase): def setUp(self): self.basedir = "/var/rrd" self.datestring = "2015-11-30" self.project = DataLogger.get_projects(self.basedir)[0] self.tablename = DataLogger.get_tablenames(self.basedir, self.project)[0] self.datalogger = DataLogger(self.basedir, self.project, self.tablename) def test_simple(self): self.assertTrue(self.datalogger.project == self.project) self.assertTrue(self.datalogger.tablename == self.tablename) self.assertTrue(isinstance(self.datalogger.delimiter, basestring)) self.assertTrue(isinstance(self.datalogger.ts_keyname, basestring)) self.assertTrue(isinstance(self.datalogger.headers, tuple)) self.assertTrue(isinstance(self.datalogger.value_keynames, tuple)) self.assertTrue(all((keyname in self.datalogger.headers for keyname in self.datalogger.value_keynames))) self.assertTrue(isinstance(self.datalogger.index_keynames, tuple)) self.assertTrue(all((keyname in self.datalogger.headers for keyname in self.datalogger.index_keynames))) self.assertTrue(isinstance(self.datalogger.blacklist, tuple)) self.assertTrue(all((keyname in self.datalogger.headers for keyname in self.datalogger.blacklist))) self.assertTrue(isinstance(self.datalogger.raw_basedir, basestring)) self.assertTrue(os.path.exists(self.datalogger.raw_basedir)) self.assertTrue(os.path.isdir(self.datalogger.raw_basedir)) self.assertTrue(isinstance(self.datalogger.global_cachedir, basestring)) self.assertTrue(os.path.exists(self.datalogger.global_cachedir)) self.assertTrue(os.path.isdir(self.datalogger.global_cachedir)) # meta is something like this # {u'ts_keyname': u'ts', # 'stat_func_names': [u'count', ... ], # u'interval': 300, # u'blacklist': [], # u'headers': [u'ts', u'http_host', ... ], # u'delimiter': u'\t', # u'value_keynames': { # u'actconn': u'asis', # u'hits': u'asis', # ... # }, # u'index_keynames': [u'http_host']} self.assertTrue(self.datalogger.meta["headers"] == list(self.datalogger.headers)) self.assertTrue(self.datalogger.meta["value_keynames"].keys() == list(self.datalogger.value_keynames)) self.assertTrue(self.datalogger.meta["index_keynames"] == list(self.datalogger.index_keynames)) self.assertTrue(self.datalogger.meta["blacklist"] == list(self.datalogger.blacklist)) self.assertTrue(self.datalogger.meta["delimiter"] == self.datalogger.delimiter) self.assertTrue(self.datalogger.meta["ts_keyname"] == self.datalogger.ts_keyname) self.assertTrue(isinstance(self.datalogger.meta["stat_func_names"], list)) def test_statics(self): self.assertTrue(isinstance(DataLogger.get_user(self.basedir), basestring)) self.assertTrue(isinstance(DataLogger.get_group(self.basedir), basestring)) self.assertTrue(isinstance(DataLogger.get_yesterday_datestring(), basestring)) lbd = DataLogger.get_last_business_day_datestring() self.assertTrue(isinstance(DataLogger.get_last_business_day_datestring(), basestring)) self.assertTrue(isinstance(DataLogger.datestring_to_date(lbd), datetime.date)) for datestring in DataLogger.datewalker("2016-01-01", "2016-02-29"): self.assertTrue(isinstance(datestring, basestring)) for datestring in DataLogger.monthwalker("2016-02"): self.assertTrue(isinstance(datestring, basestring)) self.assertEqual(list(DataLogger.monthwalker("2016-02"))[-1], "2016-02-29") self.assertTrue(isinstance(DataLogger.get_ts_for_datestring("2016-01-01"), tuple)) self.assertTrue(isinstance(DataLogger.get_ts_for_datestring("2016-01-01")[0], float)) self.assertTrue(isinstance(DataLogger.get_ts_for_datestring("2016-01-01")[1], float)) def test_data(self): self.datalogger.load_tsa(self.datestring) self.datalogger.load_tsastats(self.datestring) self.datalogger.load_correlationmatrix(self.datestring) self.datalogger.load_quantile(self.datestring)