def load_quantile(self, datestring): """ retuns quantile for this specific tsa, either load cache version, or recreate from tsa parameters: datestring <str> returns: <QuantileArray> """ cachedir = self.__get_cachedir(datestring) cachefilename = os.path.join(cachedir, "quantile.json") quantile_array = None if os.path.isfile(cachefilename): quantile_array = QuantileArray.load(open(cachefilename, "rb")) else: logging.info("cachefile %s does not exist, fallback read from tsa archive", cachefilename) tsa = self.load_tsa(datestring) tsa.cache = True # to enable in memory caching of timeseries # huge performance improvement, from 500s to 70s tsastats = self.load_tsastats(datestring) quantile_array = QuantileArray(tsa, tsastats) quantile_array.dump(open(cachefilename, "wb")) return quantile_array
def import_tsa(self, datestring, tsa): """ store tsa given in parameter in global_cache to make the data available usually this could be modfied existing tsa extended by some keys, or filtered or ... the structure has to be predefined in meta data the tsa can afterwards be accessed via normal frontends (web, api) parameters: tsa <TimeseriesArrayLazy> object """ assert self.__index_keynames == tsa.index_keynames assert self.__value_keynames == tuple(tsa.value_keynames) cachedir = self.__get_cachedir(datestring) cachefilename = os.path.join(cachedir, TimeseriesArrayLazy.get_dumpfilename(tsa.index_keynames)) if not os.path.isfile(cachefilename): tsa.dump_split(cachedir) tsastats = TimeseriesArrayStats(tsa) tsastats.dump(cachedir) qantile = QuantileArray(tsa, tsastats) q_cachefilename = os.path.join(cachedir, "quantile.json") qantile.dump(open(q_cachefilename, "wb")) else: raise StandardError("TSA Archive %s exists already in cache" % cachefilename)
def fallback(): """ fallback method to use, if reading from cache data is not possible """ tsa = self.load_tsa_raw(datestring, timedelta) tsa.dump_split(cachedir) # save full data # read the data afterwards to make sure there is no problem, if validate is True: tsa = TimeseriesArrayLazy.load_split(cachedir, self.__index_keynames, filterkeys=filterkeys, index_pattern=index_pattern, datatypes=self.__datatypes) # also generate TSASTATS and dump to cache directory tsastats = TimeseriesArrayStats(tsa) # generate full Stats tsastats.dump(cachedir) # save # and at last but not least quantile qantile = QuantileArray(tsa, tsastats) cachefilename = os.path.join(cachedir, "quantile.json") qantile.dump(open(cachefilename, "wb")) # finally return tsa return tsa