def import_tsa(self, datestring, tsa): """ store tsa given in parameter in global_cache to make the data available usually this could be modfied existing tsa extended by some keys, or filtered or ... the structure has to be predefined in meta data the tsa can afterwards be accessed via normal frontends (web, api) parameters: tsa <TimeseriesArrayLazy> object """ assert self.__index_keynames == tsa.index_keynames assert self.__value_keynames == tuple(tsa.value_keynames) cachedir = self.__get_cachedir(datestring) cachefilename = os.path.join(cachedir, TimeseriesArrayLazy.get_dumpfilename(tsa.index_keynames)) if not os.path.isfile(cachefilename): tsa.dump_split(cachedir) tsastats = TimeseriesArrayStats(tsa) tsastats.dump(cachedir) qantile = QuantileArray(tsa, tsastats) q_cachefilename = os.path.join(cachedir, "quantile.json") qantile.dump(open(q_cachefilename, "wb")) else: raise StandardError("TSA Archive %s exists already in cache" % cachefilename)
def load_tsa(self, datestring, filterkeys=None, index_pattern=None, timedelta=0, cleancache=False, validate=False): """ caching version to load_tsa_raw if never called, get ts from load_tsa_raw, and afterwards dump_tsa on every consecutive call read from cached version use cleancache to remove caches parameters: datestring <str> filterkeys <tuple> or None default None index_pattern <str> or None default None timedelta <int> default 0 cleancache <bool> default False validate <bool> if data is read from raw, dump it after initail read, and reread it afterwards to make sure the stored tsa is OK thats an performance issue returns <TimeseriesArrayLazy> object read from cachefile or from raw data """ try: assert not_today(datestring) except AssertionError: raise DataLoggerLiveDataError("Reading from live data is not allowed") cachedir = self.__get_cachedir(datestring) cachefilename = os.path.join(cachedir, TimeseriesArrayLazy.get_dumpfilename(self.__index_keynames)) def fallback(): """ fallback method to use, if reading from cache data is not possible """ tsa = self.load_tsa_raw(datestring, timedelta) tsa.dump_split(cachedir) # save full data # read the data afterwards to make sure there is no problem, if validate is True: tsa = TimeseriesArrayLazy.load_split(cachedir, self.__index_keynames, filterkeys=filterkeys, index_pattern=index_pattern, datatypes=self.__datatypes) # also generate TSASTATS and dump to cache directory tsastats = TimeseriesArrayStats(tsa) # generate full Stats tsastats.dump(cachedir) # save # and at last but not least quantile qantile = QuantileArray(tsa, tsastats) cachefilename = os.path.join(cachedir, "quantile.json") qantile.dump(open(cachefilename, "wb")) # finally return tsa return tsa if not os.path.isfile(cachefilename): logging.info("cachefile %s does not exist, fallback read from raw data file", cachefilename) return fallback() if (os.path.isfile(cachefilename)) and (cleancache == True): logging.info("deleting cachefile %s and read from raw data file", cachefilename) os.unlink(cachefilename) return fallback() logging.debug("loading stored TimeseriesArrayLazy object file %s", cachefilename) try: tsa = TimeseriesArrayLazy.load_split(cachedir, self.__index_keynames, filterkeys=filterkeys, index_pattern=index_pattern, datatypes=self.__datatypes) return tsa except IOError: logging.error("IOError while reading from %s, using fallback", cachefilename) os.unlink(cachefilename) return fallback() except EOFError: logging.error("EOFError while reading from %s, using fallback", cachefilename) os.unlink(cachefilename) return fallback()