def load_quantile(self, datestring):
"""
retuns quantile for this specific tsa, either load cache version,
or recreate from tsa
parameters:
datestring <str>
returns:
<QuantileArray>
"""
cachedir = self.__get_cachedir(datestring)
cachefilename = os.path.join(cachedir, "quantile.json")
quantile_array = None
if os.path.isfile(cachefilename):
quantile_array = QuantileArray.load(open(cachefilename, "rb"))
else:
logging.info("cachefile %s does not exist, fallback read from tsa archive", cachefilename)
tsa = self.load_tsa(datestring)
tsa.cache = True # to enable in memory caching of timeseries
# huge performance improvement, from 500s to 70s
tsastats = self.load_tsastats(datestring)
quantile_array = QuantileArray(tsa, tsastats)
quantile_array.dump(open(cachefilename, "wb"))
return quantile_array
def import_tsa(self, datestring, tsa):
"""
store tsa given in parameter in global_cache to make the data available
usually this could be modfied existing tsa extended by some keys, or filtered or ...
the structure has to be predefined in meta data
the tsa can afterwards be accessed via normal frontends (web, api)
parameters:
tsa <TimeseriesArrayLazy> object
"""
assert self.__index_keynames == tsa.index_keynames
assert self.__value_keynames == tuple(tsa.value_keynames)
cachedir = self.__get_cachedir(datestring)
cachefilename = os.path.join(cachedir, TimeseriesArrayLazy.get_dumpfilename(tsa.index_keynames))
if not os.path.isfile(cachefilename):
tsa.dump_split(cachedir)
tsastats = TimeseriesArrayStats(tsa)
tsastats.dump(cachedir)
qantile = QuantileArray(tsa, tsastats)
q_cachefilename = os.path.join(cachedir, "quantile.json")
qantile.dump(open(q_cachefilename, "wb"))
else:
raise StandardError("TSA Archive %s exists already in cache" % cachefilename)
def fallback():
"""
fallback method to use, if reading from cache data is not possible
"""
tsa = self.load_tsa_raw(datestring, timedelta)
tsa.dump_split(cachedir) # save full data
# read the data afterwards to make sure there is no problem,
if validate is True:
tsa = TimeseriesArrayLazy.load_split(cachedir, self.__index_keynames, filterkeys=filterkeys, index_pattern=index_pattern, datatypes=self.__datatypes)
# also generate TSASTATS and dump to cache directory
tsastats = TimeseriesArrayStats(tsa) # generate full Stats
tsastats.dump(cachedir) # save
# and at last but not least quantile
qantile = QuantileArray(tsa, tsastats)
cachefilename = os.path.join(cachedir, "quantile.json")
qantile.dump(open(cachefilename, "wb"))
# finally return tsa
return tsa
