def fallback():
"""
fallback method to use, if reading from cache data is not possible
"""
tsa = self.load_tsa_raw(datestring, timedelta)
tsa.dump_split(cachedir) # save full data
# read the data afterwards to make sure there is no problem,
if validate is True:
tsa = TimeseriesArrayLazy.load_split(cachedir, self.__index_keynames, filterkeys=filterkeys, index_pattern=index_pattern, datatypes=self.__datatypes)
# also generate TSASTATS and dump to cache directory
tsastats = TimeseriesArrayStats(tsa) # generate full Stats
tsastats.dump(cachedir) # save
# and at last but not least quantile
qantile = QuantileArray(tsa, tsastats)
cachefilename = os.path.join(cachedir, "quantile.json")
qantile.dump(open(cachefilename, "wb"))
# finally return tsa
return tsa
def load_tsa(self, datestring, filterkeys=None, index_pattern=None, timedelta=0, cleancache=False, validate=False):
"""
caching version to load_tsa_raw
if never called, get ts from load_tsa_raw, and afterwards dump_tsa
on every consecutive call read from cached version
use cleancache to remove caches
parameters:
datestring <str>
filterkeys <tuple> or None default None
index_pattern <str> or None default None
timedelta <int> default 0
cleancache <bool> default False
validate <bool> if data is read from raw, dump it after initail read,
and reread it afterwards to make sure the stored tsa is OK
thats an performance issue
returns
<TimeseriesArrayLazy> object read from cachefile or from raw data
"""
try:
assert not_today(datestring)
except AssertionError:
raise DataLoggerLiveDataError("Reading from live data is not allowed")
cachedir = self.__get_cachedir(datestring)
cachefilename = os.path.join(cachedir, TimeseriesArrayLazy.get_dumpfilename(self.__index_keynames))
def fallback():
"""
fallback method to use, if reading from cache data is not possible
"""
tsa = self.load_tsa_raw(datestring, timedelta)
tsa.dump_split(cachedir) # save full data
# read the data afterwards to make sure there is no problem,
if validate is True:
tsa = TimeseriesArrayLazy.load_split(cachedir, self.__index_keynames, filterkeys=filterkeys, index_pattern=index_pattern, datatypes=self.__datatypes)
# also generate TSASTATS and dump to cache directory
tsastats = TimeseriesArrayStats(tsa) # generate full Stats
tsastats.dump(cachedir) # save
# and at last but not least quantile
qantile = QuantileArray(tsa, tsastats)
cachefilename = os.path.join(cachedir, "quantile.json")
qantile.dump(open(cachefilename, "wb"))
# finally return tsa
return tsa
if not os.path.isfile(cachefilename):
logging.info("cachefile %s does not exist, fallback read from raw data file", cachefilename)
return fallback()
if (os.path.isfile(cachefilename)) and (cleancache == True):
logging.info("deleting cachefile %s and read from raw data file", cachefilename)
os.unlink(cachefilename)
return fallback()
logging.debug("loading stored TimeseriesArrayLazy object file %s", cachefilename)
try:
tsa = TimeseriesArrayLazy.load_split(cachedir, self.__index_keynames, filterkeys=filterkeys, index_pattern=index_pattern, datatypes=self.__datatypes)
return tsa
except IOError:
logging.error("IOError while reading from %s, using fallback", cachefilename)
os.unlink(cachefilename)
return fallback()
except EOFError:
logging.error("EOFError while reading from %s, using fallback", cachefilename)
os.unlink(cachefilename)
return fallback()
