def __init__(self, filename, retain_old=True, raise_unpickable=False):
"""Initializer.
Checks if the file can be accessed and load the data therein if any. If the file does not yet exist, start
with an empty shelf. This ensures that old data is readily available when the FileCache instance is created.
The file is closed after reading the data.
@type filename: string
@param filename: (absolute) path to the cache file.
"""
self.log = fancylogger.getLogger(self.__class__.__name__, fname=False)
self.filename = filename
self.retain_old = retain_old
self.new_shelf = {}
if not retain_old:
self.log.info("Starting with a new empty cache, not retaining previous info if any.")
self.shelf = {}
return
try:
with open(self.filename, 'rb') as f:
try:
g = gzip.GzipFile(mode='rb', fileobj=f) # no context manager available in python 26 yet
s = g.read()
except (IOError) as err:
self.log.error("Cannot load data from cache file %s as gzipped json", self.filename)
try:
f.seek(0)
self.shelf = pickle.load(f)
except pickle.UnpicklingError as err:
msg = "Problem loading pickle data from %s (corrupt data)" % (self.filename,)
if raise_unpickable:
self.log.raiseException(msg)
else:
self.log.error("%s. Continue with empty shelf: %s", msg, err)
self.shelf = {}
except (OSError, IOError):
self.log.raiseException("Could not load pickle data from %s", self.filename)
else:
try:
self.shelf = jsonpickle.decode(s)
except ValueError as err:
self.log.error("Cannot decode JSON from %s [%s]", self.filename, err)
self.log.info("Cache in %s starts with an empty shelf", self.filename)
self.shelf = {}
finally:
g.close()
except (OSError, IOError, ValueError, FileNotFoundErrorExc) as err:
self.log.warning("Could not access the file cache at %s [%s]", self.filename, err)
self.shelf = {}
self.log.info("Cache in %s starts with an empty shelf", (self.filename,))
def test_picke(self):
"""Tests for pickle module provided by py2vs3."""
test_pickle_file = os.path.join(self.tmpdir, 'test.pickle')
test_dict = {1: 'one', 2: 'two'}
fp = open(test_pickle_file, 'wb')
pickle.dump(test_dict, fp)
fp.close()
self.assertTrue(os.path.exists(test_pickle_file))
fp = open(test_pickle_file, 'rb')
loaded_pickle = pickle.load(fp)
fp.close()
self.assertEqual(test_dict, loaded_pickle)
def read(self):
"""Read a timestamp value from a pickled file.
@type filename: string representing the filename of the file to read from.
@returns: the timestamp in the same format it was stored in.
@raise
"""
try:
timestamp = pickle.load(open(self.filename, "rb"))
except (IOError, OSError, FileNotFoundErrorExc):
logging.exception(
"Failed to load timestamp pickle from filename %s.",
self.filename)
return None
return timestamp