def __init__(self, store, cache_expiration_time=60, cache_size_in_mb=2000, cache_id=None, max_archive_size_in_mb = 4, cache_dir='/tmp/cloudfusion'):
"""
:param store: the store whose access should be cached
:param max_archive_size_in_mb: the maximum size of an archive
:param cache_expiration_time: the time in seconds until any cache entry is expired
:param cache_size_in_mb: Approximate (soft) limit of the cache in MB.
:param hard_cache_size_limit_in_mb: Hard limit of the cache in MB, exceeding this limit should slow down write operations.
:param cache_id: Serves as identifier for a persistent cache instance.
:param cache_dir: Cache directory on local hard drive disk, default value is */tmp/cloudfusion*."""
#prevent simultaneous access to store (synchronous use of __deepcopy__ by _store SyncThread and a different method):
self.store = SynchronizeProxy(store, private_methods_to_synchronize=['__deepcopy__'])
self.max_archive_size_in_mb = max_archive_size_in_mb
if cache_id == None:
cache_id = str(random.random())
self.logger = logging.getLogger(self.get_logging_handler())
self.logger.debug("creating ChunkTransparentMultiprocessingCachingStore object")
if cache_expiration_time < 240:
self.logger.warning("Be aware of the synchronization issue https://github.com/joe42/CloudFusion/issues/16 \
or to avoid the issue set cache_expiration_time to more than 240 seconds.")
self.cache_expiration_time = cache_expiration_time
self.time_of_last_flush = time.time()
self.cache_dir = cache_dir[:-1] if cache_dir[-1:] == '/' else cache_dir # remove slash at the end
temp_dir = self.cache_dir+"/cachingstore_"+cache_id
cache = PersistentLRUCache(temp_dir, cache_expiration_time, cache_size_in_mb)
cache.set_resize_intervall(10)
self.entries = SynchronizeProxy( cache ) #[shares_resource: write self.entries]
self.sync_thread = ChunkStoreSyncThread(self.entries, self.store, temp_dir, self.logger)
self.sync_thread.start()
def test_get_keys():
test_obj = PersistentLRUCache(directory=directory)
test_obj.refresh("some_key", "43", time.time())
test_obj.write("some_other_key", "42")
assert "some_key" in test_obj.get_keys()
assert "some_other_key" in test_obj.get_keys()
assert not "some_keyXYZ" in test_obj.get_keys()
def test_get_modified():
test_obj = PersistentLRUCache(directory=directory)
modified_time = time.time()
test_obj.refresh("some_key", "43", modified_time)
assert test_obj.get_modified("some_key") == modified_time
test_obj.write("some_key", "42")
assert test_obj.get_modified("some_key") < time.time()
def test_update():
test_obj = PersistentLRUCache(directory,1)
test_obj.write("some_key", "42")
time.sleep(2)
assert test_obj.is_expired("some_key")
test_obj.update("some_key")
assert not test_obj.is_expired("some_key")
def test_persistence():
test_obj = PersistentLRUCache(directory=directory, maxsize_in_MB=0)
test_obj.write("keyX", "42")
assert test_obj.get_value("keyX") == "42"
test_obj.entries.close()
test_obj = PersistentLRUCache(directory=directory)
assert test_obj.get_value("keyX") == "42"
def test_reorder():
test_obj = PersistentLRUCache(directory=directory)
test_obj.write("/xxx", "")
test_obj.write("/yyy", "")
test_obj.get_value("/xxx")
test_obj.delete("/xxx")
test_obj.delete("/yyy")
def test_resize():
test_obj = PersistentLRUCache(directory=directory, expiration_time=0.00001,maxsize_in_MB=30)
test_obj.set_resize_intervall(0)
for i in range(10,62):
test_obj.refresh(str(i), "a"*2000000, time.time())
time.sleep(0.001)
assert test_obj.get_size_of_cached_data() < 30000003
for j in range(10,i-14+1):
assert not str(j) in test_obj.get_keys()
for j in range(10,i+1)[-14:]:
assert test_obj.get_value(str(j)) == "a"*2000000
def test_set_modified():
test_obj = PersistentLRUCache(directory=directory)
modified_time = 42
before_modification = time.time()
test_obj.write("some_key", 101)
assert test_obj.get_modified("some_key") < time.time()
assert test_obj.get_modified("some_key") > before_modification
test_obj.set_modified("some_key", modified_time)
assert test_obj.get_modified("some_key") == modified_time
def test_write():
test_obj = PersistentLRUCache(directory=directory)
test_obj.write("some_key", "42")
test_obj.write("42", "some_key")
assert test_obj.get_value("some_key") == "42"
assert test_obj.get_value("42") == "some_key"
assert test_obj.is_dirty("some_key")
def __init__(self,
store,
cache_expiration_time=60,
cache_size_in_mb=2000,
cache_id=None,
cache_dir='/tmp/cloudfusion/'):
"""
:param store: the store whose access should be cached
:param cache_expiration_time: the time in seconds until any cache entry is expired
:param cache_size_in_mb: Approximate limit of the cache in MB.
:param cache_id: Serves as identifier for a persistent cache instance.
:param cache_dir: Cache directory on local hard drive disk, default value is */tmp/cloudfusion*. """
#prevent simultaneous access to store (synchronous use of __deepcopy__ by _store SyncThread and a different method):
self.store = SynchronizeProxy(
store, private_methods_to_synchronize=['__deepcopy__'])
if cache_id == None:
cache_id = str(random.random())
self.logger = logging.getLogger(self.get_logging_handler())
self.logger.debug("creating CachingStore object")
if cache_expiration_time < 240:
self.logger.warning(
"Be aware of the synchronization issue https://github.com/joe42/CloudFusion/issues/16 \
or to avoid the issue set cache_expiration_time to more than 240 seconds."
)
# self.temp_file = tempfile.SpooledTemporaryFile()
self.cache_expiration_time = cache_expiration_time
self.time_of_last_flush = time.time()
self.cache_dir = cache_dir[:-1] if cache_dir[
-1:] == '/' else cache_dir # remove slash at the end
cache = PersistentLRUCache(
self.cache_dir + "/cachingstore_" + cache_id,
cache_expiration_time, cache_size_in_mb)
cache.set_resize_intervall(10)
self.entries = SynchronizeProxy(
cache) #[shares_resource: write self.entries]
self.sync_thread = StoreSyncThread(self.entries, self.store,
self.logger)
self.sync_thread.start()
def test_get_value():
test_obj = PersistentLRUCache(directory=directory)
test_obj.refresh("some_key", "43", time.time())
assert test_obj.get_value("some_key") == "43"
test_obj.write("some_key", "42")
assert test_obj.get_value("some_key") == "42"
def test_is_dirty():
test_obj = PersistentLRUCache(directory=directory)
test_obj.refresh("some_key", "43", time.time())
assert not test_obj.is_dirty("some_key")
test_obj.write("some_key", "42")
assert test_obj.is_dirty("some_key")
def test_resize_zerosize():
test_obj = PersistentLRUCache(directory=directory, expiration_time=0.00001, maxsize_in_MB=0)
test_obj.set_resize_intervall(0)
test_obj.refresh("some_key", "43", time.time())
time.sleep(0.001)
assert "some_key" in test_obj.get_keys()
test_obj.refresh("some_other_key", "42", time.time())
assert "some_other_key" in test_obj.get_keys()
assert not "some_key" in test_obj.get_keys() #deleted due to internal resize
assert test_obj.get_value("some_other_key") == "42"
def test_delete():
test_obj = PersistentLRUCache(directory=directory)
test_obj.write("some_key", "42")
test_obj.write("42", "some_key")
test_obj.delete("some_key")
test_obj.delete("non_existant_key")
test_obj.delete("42")
assert_raises( KeyError, test_obj.get_value, ("42") )
assert_raises( KeyError, test_obj.get_value, ("some_key") )
assert not test_obj.exists("42")
assert not test_obj.exists("some_key")
def test_exists():
test_obj = PersistentLRUCache(directory=directory)
assert not test_obj.exists("some_key")
test_obj.write("some_key", "42")
assert test_obj.exists("some_key")
assert not test_obj.exists("some_other_key")
def test_get_size_of_dirty_data():
test_obj = PersistentLRUCache(directory=directory)
assert test_obj.get_size_of_dirty_data() == 0
test_obj.refresh("some_key", "abcd", time.time())
assert test_obj.get_size_of_dirty_data() == 0
test_obj.write("some_other_key", "42")
assert test_obj.get_size_of_dirty_data() == 2
test_obj.write("some_other_key", 52)
assert test_obj.get_size_of_dirty_data() == 2
test_obj.write("some_key", "abcd")
assert test_obj.get_size_of_dirty_data() == 6
test_obj.refresh("some_other_key", "42", time.time())
assert test_obj.get_size_of_dirty_data() == 4
def test_refresh():
test_obj = PersistentLRUCache(directory=directory)
test_obj.refresh("some_key", "43", time.time())
test_obj.refresh("some_key","42", time.time())
assert test_obj.get_value("some_key") == "42"
test_obj.refresh("some_key","43", time.time()-1000)
assert test_obj.get_value("some_key") == "42", "Refresh should not have worked since the modified time of the 'disk' entry is older than the cache entry."
assert not test_obj.is_dirty("some_key")
def test_resize_dirty():
test_obj = PersistentLRUCache(directory=directory, maxsize_in_MB=0)
test_obj.set_resize_intervall(0)
for i in range(10,62):
test_obj.write(str(i), "a"*2000000)
assert test_obj.get_size_of_cached_data() > 50000000
def test_get_size_of_cached_data():
test_obj = PersistentLRUCache(directory=directory)
modified_time = time.time()
assert test_obj.get_size_of_cached_data() == 0
test_obj.refresh("some_key", "abcd", modified_time)
assert test_obj.get_size_of_cached_data() == 4
test_obj.write("some_other_key", "42")
assert test_obj.get_size_of_cached_data() == 6
test_obj.write("some_other_key", 52)
assert test_obj.get_size_of_cached_data() == 6
test_obj.refresh("some_key", "abcd", modified_time)
assert test_obj.get_size_of_cached_data() == 6
