def test_refresh():
test_obj = Cache(1)
test_obj.refresh("some_key", 43, time.time())
test_obj.refresh("some_key",42, time.time())
assert test_obj.get_value("some_key") == 42
test_obj.refresh("some_key",43, time.time()-1000)
assert test_obj.get_value("some_key") == 42, "Refresh should not have worked since the modified time of the 'disk' entry is older than the cache entry."
assert not test_obj.is_dirty("some_key")
def test_get_keys():
test_obj = Cache(1)
test_obj.refresh("some_key", 43, time.time())
test_obj.write("some_other_key", 42)
assert "some_key" in test_obj.get_keys()
assert "some_other_key" in test_obj.get_keys()
assert not "some_keyXYZ" in test_obj.get_keys()
def test_refresh():
test_obj = Cache(1)
test_obj.refresh("some_key", 43, time.time())
test_obj.refresh("some_key", 42, time.time())
assert test_obj.get_value("some_key") == 42
test_obj.refresh("some_key", 43, time.time() - 1000)
assert test_obj.get_value(
"some_key"
) == 42, "Refresh should not have worked since the modified time of the 'disk' entry is older than the cache entry."
assert not test_obj.is_dirty("some_key")
def test_update():
test_obj = Cache(1)
test_obj.write("some_key", 42)
time.sleep(2)
assert test_obj.is_expired("some_key")
test_obj.update("some_key")
assert not test_obj.is_expired("some_key")
def test_get_keys():
test_obj = Cache(1)
test_obj.refresh("some_key", 43, time.time())
test_obj.write("some_other_key", 42)
assert "some_key" in test_obj.get_keys()
assert "some_other_key" in test_obj.get_keys()
assert not "some_keyXYZ" in test_obj.get_keys()
def test_get_modified():
test_obj = Cache(1)
modified_time = time.time()
test_obj.refresh("some_key", 43, modified_time)
assert test_obj.get_modified("some_key") == modified_time
test_obj.write("some_key", 42)
assert test_obj.get_modified("some_key") < time.time()
def test_set_modified():
test_obj = Cache(1)
modified_time = 42
before_modification = time.time()
test_obj.write("some_key", 101)
assert test_obj.get_modified("some_key") < time.time()
assert test_obj.get_modified("some_key") > before_modification
test_obj.set_modified("some_key", modified_time)
assert test_obj.get_modified("some_key") == modified_time
def __init__(self, store, cache_expiration_time=60):
self.store = store
self.logger = logging.getLogger(self.get_logging_handler())
self.logger.debug("creating MetadataCachingStore object")
self.entries = MPSynchronizeProxy( MPLRUCache(cache_expiration_time,2) )
self.store_metadata = Cache(cache_expiration_time)
self.free_space_worker = GetFreeSpaceWorker(deepcopy(store), self.logger)
self.free_space_worker.start()
def test_get_modified():
test_obj = Cache(1)
modified_time = time.time()
test_obj.refresh("some_key", 43, modified_time)
assert test_obj.get_modified("some_key") == modified_time
test_obj.write("some_key", 42)
assert test_obj.get_modified("some_key") < time.time()
def test_update():
test_obj = Cache(1)
test_obj.write("some_key", 42)
time.sleep(2)
assert test_obj.is_expired("some_key")
test_obj.update("some_key")
assert not test_obj.is_expired("some_key")
def test_write():
test_obj = Cache(1)
test_obj.write("some_key", 42)
test_obj.write(42, "some_key")
assert test_obj.get_value("some_key") == 42
assert test_obj.get_value(42) == "some_key"
assert test_obj.is_dirty("some_key")
def __init__(self, store, cache_expiration_time):
"""":param store: the store whose access should be cached
:param cache_expiration_time: the time in seconds until any cache entry is expired"""
self.store = store
self.logger = logging.getLogger(self.get_logging_handler())
self.logger.debug("creating CachingStore object")
# self.temp_file = tempfile.SpooledTemporaryFile()
self.cache_expiration_time = cache_expiration_time
self.time_of_last_flush = time.time()
self.entries = Cache(cache_expiration_time)
def test_set_modified():
test_obj = Cache(1)
modified_time = 42
before_modification = time.time()
test_obj.write("some_key", 101)
assert test_obj.get_modified("some_key") < time.time()
assert test_obj.get_modified("some_key") > before_modification
test_obj.set_modified("some_key", modified_time)
assert test_obj.get_modified("some_key") == modified_time
def test_write():
test_obj = Cache(1)
test_obj.write("some_key", 42)
test_obj.write(42, "some_key")
assert test_obj.get_value("some_key") == 42
assert test_obj.get_value(42) == "some_key"
assert test_obj.is_dirty("some_key")
def __init__(self, store, cache_expiration_time=60):
self.store = store
self.logger = logging.getLogger(self.get_logging_handler())
self.logger.debug("creating MetadataCachingStore object")
self.entries = MPSynchronizeProxy( MPCache(cache_expiration_time) )
if cache_expiration_time < 240:
self.logger.warning("Be aware of the synchronization issue https://github.com/joe42/CloudFusion/issues/16 \
or to avoid the issue set cache_expiration_time to more than 240 seconds.")
self.store_metadata = Cache(cache_expiration_time)
self.free_space_worker = GetFreeSpaceWorker(deepcopy(store), self.logger)
self.free_space_worker.start()
self._last_cleaned = time.time()
manager = Manager()
self._is_cleaning_cache = Lock()
self._is_uploading_lock = Lock()
self._is_uploading = manager.Value('i', 0)
def test_get_value():
test_obj = Cache(1)
test_obj.refresh("some_key", 43, time.time())
assert test_obj.get_value("some_key") == 43
test_obj.write("some_key", 42)
assert test_obj.get_value("some_key") == 42
def test_is_dirty():
test_obj = Cache(1)
test_obj.refresh("some_key", 43, time.time())
assert not test_obj.is_dirty("some_key")
test_obj.write("some_key", 42)
assert test_obj.is_dirty("some_key")
def test_delete():
test_obj = Cache(1)
test_obj.write("some_key", 42)
test_obj.write(42, "some_key")
test_obj.delete("some_key")
test_obj.delete("non_existant_key")
test_obj.delete(42)
assert_raises( KeyError, test_obj.get_value, (42) )
assert_raises( KeyError, test_obj.get_value, ("some_key") )
assert not test_obj.exists(42)
assert not test_obj.exists("some_key")
class CachingStore(Store):
def __init__(self, store, cache_expiration_time):
"""":param store: the store whose access should be cached
:param cache_expiration_time: the time in seconds until any cache entry is expired"""
self.store = store
self.logger = logging.getLogger(self.get_logging_handler())
self.logger.debug("creating CachingStore object")
# self.temp_file = tempfile.SpooledTemporaryFile()
self.cache_expiration_time = cache_expiration_time
self.time_of_last_flush = time.time()
self.entries = Cache(cache_expiration_time)
def get_cache_expiration_time(self):
""":returns: the time in seconds until any cache entry is expired"""
return self.cache_expiration_time;
def get_time_of_last_flush(self):
""":returns: the most recent point of time at which the cache has been flushed"""
return self.time_of_last_flush;
def _is_valid_path(self, path):
return self.store._is_valid_path(path)
def _raise_error_if_invalid_path(self, path):
self.store._raise_error_if_invalid_path(path)
def get_name(self):
return self.store.get_name()
def _refresh_cache(self, path_to_file):
""" Reloads the existing entries entry with the key :param:`path_to_file` from the wrapped store, if the wrapped store version is newer.
The entries entries last updated time is set to the current point of time.
Makes a new entries entry if it does not exist yet.
If it was changed in the wrapped store after the entries entry's modified time:
* The modified time stamp of the entries entry is set to the wrapped stores modified time stamp.
* The entry is set to not dirty.
* The entry's value is set to the wrapped store's value.
:raises: NoSuchFilesytemObjectError if file does not exist in wrapped store.
TODO: exception raising does not work and needs to be implemented
"""
cached_version_is_invalid = self.is_cached_version_invalid(path_to_file)
if cached_version_is_invalid:
actual_modified_date = self._get_actual_modified_date(path_to_file)
data = self.store.get_file(path_to_file)
modified = actual_modified_date
self.entries.refresh(path_to_file, data, modified)
self.logger.debug("refreshed cached fileobject from store")
self.entries.update(path_to_file)
def is_cached_version_invalid(self, path):
""":returns: True if the stores version is newer than the cached entry or does not exist and False otherwise."""
if self.entries.exists(path):
actual_modified_date = self._get_actual_modified_date(path)
cached_modified_date = self.entries.get_modified(path)
if actual_modified_date > cached_modified_date:
return True
else:
return True
return False
def get_file(self, path_to_file):
""" :returns: string -- the data of the file with the path :param: path_to_file
If the files entry is dirty and expired it is written to the wrapped store first and set to not dirty and update time is set to the current point of time.
If the file was updated in the wrapped store, then its content will be updated if its entry is expired.
"""
self.logger.debug("cached get_file %s" % path_to_file)
if not self.entries.exists(path_to_file):
self._refresh_cache(path_to_file)
self.logger.debug("cached get_file from new entry")
return self.entries.get_value(path_to_file)
if self.entries.is_expired(path_to_file):
if self.entries.is_dirty(path_to_file): #dirty&expired->flush
self.logger.debug("cached get_file flushing")
self.__flush(path_to_file)
else: #not dirty&expired->update cache from store
self.logger.debug("cached get_file update from store if newer")
self._refresh_cache(path_to_file)
return self.entries.get_value(path_to_file) #not expired->from entries
def store_file(self, path_to_file, dest_dir="/", remote_file_name = None):
if dest_dir == "/":
dest_dir = ""
fileobject = open(path_to_file)
if not remote_file_name:
remote_file_name = os.path.basename(path_to_file)
self.store_fileobject(fileobject, dest_dir + "/" + remote_file_name)
def _get_actual_modified_date(self, path):
ret = 0
if self.store.exists(path):
ret = self.store._get_metadata(path)['modified']
return ret
def store_fileobject(self, fileobject, path):
""" Stores a fileobject to the :class:`~cloudfusion.util.entries.Cache` and if the existing fileobject has expired it is also flushed to the wrapped store.
The entry's updated and modified attributes will be reset to the current point of time.
The entry's dirty flag is set to False if the entry has expired and was hence written to the store. Otherwise it is set to True.
:param fileobject: The file object with the method read() returning data as a string
:param path: The path where the file object's data should be stored, including the filename
"""
self.logger.debug("cached storing %s" % path)
flush = self.entries.exists(path) and self.entries.is_expired(path)
self.entries.write(path, fileobject.read())
self.logger.debug("cached storing value %s..." %self.entries.get_value(path)[:10])
if flush:
self.logger.debug("cache entry for %s is expired -> flushing" % path)
self._flush(path)
def delete(self, path):#delete from metadata
self.entries.delete(path)
if self.store.exists(path):
self.store.delete(path)
def account_info(self):
self.flush()
return self.store.account_info()
def get_free_space(self):
self.flush()
return self.store.get_free_space()
def get_overall_space(self):
self.flush()
return self.store.get_overall_space()
def get_used_space(self):
self.flush()
return self.store.get_used_space()
def create_directory(self, directory):
return self.store.create_directory(directory)
def duplicate(self, path_to_src, path_to_dest):
self.__flush(path_to_src)
self.__flush(path_to_dest)
self.logger.debug("cached storing duplicate %s to %s" % (path_to_src, path_to_dest))
ret = self.store.duplicate(path_to_src, path_to_dest)
if self.entries.exists(path_to_src):
self.entries.write(path_to_dest, self.entries.get_value(path_to_src))
return ret
def move(self, path_to_src, path_to_dest):
if self.entries.exists(path_to_src) and self.entries.is_dirty(path_to_src): #less bandwith usage if it is copied locally
self.entries.write(path_to_dest, self.entries.get_value(path_to_src))
self.entries.delete(path_to_src)
return
#it already was up to date at the remote server:
self.store.move(path_to_src, path_to_dest)
if self.entries.exists(path_to_src):
self.entries.write(path_to_dest, self.entries.get_value(path_to_src))
self.entries.delete(path_to_src)
def get_modified(self, path):
return self._get_metadata(path)["modified"]
def get_directory_listing(self, directory):
self.flush()
return self.store.get_directory_listing(directory) #so far only cached by dropbox
def get_bytes(self, path):
return self._get_metadata(path)['bytes']
def exists(self, path):
if self.entries.exists(path):
return True
try:
self._get_metadata(path)
return True
except NoSuchFilesytemObjectError:
return False;
def _get_metadata(self, path):
self.flush()
return self.store._get_metadata(path)
def is_dir(self, path):
return self._get_metadata(path)["is_dir"]
def flush(self):
""" Writes all dirty entries to the wrapped store."""
self.logger.debug("flushing entries")
self.time_of_last_flush = time.time()
for path in self.entries.get_keys():
self.__flush(path)
def __flush(self, path):
""" Writes the entry with the key :param:`path` to the wrapped store, only if it is dirty."""
if not self.entries.exists(path):
return
self.entries.update(path)
if self.entries.is_dirty(path):
file = DataFileWrapper(self.entries.get_value(path))
file.fileno()#
self.store.store_fileobject(file, path)
self.entries.flush(path)
self.logger.debug("flushing %s with content starting with %s" % (path, self.entries.get_value(path)[0:10]))
def get_logging_handler(self):
return self.store.get_logging_handler()
def test_delete():
test_obj = Cache(1)
test_obj.write("some_key", 42)
test_obj.write(42, "some_key")
test_obj.delete("some_key")
test_obj.delete("non_existant_key")
test_obj.delete(42)
assert_raises(KeyError, test_obj.get_value, (42))
assert_raises(KeyError, test_obj.get_value, ("some_key"))
assert not test_obj.exists(42)
assert not test_obj.exists("some_key")
def test_get_value():
test_obj = Cache(1)
test_obj.refresh("some_key", 43, time.time())
assert test_obj.get_value("some_key") == 43
test_obj.write("some_key", 42)
assert test_obj.get_value("some_key") == 42
def test_get_size_of_dirty_data():
test_obj = Cache(1)
assert test_obj.get_size_of_dirty_data() == 0
test_obj.refresh("some_key", "abcd", time.time())
assert test_obj.get_size_of_dirty_data() == 0
test_obj.write("some_other_key", 42)
assert test_obj.get_size_of_dirty_data() == 2
test_obj.write("some_other_key", 52)
assert test_obj.get_size_of_dirty_data() == 2
test_obj.write("some_key", "abcd")
assert test_obj.get_size_of_dirty_data() == 6
test_obj.refresh("some_other_key", 42, time.time())
assert test_obj.get_size_of_dirty_data() == 4
def test_exists():
test_obj = Cache(1)
assert not test_obj.exists("some_key")
test_obj.write("some_key", 42)
assert test_obj.exists("some_key")
assert not test_obj.exists("some_other_key")
def test_is_dirty():
test_obj = Cache(1)
test_obj.refresh("some_key", 43, time.time())
assert not test_obj.is_dirty("some_key")
test_obj.write("some_key", 42)
assert test_obj.is_dirty("some_key")
def test_get_size_of_cached_data():
test_obj = Cache(1)
modified_time = time.time()
assert test_obj.get_size_of_cached_data() == 0
test_obj.refresh("some_key", "abcd", modified_time)
assert test_obj.get_size_of_cached_data() == sys.getsizeof("abcd")
test_obj.write("some_other_key", 42)
assert test_obj.get_size_of_cached_data(
) == sys.getsizeof(42) + sys.getsizeof("abcd")
test_obj.write("some_other_key", 52)
assert test_obj.get_size_of_cached_data(
) == sys.getsizeof(52) + sys.getsizeof("abcd")
test_obj.refresh("some_key", "abcd", modified_time)
assert test_obj.get_size_of_cached_data(
) == sys.getsizeof(52) + sys.getsizeof("abcd")
def __init__(self, store, cache_expiration_time):
self.store = store
self.logger = logging.getLogger(self.get_logging_handler())
self.logger.debug("creating MetadataCachingStore object")
self.entries = Cache(cache_expiration_time)
self.store_metadata = Cache(cache_expiration_time)
def test_get_size_of_cached_data():
test_obj = Cache(1)
modified_time = time.time()
assert test_obj.get_size_of_cached_data() == 0
test_obj.refresh("some_key", "abcd", modified_time)
assert test_obj.get_size_of_cached_data() == 4
test_obj.write("some_other_key", 42)
assert test_obj.get_size_of_cached_data() == 6
test_obj.write("some_other_key", 52)
assert test_obj.get_size_of_cached_data() == 6
test_obj.refresh("some_key", "abcd", modified_time)
assert test_obj.get_size_of_cached_data() == 6
class MetadataCachingStore(Store):
def __init__(self, store, cache_expiration_time=60):
self.store = store
self.logger = logging.getLogger(self.get_logging_handler())
self.logger.debug("creating MetadataCachingStore object")
self.entries = MPSynchronizeProxy( MPCache(cache_expiration_time) )
if cache_expiration_time < 240:
self.logger.warning("Be aware of the synchronization issue https://github.com/joe42/CloudFusion/issues/16 \
or to avoid the issue set cache_expiration_time to more than 240 seconds.")
self.store_metadata = Cache(cache_expiration_time)
self.free_space_worker = GetFreeSpaceWorker(deepcopy(store), self.logger)
self.free_space_worker.start()
self._last_cleaned = time.time()
manager = Manager()
self._is_cleaning_cache = Lock()
self._is_uploading_lock = Lock()
self._is_uploading = manager.Value('i', 0)
def _acquire_uploading_lock(self):
'''This method needs to be called before uploading data.'''
with self._is_cleaning_cache:
# actually Value should support get_lock() but doesn't
with self._is_uploading_lock:
self._is_uploading.value = self._is_uploading.value + 1
def _release_uploading_lock(self):
'''This method needs to be called after uploading data.'''
with self._is_cleaning_cache:
with self._is_uploading_lock:
self._is_uploading.value = self._is_uploading.value - 1
def _is_valid_path(self, path):
return self.store._is_valid_path(path)
def _raise_error_if_invalid_path(self, path):
self.store._raise_error_if_invalid_path(path)
def get_name(self):
if not self.store_metadata.exists('store_name'):
self.store_metadata.write('store_name', self.store.get_name())
return self.store_metadata.get_value('store_name')
def get_file(self, path_to_file):
self.logger.debug("meta cache get_file %s", path_to_file)
ret = self.store.get_file(path_to_file)
if not self.entries.exists(path_to_file):
self.entries.write(path_to_file, Entry())
entry = self.entries.get_value(path_to_file)
entry.set_is_file()
try:
entry.size = len(ret)
self.entries.write(path_to_file, entry)
except NoSuchFilesytemObjectError:
self.entries.delete(path_to_file)
self._remove_from_parent_dir_listing(path_to_file)
self.logger.debug("meta cache returning %s", repr(ret)[:10])
self._add_to_parent_dir_listing(path_to_file)
return ret
def _add_to_parent_dir_listing(self, path):
if path != '/':
parent_dir = os.path.dirname(path)
self._add_parent_dir_listing(path)
entry = self.entries.get_value(parent_dir)
entry.add_to_listing(path)
self.entries.write(parent_dir, entry)
def _add_parent_dir_listing(self, path):
'''Add listing for parent directory of path to cache if it does not yet exist'''
if path != '/':
parent_dir = os.path.dirname(path)
if not self.entries.exists(parent_dir):
self.entries.write(parent_dir, Entry())
entry = self.entries.get_value(parent_dir)
if entry.listing == None:
entry.listing = self.store.get_directory_listing(parent_dir)
entry.set_is_dir()
self._add_existing_items(entry, parent_dir)
self.entries.write(parent_dir, entry)
def _does_not_exist_in_parent_dir_listing(self, path):
'''':returns: True if path does not exist in the cached directory listing'''
parent_dir = os.path.dirname(path)
if path == '/':
return False
if self.entries.exists(parent_dir):
entry = self.entries.get_value(parent_dir)
if entry.listing != None and (not unicode(path) in entry.listing and not path in entry.listing):
self.logger.debug("%s does not exist in parent directory: %s...", path, repr(entry.listing[0:5]))
return True
return False
def _remove_from_parent_dir_listing(self, path):
parent_dir = os.path.dirname(path)
if self.entries.exists(parent_dir):
entry = self.entries.get_value(parent_dir)
entry.remove_from_listing(path)
self.entries.write(parent_dir, entry)
'''entry deletion only while not uploading; use multithreading lock;at the beginning of the store methods
the entries should already be created; if it fails delete them, if it succeeds update them (eventual consistency
'''
def store_file(self, path_to_file, dest_dir="/", remote_file_name = None, interrupt_event=None):
if dest_dir == "/":
dest_dir = ""
if not remote_file_name:
remote_file_name = os.path.basename(path_to_file)
data_len = file_util.get_file_size_in_bytes(path_to_file)
path = dest_dir + "/" + remote_file_name
self.logger.debug("meta cache store_file %s", dest_dir + "/" + remote_file_name)
if not self.entries.exists(path):
self.entries.write(path, Entry())
entry = self.entries.get_value(path)
entry.set_is_file()
entry.size = data_len
entry.set_modified()
self.entries.write(path, entry)
self.logger.debug("meta cache store_file %s", path)
try:
self._acquire_uploading_lock()
ret = self.store.store_file(path_to_file, dest_dir, remote_file_name, interrupt_event)
except:
# delete entry
self.entries.delete(path)
raise
finally:
self._release_uploading_lock()
entry = self.entries.get_value(path)
entry.set_is_file()
entry.size = data_len
entry.set_modified()
self.entries.write(path, entry)
self._add_to_parent_dir_listing(path)
return ret
def store_fileobject(self, fileobject, path, interrupt_event=None):
self.logger.debug("meta cache store_fileobject %s", path)
data_len = file_util.get_file_size_in_bytes(fileobject)
if not self.entries.exists(path):
self.entries.write(path, Entry())
entry = self.entries.get_value(path)
entry.set_is_file()
entry.size = data_len
entry.set_modified()
self.entries.write(path, entry)
try:
self._acquire_uploading_lock()
ret = self.store.store_fileobject(fileobject, path, interrupt_event)
except:
# delete entry
self.entries.delete(path)
raise
finally:
self._release_uploading_lock()
fileobject.close()
entry = self.entries.get_value(path)
entry.set_is_file()
entry.size = data_len
entry.set_modified()
self._add_to_parent_dir_listing(path)
self.entries.write(path, entry)
return ret
def delete(self, path, is_dir):
self.logger.debug("meta cache delete %s", path)
self.store.delete(path, is_dir)
self.entries.delete(path)
self._remove_from_parent_dir_listing(path)
def account_info(self):
if not self.store_metadata.exists('account_info'):
self.store_metadata.write('account_info', self.store.account_info())
return self.store_metadata.get_value('account_info')
def get_free_space(self):
return self.free_space_worker.get_free_bytes_in_remote_store()
def get_overall_space(self):
if not self.store_metadata.exists('overall_space') or self.store_metadata.is_expired('overall_space'):
self.store_metadata.write('overall_space', self.store.get_overall_space())
return self.store_metadata.get_value('overall_space')
def get_used_space(self):
if not self.store_metadata.exists('used_space') or self.store_metadata.is_expired('used_space'):
self.store_metadata.write('used_space', self.store.get_used_space())
return self.store_metadata.get_value('used_space')
def create_directory(self, directory):
self.logger.debug("meta cache create_directory %s", directory)
try:
ret = self.store.create_directory(directory)
except AlreadyExistsError:
raise
if not self.entries.exists(directory):
self.entries.write(directory, Entry())
entry = self.entries.get_value(directory)
entry.set_is_dir()
entry.listing = []
entry.set_modified()
self.entries.write(directory, entry)
self._add_to_parent_dir_listing(directory)
return ret
def duplicate(self, path_to_src, path_to_dest):
self.logger.debug("meta cache duplicate %s to %s", path_to_src, path_to_dest)
ret = self.store.duplicate(path_to_src, path_to_dest)
if self.entries.exists(path_to_src):
entry = deepcopy(self.entries.get_value(path_to_src))
self.entries.write(path_to_dest, entry)
else:
self.entries.write(path_to_dest, Entry())
entry = self.entries.get_value(path_to_dest)
entry.set_modified()
self.entries.write(path_to_dest, entry)
self._add_to_parent_dir_listing(path_to_dest)
self.logger.debug("duplicated %s to %s", path_to_src, path_to_dest)
return ret
def move(self, path_to_src, path_to_dest):
self.logger.debug("meta cache move %s to %s", path_to_src, path_to_dest)
self.store.move(path_to_src, path_to_dest)
if self.entries.exists(path_to_src):
entry = self.entries.get_value(path_to_src)
self.entries.write(path_to_dest, entry)
else:
self.entries.write(path_to_dest, Entry())
entry = self.entries.get_value(path_to_dest)
entry.set_modified()
self.entries.write(path_to_dest, entry)
self.entries.delete(path_to_src)
self._remove_from_parent_dir_listing(path_to_src)
self._add_to_parent_dir_listing(path_to_dest)
def get_modified(self, path):
self.logger.debug("meta cache get_modified %s", path)
if self.entries.exists(path):
entry = self.entries.get_value(path)
if not entry.modified == None:
return entry.modified
modified = self.store.get_modified(path)
if not self.entries.exists(path):
self.entries.write(path, Entry())
entry = self.entries.get_value(path)
entry.set_modified(modified)
self.entries.write(path, entry)
return entry.modified
def get_directory_listing(self, directory):
self.logger.debug("meta cache get_directory_listing %s", directory)
if self.entries.exists(directory):
entry = self.entries.get_value(directory)
if not entry.listing == None:
self.logger.debug("return cached listing %s", repr(entry.listing))
return list(entry.listing)
listing = self.store.get_directory_listing(directory)
self.logger.debug("meta cache caching %s", repr(listing))
if not self.entries.exists(directory):
self.entries.write(directory, Entry())
entry = self.entries.get_value(directory)
entry.listing = listing
self._add_existing_items(entry, directory)
entry.set_is_dir()
self.entries.write(directory, entry)
self.logger.debug("asserted %s", repr(self.entries.get_value(directory).listing))
assert self.entries.get_value(directory).listing == entry.listing
return list(entry.listing)
def _add_existing_items(self, dir_entry, dir_entry_path):
'''Add existing files or directories to *dir_entry* because they might have been
uploaded recently and might not be retrievable by a directory listing from the storage provider.'''
for path in self.entries.get_keys():
if os.path.dirname(path) == dir_entry_path and path != '/':
try:
if not self.entries.is_expired(path):
dir_entry.add_to_listing(path)
except KeyError, e: #KeyError means that the entry has been deleted by __clean_cache
pass
def test_get_size_of_dirty_data():
test_obj = Cache(1)
assert test_obj.get_size_of_dirty_data() == 0
test_obj.refresh("some_key", "abcd", time.time())
assert test_obj.get_size_of_dirty_data() == 0
test_obj.write("some_other_key", 42)
assert test_obj.get_size_of_dirty_data() == sys.getsizeof(42)
test_obj.write("some_other_key", 52)
assert test_obj.get_size_of_dirty_data() == sys.getsizeof(52)
test_obj.write("some_key", "abcd")
assert test_obj.get_size_of_dirty_data(
) == sys.getsizeof(52) + sys.getsizeof("abcd")
test_obj.refresh("some_other_key", 42, time.time())
assert test_obj.get_size_of_dirty_data() == sys.getsizeof("abcd")
def test_exists():
test_obj = Cache(1)
assert not test_obj.exists("some_key")
test_obj.write("some_key", 42)
assert test_obj.exists("some_key")
assert not test_obj.exists("some_other_key")
class MetadataCachingStore(Store):
def __init__(self, store, cache_expiration_time):
self.store = store
self.logger = logging.getLogger(self.get_logging_handler())
self.logger.debug("creating MetadataCachingStore object")
self.entries = Cache(cache_expiration_time)
self.store_metadata = Cache(cache_expiration_time)
def _is_valid_path(self, path):
return self.store._is_valid_path(path)
def _raise_error_if_invalid_path(self, path):
self.store._raise_error_if_invalid_path(path)
def get_name(self):
if not self.store_metadata.exists('store_name'):
self.store_metadata.write('store_name', self.store.get_name())
return self.store_metadata.get_value('store_name')
def get_file(self, path_to_file):
self.logger.debug("meta cache get_file %s" % path_to_file)
ret = self.store.get_file(path_to_file)
self.logger.debug("meta cache got %s" % repr(ret)[:10])
if not self.entries.exists(path_to_file):
self.entries.write(path_to_file, Entry())
entry = self.entries.get_value(path_to_file)
entry.set_is_file()
try:
entry.size = len(ret)
except:
pass
self.logger.debug("meta cache returning %s" % repr(ret)[:10])
return ret
def _add_to_parent_dir_listing(self, path):
parent_dir = os.path.dirname(path)
if not self.entries.exists(parent_dir):
self.entries.write(parent_dir, Entry())
entry = self.entries.get_value(parent_dir)
entry.set_is_dir()
entry.add_to_listing(path)
def _remove_from_parent_dir_listing(self, path):
parent_dir = os.path.dirname(path)
if self.entries.exists(parent_dir):
entry = self.entries.get_value(parent_dir)
entry.remove_from_listing(path)
def store_file(self, path_to_file, dest_dir="/", remote_file_name = None):
if dest_dir == "/":
dest_dir = ""
if not remote_file_name:
remote_file_name = os.path.basename(path_to_file)
self.logger.debug("meta cache store_file %s" % dest_dir + "/" + remote_file_name)
fileobject = open(path_to_file)
self.store_fileobject(fileobject, dest_dir + "/" + remote_file_name)
fileobject.close()
def store_fileobject(self, fileobject, path):
self.logger.debug("meta cache store_fileobject %s" % path)
fileobject.seek(0)
data_len = len(fileobject.read())
fileobject.seek(0)
self.store.store_fileobject(fileobject, path)
fileobject.close()
if not self.entries.exists(path):
self.entries.write(path, Entry())
entry = self.entries.get_value(path)
entry.set_is_file()
entry.size = data_len
entry.set_modified()
self._add_to_parent_dir_listing(path)
def delete(self, path):
self.logger.debug("meta cache delete %s" % path)
self.store.delete(path)
self.entries.delete(path)
self._remove_from_parent_dir_listing(path)
def account_info(self):
return self.store.account_info()
def get_free_space(self):
return self.store.get_free_space()
def get_overall_space(self):
return self.store.get_overall_space()
def get_used_space(self):
return self.store.get_used_space()
def create_directory(self, directory):
self.logger.debug("meta cache create_directory %s" % directory)
ret = self.store.create_directory(directory)
if not self.entries.exists(directory):
self.entries.write(directory, Entry())
entry = self.entries.get_value(directory)
entry.set_is_dir()
entry.listing = []
entry.set_modified()
self._add_to_parent_dir_listing(directory)
return ret
def duplicate(self, path_to_src, path_to_dest):
self.logger.debug("meta cache duplicate %s to %s" % (path_to_src, path_to_dest))
ret = self.store.duplicate(path_to_src, path_to_dest)
if self.entries.exists(path_to_src):
entry = self.entries.get_value(path_to_src)
self.entries.write(path_to_dest, entry)
else:
self.entries.write(path_to_dest, Entry())
entry = self.entries.get_value(path_to_src)
entry.set_modified()
self._add_to_parent_dir_listing(path_to_dest)
self.logger.debug("duplicated %s to %s" % (path_to_src, path_to_dest))
return ret
def move(self, path_to_src, path_to_dest):
self.logger.debug("meta cache move %s to %s" % (path_to_src, path_to_dest))
self.store.move(path_to_src, path_to_dest)
if self.entries.exists(path_to_src):
entry = self.entries.get_value(path_to_src)
self.entries.write(path_to_dest, entry)
else:
self.entries.write(path_to_dest, Entry())
entry = self.entries.get_value(path_to_src)
entry.set_modified()
self.entries.delete(path_to_src)
self._remove_from_parent_dir_listing(path_to_src)
self._add_to_parent_dir_listing(path_to_dest)
def get_modified(self, path):
self.logger.debug("meta cache get_modified %s" % path)
if self.entries.exists(path):
entry = self.entries.get_value(path)
if not entry.modified == None:
return entry.modified
modified = self.store.get_modified(path)
if not self.entries.exists(path):
self.entries.write(path, Entry())
entry = self.entries.get_value(path)
entry.set_modified(modified)
return entry.modified
def get_directory_listing(self, directory):
self.logger.debug("meta cache get_directory_listing %s" % directory)
if self.entries.exists(directory):
entry = self.entries.get_value(directory)
if not entry.listing == None:
self.logger.debug("return cached listing %s" % repr(entry.listing))
return list(entry.listing)
listing = self.store.get_directory_listing(directory)
self.logger.debug("meta cache caching %s" % repr(listing))
if not self.entries.exists(directory):
self.entries.write(directory, Entry())
entry = self.entries.get_value(directory)
entry.listing = listing
assert self.entries.get_value(directory).listing == entry.listing
self.logger.debug("asserted %s" % repr(self.entries.get_value(directory).listing))
return list(entry.listing)
def get_bytes(self, path):
self.logger.debug("meta cache get_bytes %s" % path)
if self.entries.exists(path):
entry = self.entries.get_value(path)
if not entry.size == None:
return entry.size
size = self.store.get_bytes(path)
if not self.entries.exists(path):
self.entries.write(path, Entry())
entry = self.entries.get_value(path)
entry.size = size
return entry.size
def exists(self, path):
self.logger.debug("meta cache exists %s" % path)
if not self.entries.exists(path):
if self.store.exists(path):
self.entries.write(path, Entry())
return self.entries.exists(path)
def _get_metadata(self, path):
self.logger.debug("meta cache _get_metadata %s" % path)
if self.entries.exists(path):
entry = self.entries.get_value(path)
self.logger.debug("entry exists")
if not None in [entry.is_dir, entry.modified, entry.size]:
return {'is_dir': entry.is_dir, 'modified': entry.modified, 'bytes': entry.size}
self.logger.debug("meta cache _get_metadata entry does not exist")
metadata = self.store._get_metadata(path)
if not self.entries.exists(path):
self.entries.write(path, Entry())
entry = self.entries.get_value(path)
if metadata['is_dir']:
entry.set_is_dir()
else:
entry.set_is_file()
entry.modified = metadata['modified']
entry.size = metadata['bytes']
return {'is_dir': entry.is_dir, 'modified': entry.modified, 'bytes': entry.size}
def is_dir(self, path):
self.logger.debug("meta cache is_dir %s" % path)
if self.entries.exists(path):
entry = self.entries.get_value(path)
if not entry.is_dir == None:
return entry.is_dir
is_dir = self.store.is_dir(path)
if not self.entries.exists(path):
self.entries.write(path, Entry())
entry = self.entries.get_value(path)
if is_dir:
entry.set_is_dir()
return entry.is_dir
def get_logging_handler(self):
return self.store.get_logging_handler()
def flush(self):
self.store.flush()
class MetadataCachingStore(Store):
def __init__(self, store, cache_expiration_time=60):
self.store = store
self.logger = logging.getLogger(self.get_logging_handler())
self.logger.debug("creating MetadataCachingStore object")
self.entries = MPSynchronizeProxy( MPLRUCache(cache_expiration_time,2) )
self.store_metadata = Cache(cache_expiration_time)
self.free_space_worker = GetFreeSpaceWorker(deepcopy(store), self.logger)
self.free_space_worker.start()
def _is_valid_path(self, path):
return self.store._is_valid_path(path)
def _raise_error_if_invalid_path(self, path):
self.store._raise_error_if_invalid_path(path)
def get_name(self):
if not self.store_metadata.exists('store_name'):
self.store_metadata.write('store_name', self.store.get_name())
return self.store_metadata.get_value('store_name')
def get_file(self, path_to_file):
self.logger.debug("meta cache get_file %s", path_to_file)
ret = self.store.get_file(path_to_file)
if self.entries.exists(path_to_file) and self.entries.is_expired(path_to_file):
self.entries.delete(path_to_file)
if not self.entries.exists(path_to_file):
self.entries.write(path_to_file, Entry())
entry = self.entries.get_value(path_to_file)
entry.set_is_file()
try:
entry.size = len(ret)
self.entries.write(path_to_file, entry)
except:
self.entries.delete(path_to_file)
self.logger.debug("meta cache returning %s", repr(ret)[:10])
self._add_to_parent_dir_listing(path_to_file)
return ret
def _add_to_parent_dir_listing(self, path):
if path != '/':
parent_dir = os.path.dirname(path)
self._add_parent_dir_listing(path)
entry = self.entries.get_value(parent_dir)
entry.add_to_listing(path)
self.entries.write(parent_dir, entry)
def _add_parent_dir_listing(self, path):
'''Add listing for parent directory of path to cache if it does not yet exist'''
if path != '/':
parent_dir = os.path.dirname(path)
if not self.entries.exists(parent_dir):
self.entries.write(parent_dir, Entry())
entry = self.entries.get_value(parent_dir)
if entry.listing == None:
entry.listing = self.store.get_directory_listing(parent_dir)
entry.set_is_dir()
self.entries.write(parent_dir, entry)
def _does_not_exist_in_parent_dir_listing(self, path):
'''':returns: True if path does not exist in the cached directory listing'''
parent_dir = os.path.dirname(path)
if path == '/':
return False
if self.entries.exists(parent_dir):
entry = self.entries.get_value(parent_dir)
if entry.listing != None and (not unicode(path) in entry.listing and not path in entry.listing):
self.logger.debug("%s does not exist in parent directory: %s..."%(path, repr(entry.listing[0:5])))
return True
return False
def _remove_from_parent_dir_listing(self, path):
parent_dir = os.path.dirname(path)
if self.entries.exists(parent_dir):
entry = self.entries.get_value(parent_dir)
entry.remove_from_listing(path)
self.entries.write(parent_dir, entry)
def store_file(self, path_to_file, dest_dir="/", remote_file_name = None, interrupt_event=None):
if dest_dir == "/":
dest_dir = ""
if not remote_file_name:
remote_file_name = os.path.basename(path_to_file)
self.logger.debug("meta cache store_file %s", dest_dir + "/" + remote_file_name)
with open(path_to_file) as fileobject:
fileobject.seek(0,2)
data_len = fileobject.tell()
path = dest_dir + "/" + remote_file_name
self.logger.debug("meta cache store_file %s", path)
ret = self.store.store_file(path_to_file, dest_dir, remote_file_name, interrupt_event)
if self.entries.exists(path) and self.entries.is_expired(path):
self.entries.delete(path)
if not self.entries.exists(path):
self.entries.write(path, Entry())
entry = self.entries.get_value(path)
entry.set_is_file()
entry.size = data_len
entry.set_modified()
self.entries.write(path, entry)
self._add_to_parent_dir_listing(path)
return ret
def __get_size(self, fileobject):
pos = fileobject.tell()
fileobject.seek(0,2)
size = fileobject.tell()
fileobject.seek(pos, 0)
return size
def store_fileobject(self, fileobject, path, interrupt_event=None):
self.logger.debug("meta cache store_fileobject %s", path)
data_len = self.__get_size(fileobject)
try:
ret = self.store.store_fileobject(fileobject, path, interrupt_event)
finally:
fileobject.close()
if self.entries.exists(path) and self.entries.is_expired(path):
self.entries.delete(path)
if not self.entries.exists(path):
self.entries.write(path, Entry())
entry = self.entries.get_value(path)
entry.set_is_file()
entry.size = data_len
entry.set_modified()
self._add_to_parent_dir_listing(path)
self.entries.write(path, entry)
return ret
def delete(self, path, is_dir):
self.logger.debug("meta cache delete %s", path)
self.store.delete(path, is_dir)
self.entries.delete(path)
self._remove_from_parent_dir_listing(path)
def account_info(self):
if not self.store_metadata.exists('account_info'):
self.store_metadata.write('account_info', self.store.account_info())
return self.store_metadata.get_value('account_info')
def get_free_space(self):
return self.free_space_worker.get_free_bytes_in_remote_store()
def get_overall_space(self):
if not self.store_metadata.exists('overall_space') or self.store_metadata.is_expired('overall_space'):
self.store_metadata.write('overall_space', self.store.get_overall_space())
return self.store_metadata.get_value('overall_space')
def get_used_space(self):
if not self.store_metadata.exists('used_space') or self.store_metadata.is_expired('used_space'):
self.store_metadata.write('used_space', self.store.get_used_space())
return self.store_metadata.get_value('used_space')
def create_directory(self, directory):
self.logger.debug("meta cache create_directory %s", directory)
ret = self.store.create_directory(directory)
if self.entries.exists(directory) and self.entries.is_expired(directory):
self.entries.delete(directory)
if not self.entries.exists(directory):
self.entries.write(directory, Entry())
entry = self.entries.get_value(directory)
entry.set_is_dir()
entry.listing = []
entry.set_modified()
self.entries.write(directory, entry)
self._add_to_parent_dir_listing(directory)
return ret
def duplicate(self, path_to_src, path_to_dest):
self.logger.debug("meta cache duplicate %s to %s", path_to_src, path_to_dest)
ret = self.store.duplicate(path_to_src, path_to_dest)
if self.entries.exists(path_to_src) and self.entries.is_expired(path_to_src):
self.entries.delete(path_to_src)
if self.entries.exists(path_to_src):
entry = deepcopy(self.entries.get_value(path_to_src))
self.entries.write(path_to_dest, entry)
else:
self.entries.write(path_to_dest, Entry())
entry = self.entries.get_value(path_to_dest)
entry.set_modified()
self.entries.write(path_to_dest, entry)
self._add_to_parent_dir_listing(path_to_dest)
self.logger.debug("duplicated %s to %s", path_to_src, path_to_dest)
return ret
def move(self, path_to_src, path_to_dest):
self.logger.debug("meta cache move %s to %s", path_to_src, path_to_dest)
self.store.move(path_to_src, path_to_dest)
if self.entries.exists(path_to_dest) and self.entries.is_expired(path_to_src):
self.entries.delete(path_to_src)
if self.entries.exists(path_to_src):
entry = self.entries.get_value(path_to_src)
self.entries.write(path_to_dest, entry)
else:
self.entries.write(path_to_dest, Entry())
entry = self.entries.get_value(path_to_src)
entry.set_modified()
self.entries.write(path_to_dest, entry)
self.entries.delete(path_to_src)
self._remove_from_parent_dir_listing(path_to_src)
self._add_to_parent_dir_listing(path_to_dest)
def get_modified(self, path):
self.logger.debug("meta cache get_modified %s", path)
if self.entries.exists(path) and self.entries.is_expired(path):
self.entries.delete(path)
if self.entries.exists(path):
entry = self.entries.get_value(path)
if not entry.modified == None:
return entry.modified
modified = self.store.get_modified(path)
if not self.entries.exists(path):
self.entries.write(path, Entry())
entry = self.entries.get_value(path)
entry.set_modified(modified)
self.entries.write(path, entry)
return entry.modified
def get_directory_listing(self, directory):
self.logger.debug("meta cache get_directory_listing %s", directory)
if self.entries.exists(directory) and self.entries.is_expired(directory):
self.entries.delete(directory)
if self.entries.exists(directory):
entry = self.entries.get_value(directory)
if not entry.listing == None:
self.logger.debug("return cached listing %s", repr(entry.listing))
return list(entry.listing)
listing = self.store.get_directory_listing(directory)
self.logger.debug("meta cache caching %s", repr(listing))
if not self.entries.exists(directory):
self.entries.write(directory, Entry())
entry = self.entries.get_value(directory)
entry.listing = listing
self.entries.write(directory, entry)
self.logger.debug("asserted %s", repr(self.entries.get_value(directory).listing))
assert self.entries.get_value(directory).listing == entry.listing
return list(entry.listing)
def get_bytes(self, path):
self.logger.debug("meta cache get_bytes %s", path)
if self.entries.exists(path) and self.entries.is_expired(path):
self.entries.delete(path)
if self.entries.exists(path):
entry = self.entries.get_value(path)
if not entry.size == None:
return entry.size
size = self.store.get_bytes(path)
if not self.entries.exists(path):
self.entries.write(path, Entry())
entry = self.entries.get_value(path)
entry.size = size
self.entries.write(path, entry)
return entry.size
def exists(self, path):
self.logger.debug("meta cache exists %s", path)
if self.entries.exists(path) and self.entries.is_expired(path):
self.entries.delete(path)
if not self.entries.exists(path):
if self.store.exists(path):
self.entries.write(path, Entry())
return self.entries.exists(path)
def get_metadata(self, path):
self.logger.debug("meta cache get_metadata %s", path)
if self.entries.exists(path) and self.entries.is_expired(path):
self.logger.debug("1.1")
self.entries.delete(path)
self._add_parent_dir_listing(path)
if self._does_not_exist_in_parent_dir_listing(path):
raise NoSuchFilesytemObjectError(path,0)
if self.entries.exists(path):
entry = self.entries.get_value(path)
self.logger.debug("entry exists")
if not None in [entry.is_dir, entry.modified, entry.size]:
return {'is_dir': entry.is_dir, 'modified': entry.modified, 'bytes': entry.size}
self.logger.debug("meta cache get_metadata entry does not exist or is expired")
metadata = self.store.get_metadata(path)
entry = self._prepare_entry(path, metadata)
self.entries.write(path, entry)
if not entry.is_dir and isinstance(self.store, BulkGetMetadata):
self._prefetch_directory(os.path.dirname(path))
return {'is_dir': entry.is_dir, 'modified': entry.modified, 'bytes': entry.size}
def _prepare_entry(self, path, metadata):
if self.entries.exists(path):
entry = self.entries.get_value(path) #preserve listings
else:
entry = Entry()
if metadata['is_dir']:
entry.set_is_dir()
else:
entry.set_is_file()
entry.modified = metadata['modified']
entry.size = metadata['bytes']
return entry
def _prefetch_directory(self, path):
self.logger.debug("prefetch %s", path)
bulk = self.store.get_bulk_metadata(path)
bulk.items()
dict = {}
for path, metadata in bulk.items():
e = Entry()
dict[path] = self._prepare_entry(path,metadata)
try:
self.entries.bulk_write(dict)
except Exception,e:
import traceback
traceback.print_exc()
self.logger.debug("prefetch succeeded %s", path)
