def __init__(self):
# initialize state but ONLY if it hasn't already been initialized
if not hasattr(self, '_disk_lock'):
self._disk_lock = threading.Lock()
if not hasattr(self, '_write_lock'):
self._write_lock = threading.Lock()
if not hasattr(self, '_save_lock'):
self._save_lock = threading.Lock()
if not hasattr(self, '_objects'):
self._objects = {}
if not hasattr(self, '_dirty'):
self._dirty = False
if not hasattr(self, '_save_queue'):
self._save_queue = []
if not hasattr(self, '_pool'):
self._pool = ThreadPool(2)
self.cache_path = self._get_cache_path()
def __init__(self):
# initialize state but ONLY if it hasn't already been initialized
if not hasattr(self, '_disk_lock'):
self._disk_lock = threading.Lock()
if not hasattr(self, '_write_lock'):
self._write_lock = threading.Lock()
if not hasattr(self, '_save_lock'):
self._save_lock = threading.Lock()
if not hasattr(self, '_objects'):
self._objects = {}
if not hasattr(self, '_dirty'):
self._dirty = False
if not hasattr(self, '_save_queue'):
self._save_queue = []
if not hasattr(self, '_pool'):
self._pool = ThreadPool(2)
self.cache_path = self._get_cache_path()
class Cache(object):
'''
default cache object and definition
implements the shared functionality between the various caches
'''
def __new__(cls, *args, **kwargs):
# don't allow this class to be instantiated directly
if cls is Cache:
raise NotImplemented
return super(Cache, cls).__new__(cls)
def __init__(self):
# initialize state but ONLY if it hasn't already been initialized
if not hasattr(self, '_disk_lock'):
self._disk_lock = threading.Lock()
if not hasattr(self, '_write_lock'):
self._write_lock = threading.Lock()
if not hasattr(self, '_save_lock'):
self._save_lock = threading.Lock()
if not hasattr(self, '_objects'):
self._objects = {}
if not hasattr(self, '_dirty'):
self._dirty = False
if not hasattr(self, '_save_queue'):
self._save_queue = []
if not hasattr(self, '_pool'):
self._pool = ThreadPool(2)
self.cache_path = self._get_cache_path()
def get(self, key):
'''
retrieve the cached value for the corresponding key
raises CacheMiss if value has not been cached
:param key:
the key that the value has been stored under
'''
if key is None:
raise ValueError('key cannot be None')
try:
result = self._objects[key]
except KeyError:
# note: will raise CacheMiss if can't be found
result = self.load(key)
if result == _invalid_object:
raise CacheMiss('{0} is invalid'.format(key))
# return a copy of any objects
try:
if hasattr(result, '__dict__') or hasattr(result, '__slots__'):
result = copy.copy(result)
except:
pass
return result
def has(self, key):
'''
check if cache has a value for the corresponding key
:param key:
the key that the value has been stored under
'''
if key is None:
raise ValueError('key cannot be None')
return (key in self._objects and self._objects[key] != _invalid_object)
def set(self, key, obj):
'''
set the cache value for the given key
:param key:
the key to store the value under
:param obj:
the value to store; note that obj *must* be picklable
'''
if key is None:
raise ValueError('key cannot be None')
try:
pickle.dumps(obj, protocol=-1)
except pickle.PicklingError:
raise ValueError('obj must be picklable')
if isinstance(obj, list):
obj = tuple(obj)
elif isinstance(obj, dict):
obj = frozendict(obj)
elif isinstance(obj, set):
obj = frozenset(obj)
with self._write_lock:
self._objects[key] = obj
self._dirty = True
self._schedule_save()
def cache(self, key, func):
'''
convenience method to attempt to get the value from the cache and
generate the value if it hasn't been cached yet or the entry has
otherwise been invalidated
:param key:
the key to retrieve or set
:param func:
a callable that takes no arguments and when invoked will return
the proper value
'''
if key is None:
raise ValueError('key cannot be None')
try:
return self.get(key)
except:
result = func()
self.set(key, result)
return result
def invalidate(self, key=None):
'''
invalidates either this whole cache, a single entry or a list of
entries in this cache
:param key:
the key of the entry to invalidate; if None, the entire cache
will be invalidated
'''
def _invalidate(key):
try:
self._objects[key] = _invalid_object
except:
print('error occurred while invalidating {0}'.format(key))
traceback.print_exc()
with self._write_lock:
if key is None:
for k in self._objects.keys():
_invalidate(k)
else:
if isinstance(key, strbase):
_invalidate(key)
else:
for k in key:
_invalidate(k)
self._schedule_save()
def _get_cache_path(self):
return _global_cache_path()
def load(self, key=None):
'''
loads the value specified from the disk and stores it in the in-memory
cache
:param key:
the key to load from disk; if None, all entries in the cache
will be read from disk
'''
with self._write_lock:
if key is None:
for entry in os.listdir(self.cache_path):
if os.path.isfile(entry):
entry_name = os.path.basename[entry]
try:
self._objects[entry_name] = self._read(entry_name)
except:
print(
u'error while loading {0}'.format(entry_name))
else:
self._objects[key] = self._read(key)
if key is not None:
return self._objects[key]
def load_async(self, key=None):
'''
an async version of load; does the loading in a new thread
'''
self._pool.apply_async(self.load, key)
def _read(self, key):
file_path = os.path.join(self.cache_path, key)
with self._disk_lock:
try:
with open(file_path, 'rb') as f:
return pickle.load(f)
except:
raise CacheMiss(u'cannot read cache file {0}'.format(key))
def save(self, key=None):
'''
saves the cache entry specified to disk
:param key:
the entry to flush to disk; if None, all entries in the cache will
be written to disk
'''
if not self._dirty:
return
# lock is aquired here so that all keys being flushed reflect the
# same state; note that this blocks disk reads, but not cache reads
with self._disk_lock:
# operate on a stable copy of the object
with self._write_lock:
_objs = pickle.loads(pickle.dumps(self._objects, protocol=-1))
self._dirty = False
if key is None:
# remove all InvalidObjects
delete_keys = [k for k in _objs if _objs[k] == _invalid_object]
for k in delete_keys:
del _objs[k]
file_path = os.path.join(self.cache_path, key)
try:
os.path.remove(file_path)
except OSError:
pass
if _objs:
make_dirs(self.cache_path)
for k in _objs.keys():
try:
self._write(k, _objs)
except:
traceback.print_exc()
else:
# cache has been emptied, so remove it
try:
shutil.rmtree(self.cache_path)
except:
print('error while deleting {0}'.format(
self.cache_path))
traceback.print_exc()
elif key in _objs:
if _objs[key] == _invalid_object:
file_path = os.path.join(self.cache_path, key)
try:
os.path.remove(file_path)
except:
print('error while deleting {0}'.format(file_path))
traceback.print_exc()
else:
make_dirs(self.cache_path)
self._write(key, _objs)
def save_async(self, key=None):
'''
an async version of save; does the save in a new thread
'''
try:
self._pool.apply_async(self.save, key)
except ValueError:
pass
def _write(self, key, obj):
try:
_obj = obj[key]
except KeyError:
raise CacheMiss()
try:
with open(os.path.join(self.cache_path, key), 'wb') as f:
pickle.dump(_obj, f, protocol=-1)
except OSError:
print('error while writing to {0}'.format(key))
traceback.print_exc()
raise CacheMiss()
def _schedule_save(self):
with self._save_lock:
self._save_queue.append(0)
threading.Timer(0.5, self._debounce_save).start()
def _debounce_save(self):
with self._save_lock:
if len(self._save_queue) > 1:
self._save_queue.pop()
else:
self._save_queue = []
sublime.set_timeout(self.save_async, 0)
# ensure cache is saved to disk when removed from memory
def __del__(self):
self.save_async()
self._pool.terminate()
class Cache(object):
'''
default cache object and definition
implements the shared functionality between the various caches
'''
def __new__(cls, *args, **kwargs):
# don't allow this class to be instantiated directly
if cls is Cache:
raise NotImplemented
return super(Cache, cls).__new__(cls)
def __init__(self):
# initialize state but ONLY if it hasn't already been initialized
if not hasattr(self, '_disk_lock'):
self._disk_lock = threading.Lock()
if not hasattr(self, '_write_lock'):
self._write_lock = threading.Lock()
if not hasattr(self, '_save_lock'):
self._save_lock = threading.Lock()
if not hasattr(self, '_objects'):
self._objects = {}
if not hasattr(self, '_dirty'):
self._dirty = False
if not hasattr(self, '_save_queue'):
self._save_queue = []
if not hasattr(self, '_pool'):
self._pool = ThreadPool(2)
self.cache_path = self._get_cache_path()
def get(self, key):
'''
retrieve the cached value for the corresponding key
raises CacheMiss if value has not been cached
:param key:
the key that the value has been stored under
'''
if key is None:
raise ValueError('key cannot be None')
try:
result = self._objects[key]
except KeyError:
# note: will raise CacheMiss if can't be found
result = self.load(key)
if result == _invalid_object:
raise CacheMiss('{0} is invalid'.format(key))
# return a copy of any objects
try:
if hasattr(result, '__dict__') or hasattr(result, '__slots__'):
result = copy.copy(result)
except:
pass
return result
def has(self, key):
'''
check if cache has a value for the corresponding key
:param key:
the key that the value has been stored under
'''
if key is None:
raise ValueError('key cannot be None')
return (
key in self._objects and
self._objects[key] != _invalid_object
)
def set(self, key, obj):
'''
set the cache value for the given key
:param key:
the key to store the value under
:param obj:
the value to store; note that obj *must* be picklable
'''
if key is None:
raise ValueError('key cannot be None')
try:
pickle.dumps(obj, protocol=-1)
except pickle.PicklingError:
raise ValueError('obj must be picklable')
if isinstance(obj, list):
obj = tuple(obj)
elif isinstance(obj, dict):
obj = frozendict(obj)
elif isinstance(obj, set):
obj = frozenset(obj)
with self._write_lock:
self._objects[key] = obj
self._dirty = True
self._schedule_save()
def cache(self, key, func):
'''
convenience method to attempt to get the value from the cache and
generate the value if it hasn't been cached yet or the entry has
otherwise been invalidated
:param key:
the key to retrieve or set
:param func:
a callable that takes no arguments and when invoked will return
the proper value
'''
if key is None:
raise ValueError('key cannot be None')
try:
return self.get(key)
except:
result = func()
self.set(key, result)
return result
def invalidate(self, key=None):
'''
invalidates either this whole cache, a single entry or a list of
entries in this cache
:param key:
the key of the entry to invalidate; if None, the entire cache
will be invalidated
'''
def _invalidate(key):
try:
self._objects[key] = _invalid_object
except:
print('error occurred while invalidating {0}'.format(key))
traceback.print_exc()
with self._write_lock:
if key is None:
for k in self._objects.keys():
_invalidate(k)
else:
if isinstance(key, strbase):
_invalidate(key)
else:
for k in key:
_invalidate(k)
self._schedule_save()
def _get_cache_path(self):
return _global_cache_path()
def load(self, key=None):
'''
loads the value specified from the disk and stores it in the in-memory
cache
:param key:
the key to load from disk; if None, all entries in the cache
will be read from disk
'''
with self._write_lock:
if key is None:
for entry in os.listdir(self.cache_path):
if os.path.isfile(entry):
entry_name = os.path.basename[entry]
try:
self._objects[entry_name] = self._read(entry_name)
except:
print(
u'error while loading {0}'.format(entry_name))
else:
self._objects[key] = self._read(key)
if key is not None:
return self._objects[key]
def load_async(self, key=None):
'''
an async version of load; does the loading in a new thread
'''
self._pool.apply_async(self.load, key)
def _read(self, key):
file_path = os.path.join(self.cache_path, key)
with self._disk_lock:
try:
with open(file_path, 'rb') as f:
return pickle.load(f)
except:
raise CacheMiss(u'cannot read cache file {0}'.format(key))
def save(self, key=None):
'''
saves the cache entry specified to disk
:param key:
the entry to flush to disk; if None, all entries in the cache will
be written to disk
'''
if not self._dirty:
return
# lock is aquired here so that all keys being flushed reflect the
# same state; note that this blocks disk reads, but not cache reads
with self._disk_lock:
# operate on a stable copy of the object
with self._write_lock:
_objs = pickle.loads(pickle.dumps(self._objects, protocol=-1))
self._dirty = False
if key is None:
# remove all InvalidObjects
delete_keys = [
k for k in _objs if _objs[k] == _invalid_object
]
for k in delete_keys:
del _objs[k]
file_path = os.path.join(self.cache_path, key)
try:
os.path.remove(file_path)
except OSError:
pass
if _objs:
make_dirs(self.cache_path)
for k in _objs.keys():
try:
self._write(k, _objs)
except:
traceback.print_exc()
else:
# cache has been emptied, so remove it
try:
shutil.rmtree(self.cache_path)
except:
print(
'error while deleting {0}'.format(self.cache_path))
traceback.print_exc()
elif key in _objs:
if _objs[key] == _invalid_object:
file_path = os.path.join(self.cache_path, key)
try:
os.path.remove(file_path)
except:
print('error while deleting {0}'.format(file_path))
traceback.print_exc()
else:
make_dirs(self.cache_path)
self._write(key, _objs)
def save_async(self, key=None):
'''
an async version of save; does the save in a new thread
'''
try:
self._pool.apply_async(self.save, key)
except ValueError:
pass
def _write(self, key, obj):
try:
_obj = obj[key]
except KeyError:
raise CacheMiss()
try:
with open(os.path.join(self.cache_path, key), 'wb') as f:
pickle.dump(_obj, f, protocol=-1)
except OSError:
print('error while writing to {0}'.format(key))
traceback.print_exc()
raise CacheMiss()
def _schedule_save(self):
with self._save_lock:
self._save_queue.append(0)
threading.Timer(0.5, self._debounce_save).start()
def _debounce_save(self):
with self._save_lock:
if len(self._save_queue) > 1:
self._save_queue.pop()
else:
self._save_queue = []
sublime.set_timeout(self.save_async, 0)
# ensure cache is saved to disk when removed from memory
def __del__(self):
self.save_async()
self._pool.terminate()