def __init__(self, uncompressed_max_bytes=30*MiB, compressed_max_bytes=20*MiB):
self.uncompressed_max_bytes = uncompressed_max_bytes
self.compressed_max_bytes = compressed_max_bytes
# Note: We use dulwich's LRU cache to store the tagsfile paths here,
# but we could easily replace it by any other (LRU) cache implementation.
self._uncompressed_cache = LRUSizeCache(uncompressed_max_bytes, compute_size=os.path.getsize)
self._compressed_cache = LRUSizeCache(compressed_max_bytes, compute_size=os.path.getsize)
self._clearing = False
self._lock = threading.Lock()
def __init__(self,
uncompressed_max_bytes=30 * MiB,
compressed_max_bytes=20 * MiB):
self.uncompressed_max_bytes = uncompressed_max_bytes
self.compressed_max_bytes = compressed_max_bytes
# Note: We use dulwich's LRU cache to store the tagsfile paths here,
# but we could easily replace it by any other (LRU) cache implementation.
self._uncompressed_cache = LRUSizeCache(uncompressed_max_bytes,
compute_size=os.path.getsize)
self._compressed_cache = LRUSizeCache(compressed_max_bytes,
compute_size=os.path.getsize)
self._clearing = False
self._lock = threading.Lock()
def __init__(self, scon, filename):
""" Initialize a SwiftPackReader
:param scon: a `SwiftConnector` instance
:param filename: the pack filename
"""
self.scon = scon
self._filename = filename
self._header_size = 12
headers = self.scon.get_object_stat(self._filename)
self.pack_length = int(headers['content-length'])
pack_reader = SwiftPackReader(self.scon, self._filename,
self.pack_length)
(version, self._num_objects) = read_pack_header(pack_reader.read)
self._offset_cache = LRUSizeCache(1024*1024*self.scon.cache_length,
compute_size=_compute_object_size)
self.pack = None
def __init__(self, filename):
"""Create a PackData object that represents the pack in the given filename.
The file must exist and stay readable until the object is disposed of. It
must also stay the same size. It will be mapped whenever needed.
Currently there is a restriction on the size of the pack as the python
mmap implementation is flawed.
"""
self._filename = filename
assert os.path.exists(filename), "%s is not a packfile" % filename
self._size = os.path.getsize(filename)
self._header_size = 12
assert self._size >= self._header_size, "%s is too small for a packfile (%d < %d)" % (
filename, self._size, self._header_size)
self._file = open(self._filename, 'rb')
self._read_header()
self._offset_cache = LRUSizeCache(1024 * 1024 * 20,
compute_size=_compute_object_size)
class CTagsCache(object):
"""A ctags cache. Both uncompressed and compressed entries are kept in
temporary files created by `tempfile.mkstemp` which are deleted from disk
when the Python interpreter is shut down.
:param uncompressed_max_bytes: Maximum size of the uncompressed cache sector
:param compressed_max_bytes: Maximum size of the compressed cache sector
The lifecycle of a cache entry is as follows.
- When first created, a tagsfile is put into the uncompressed cache sector.
- When free space is required for other uncompressed tagsfiles, it may be
moved to the compressed cache sector. Gzip is used to compress the tagsfile.
- When free space is required for other compressed tagsfiles, it may be
evicted from the cache entirely.
- When the tagsfile is requested and it's in the compressed cache sector,
it is moved back to the uncompressed sector prior to using it.
"""
def __init__(self,
uncompressed_max_bytes=30 * MiB,
compressed_max_bytes=20 * MiB):
self.uncompressed_max_bytes = uncompressed_max_bytes
self.compressed_max_bytes = compressed_max_bytes
# Note: We use dulwich's LRU cache to store the tagsfile paths here,
# but we could easily replace it by any other (LRU) cache implementation.
self._uncompressed_cache = LRUSizeCache(uncompressed_max_bytes,
compute_size=os.path.getsize)
self._compressed_cache = LRUSizeCache(compressed_max_bytes,
compute_size=os.path.getsize)
self._clearing = False
self._lock = threading.Lock()
def __del__(self):
self.clear()
def clear(self):
"""Clear both the uncompressed and compressed caches."""
# Don't waste time moving tagsfiles from uncompressed to compressed cache,
# but remove them directly instead:
self._clearing = True
self._uncompressed_cache.clear()
self._compressed_cache.clear()
self._clearing = False
def get_tagsfile(self, git_repo_path, git_rev):
"""Get the ctags tagsfile for the given Git repository and revision.
- If the tagsfile is still in cache, and in uncompressed form, return it
without any further cost.
- If the tagsfile is still in cache, but in compressed form, uncompress
it, put it into uncompressed space, and return the uncompressed version.
- If the tagsfile isn't in cache at all, create it, put it into
uncompressed cache and return the newly created version.
"""
# Always require full SHAs
assert len(git_rev) == 40
# Avoiding race conditions, The Sledgehammer Way
with self._lock:
if git_rev in self._uncompressed_cache:
return self._uncompressed_cache[git_rev]
if git_rev in self._compressed_cache:
compressed_tagsfile_path = self._compressed_cache[git_rev]
uncompressed_tagsfile_path = uncompress_tagsfile(
compressed_tagsfile_path)
self._compressed_cache._remove_node(
self._compressed_cache._cache[git_rev])
else:
# Not in cache.
uncompressed_tagsfile_path = create_tagsfile(
git_repo_path, git_rev)
self._uncompressed_cache.add(git_rev, uncompressed_tagsfile_path,
self._clear_uncompressed_entry)
return uncompressed_tagsfile_path
def _clear_uncompressed_entry(self, git_rev, uncompressed_tagsfile_path):
"""Called by LRUSizeCache whenever an entry is to be evicted from
uncompressed cache.
Most of the times this happens when space is needed
in uncompressed cache, in which case we move the tagsfile to compressed
cache. When clearing the cache, we don't bother moving entries to
uncompressed space; we delete them directly instead.
"""
if not self._clearing:
# If we're clearing the whole cache, don't waste time moving tagsfiles
# from uncompressed to compressed cache, but remove them directly instead.
self._compressed_cache.add(
git_rev, compress_tagsfile(uncompressed_tagsfile_path),
self._clear_compressed_entry)
delete_tagsfile(uncompressed_tagsfile_path)
def _clear_compressed_entry(self, git_rev, compressed_tagsfile_path):
"""Called by LRUSizeCache whenever an entry to be evicted from
compressed cache.
This happens when space is needed for new compressed
tagsfiles. We delete the evictee from the cache entirely.
"""
delete_tagsfile(compressed_tagsfile_path)
class CTagsCache(object):
"""A ctags cache. Both uncompressed and compressed entries are kept in
temporary files created by `tempfile.mkstemp` which are deleted from disk
when the Python interpreter is shut down.
:param uncompressed_max_bytes: Maximum size of the uncompressed cache sector
:param compressed_max_bytes: Maximum size of the compressed cache sector
The lifecycle of a cache entry is as follows.
- When first created, a tagsfile is put into the uncompressed cache sector.
- When free space is required for other uncompressed tagsfiles, it may be
moved to the compressed cache sector. Gzip is used to compress the tagsfile.
- When free space is required for other compressed tagsfiles, it may be
evicted from the cache entirely.
- When the tagsfile is requested and it's in the compressed cache sector,
it is moved back to the uncompressed sector prior to using it.
"""
def __init__(self, uncompressed_max_bytes=30*MiB, compressed_max_bytes=20*MiB):
self.uncompressed_max_bytes = uncompressed_max_bytes
self.compressed_max_bytes = compressed_max_bytes
# Note: We use dulwich's LRU cache to store the tagsfile paths here,
# but we could easily replace it by any other (LRU) cache implementation.
self._uncompressed_cache = LRUSizeCache(uncompressed_max_bytes, compute_size=os.path.getsize)
self._compressed_cache = LRUSizeCache(compressed_max_bytes, compute_size=os.path.getsize)
self._clearing = False
self._lock = threading.Lock()
def __del__(self):
self.clear()
def clear(self):
"""Clear both the uncompressed and compressed caches."""
# Don't waste time moving tagsfiles from uncompressed to compressed cache,
# but remove them directly instead:
self._clearing = True
self._uncompressed_cache.clear()
self._compressed_cache.clear()
self._clearing = False
def get_tagsfile(self, git_repo_path, git_rev):
"""Get the ctags tagsfile for the given Git repository and revision.
- If the tagsfile is still in cache, and in uncompressed form, return it
without any further cost.
- If the tagsfile is still in cache, but in compressed form, uncompress
it, put it into uncompressed space, and return the uncompressed version.
- If the tagsfile isn't in cache at all, create it, put it into
uncompressed cache and return the newly created version.
"""
# Always require full SHAs
assert len(git_rev) == 40
# Avoiding race conditions, The Sledgehammer Way
with self._lock:
if git_rev in self._uncompressed_cache:
return self._uncompressed_cache[git_rev]
if git_rev in self._compressed_cache:
compressed_tagsfile_path = self._compressed_cache[git_rev]
uncompressed_tagsfile_path = uncompress_tagsfile(compressed_tagsfile_path)
self._compressed_cache._remove_node(self._compressed_cache._cache[git_rev])
else:
# Not in cache.
uncompressed_tagsfile_path = create_tagsfile(git_repo_path, git_rev)
self._uncompressed_cache.add(git_rev, uncompressed_tagsfile_path,
self._clear_uncompressed_entry)
return uncompressed_tagsfile_path
def _clear_uncompressed_entry(self, git_rev, uncompressed_tagsfile_path):
"""Called by LRUSizeCache whenever an entry is to be evicted from
uncompressed cache.
Most of the times this happens when space is needed
in uncompressed cache, in which case we move the tagsfile to compressed
cache. When clearing the cache, we don't bother moving entries to
uncompressed space; we delete them directly instead.
"""
if not self._clearing:
# If we're clearing the whole cache, don't waste time moving tagsfiles
# from uncompressed to compressed cache, but remove them directly instead.
self._compressed_cache.add(git_rev, compress_tagsfile(uncompressed_tagsfile_path),
self._clear_compressed_entry)
delete_tagsfile(uncompressed_tagsfile_path)
def _clear_compressed_entry(self, git_rev, compressed_tagsfile_path):
"""Called by LRUSizeCache whenever an entry to be evicted from
compressed cache.
This happens when space is needed for new compressed
tagsfiles. We delete the evictee from the cache entirely.
"""
delete_tagsfile(compressed_tagsfile_path)
