def open(self, time_fn=None):
"""Opens NamedCaches for mutation operations, such as request or trim.
Only on caller can open the cache manager at a time. If the same thread
calls this function after opening it earlier, the call will deadlock.
time_fn is a function that returns timestamp (float) and used to take
timestamps when new caches are requested.
Returns a context manager that must be closed as soon as possible.
"""
state_path = os.path.join(self.root_dir, u'state.json')
with self._lock:
if os.path.isfile(state_path):
self._lru = lru.LRUDict.load(state_path)
else:
self._lru = lru.LRUDict()
if time_fn:
self._lru.time_fn = time_fn
try:
yield
finally:
file_path.ensure_tree(self.root_dir)
self._lru.save(state_path)
self._lru = None
def __init__(self, cache_dir, policies, hash_algo):
"""
Arguments:
cache_dir: directory where to place the cache.
policies: cache retention policies.
algo: hashing algorithm used.
"""
super(DiskCache, self).__init__()
self.cache_dir = cache_dir
self.policies = policies
self.hash_algo = hash_algo
self.state_file = os.path.join(cache_dir, self.STATE_FILE)
# All protected methods (starting with '_') except _path should be called
# with this lock locked.
self._lock = threading_utils.LockWithAssert()
self._lru = lru.LRUDict()
# Profiling values.
self._added = []
self._removed = []
self._free_disk = 0
with tools.Profiler('Setup'):
with self._lock:
self._load()
def test_basic_dict_funcs(self):
lru_dict = lru.LRUDict()
# Add a bunch.
data = {1: 'one', 2: 'two', 3: 'three'}
for k, v in data.items():
lru_dict.add(k, v)
# Check its there.
self.assert_same_data(lru_dict, data)
# Replace value.
lru_dict.add(1, 'one!!!')
data[1] = 'one!!!'
self.assert_same_data(lru_dict, data)
# Check pop works.
self.assertEqual(lru_dict.pop(2), 'two')
data.pop(2)
self.assert_same_data(lru_dict, data)
# Pop missing key.
with self.assertRaises(KeyError):
lru_dict.pop(2)
# Touch has no effect on set of keys and values.
lru_dict.touch(1)
self.assert_same_data(lru_dict, data)
# Touch fails on missing key.
with self.assertRaises(KeyError):
lru_dict.touch(22)
def __init__(self, cache_dir, policies, time_fn=None):
"""Initializes NamedCaches.
Arguments:
- cache_dir is a directory for persistent cache storage.
- policies is a CachePolicies instance.
- time_fn is a function that returns timestamp (float) and used to take
timestamps when new caches are requested. Used in unit tests.
"""
super(NamedCache, self).__init__(cache_dir)
self._policies = policies
# LRU {cache_name -> tuple(cache_location, size)}
self.state_file = os.path.join(cache_dir, self.STATE_FILE)
self._lru = lru.LRUDict()
if not fs.isdir(self.cache_dir):
fs.makedirs(self.cache_dir)
elif os.path.isfile(self.state_file):
try:
self._lru = lru.LRUDict.load(self.state_file)
except ValueError:
logging.exception('failed to load named cache state file')
logging.warning('deleting named caches')
file_path.rmtree(self.cache_dir)
with self._lock:
self._try_upgrade()
if time_fn:
self._lru.time_fn = time_fn
def __init__(self, cache_dir, policies, hash_algo, trim, time_fn=None):
"""
Arguments:
cache_dir: directory where to place the cache.
policies: CachePolicies instance, cache retention policies.
algo: hashing algorithm used.
trim: if True to enforce |policies| right away.
It can be done later by calling trim() explicitly.
"""
# All protected methods (starting with '_') except _path should be called
# with self._lock held.
super(DiskContentAddressedCache, self).__init__(cache_dir)
self.policies = policies
self.hash_algo = hash_algo
self.state_file = os.path.join(cache_dir, self.STATE_FILE)
# Items in a LRU lookup dict(digest: size).
self._lru = lru.LRUDict()
# Current cached free disk space. It is updated by self._trim().
file_path.ensure_tree(self.cache_dir)
self._free_disk = file_path.get_free_space(self.cache_dir)
# The first item in the LRU cache that must not be evicted during this run
# since it was referenced. All items more recent that _protected in the LRU
# cache are also inherently protected. It could be a set() of all items
# referenced but this increases memory usage without a use case.
self._protected = None
# Cleanup operations done by self._load(), if any.
self._operations = []
with tools.Profiler('Setup'):
with self._lock:
self._load(trim, time_fn)
def open(self, time_fn=None):
"""Opens NamedCaches for mutation operations, such as install.
Only one caller can open the cache manager at a time. If the same thread
calls this function after opening it earlier, the call will deadlock.
time_fn is a function that returns timestamp (float) and used to take
timestamps when new caches are requested.
Returns a context manager that must be closed as soon as possible.
"""
with self._lock:
state_path = os.path.join(self.root_dir, u'state.json')
assert self._lru is None, 'acquired lock, but self._lru is not None'
if os.path.isfile(state_path):
try:
self._lru = lru.LRUDict.load(state_path)
except ValueError:
logging.exception('failed to load named cache state file')
logging.warning('deleting named caches')
file_path.rmtree(self.root_dir)
self._lru = self._lru or lru.LRUDict()
if time_fn:
self._lru.time_fn = time_fn
try:
yield
finally:
file_path.ensure_tree(self.root_dir)
self._lru.save(state_path)
self._lru = None
def __init__(self, file_mode_mask=0500):
"""Args:
file_mode_mask: bit mask to AND file mode with. Default value will make
all mapped files to be read only.
"""
super(MemoryContentAddressedCache, self).__init__(None)
self._file_mode_mask = file_mode_mask
# Items in a LRU lookup dict(digest: size).
self._lru = lru.LRUDict()
def test_magic_methods_empty(self):
"""Tests __nonzero__, __iter, __len__, __getitem__ and __contains__."""
# Check for empty dict.
lru_dict = lru.LRUDict()
self.assertFalse(lru_dict)
self.assertEqual(len(lru_dict), 0)
self.assertFalse(1 in lru_dict)
self.assertFalse([i for i in lru_dict])
with self.assertRaises(KeyError):
_ = lru_dict[1]
def test_magic_methods_nonempty(self):
"""Tests __nonzero__, __iter, __len__, __getitem__ and __contains__."""
# Dict with one item.
lru_dict = lru.LRUDict()
lru_dict.add(1, 'one')
self.assertTrue(lru_dict)
self.assertEqual(len(lru_dict), 1)
self.assertTrue(1 in lru_dict)
self.assertFalse(2 in lru_dict)
self.assertTrue([i for i in lru_dict])
self.assertEqual('one', lru_dict[1])
def test_load_save(self):
def pairs(d):
return [(k, d[k]) for k in d]
def save_and_load(lru_dict):
handle, tmp_name = tempfile.mkstemp(prefix=u'lru_test')
os.close(handle)
try:
# Old format.
with open(tmp_name, 'w') as f:
json.dump(pairs(lru_dict), f)
loaded_old_format = lru_dict.load(tmp_name)
# Current format.
lru_dict.save(tmp_name)
loaded = lru_dict.load(tmp_name)
self.assertEqual(pairs(loaded), pairs(loaded_old_format))
return loaded
finally:
try:
os.unlink(tmp_name)
except OSError:
pass
data = [1, 2, 3]
# Edge case.
empty = save_and_load(lru.LRUDict())
self.assertFalse(empty)
# Normal flow.
lru_dict = save_and_load(self.prepare_lru_dict(data))
self.assert_order(lru_dict, data)
# After touches.
lru_dict = self.prepare_lru_dict(data)
lru_dict.touch(2)
lru_dict = save_and_load(lru_dict)
self.assert_order(lru_dict, [1, 3, 2])
# After pop.
lru_dict = self.prepare_lru_dict(data)
lru_dict.pop(2)
lru_dict = save_and_load(lru_dict)
self.assert_order(lru_dict, [1, 3])
# After add.
lru_dict = self.prepare_lru_dict(data)
lru_dict.add(4, 4)
lru_dict = save_and_load(lru_dict)
self.assert_order(lru_dict, data + [4])
def test_magic_methods(self):
# Check __nonzero__, __len__ and __contains__ for empty dict.
lru_dict = lru.LRUDict()
self.assertFalse(lru_dict)
self.assertEqual(len(lru_dict), 0)
self.assertFalse(1 in lru_dict)
# Dict with one item.
lru_dict.add(1, 'one')
self.assertTrue(lru_dict)
self.assertEqual(len(lru_dict), 1)
self.assertTrue(1 in lru_dict)
self.assertFalse(2 in lru_dict)
def __init__(self, cache_dir, policies, time_fn=None):
"""Initializes NamedCaches.
Arguments:
- cache_dir is a directory for persistent cache storage.
- policies is a CachePolicies instance.
- time_fn is a function that returns timestamp (float) and used to take
timestamps when new caches are requested. Used in unit tests.
"""
super(NamedCache, self).__init__(cache_dir)
self._policies = policies
# LRU {cache_name -> tuple(cache_location, size)}
self.state_file = os.path.join(cache_dir, self.STATE_FILE)
self._lru = lru.LRUDict()
if not fs.isdir(self.cache_dir):
fs.makedirs(self.cache_dir)
elif fs.isfile(self.state_file):
try:
self._lru = lru.LRUDict.load(self.state_file)
for _, size in self._lru.values():
if not isinstance(size, six.integer_types):
with open(self.state_file, 'r') as f:
logging.info('named cache state file: %s\n%s',
self.state_file, f.read())
raise ValueError("size is not integer: %s" % size)
except ValueError:
logging.exception(
'NamedCache: failed to load named cache state file; obliterating'
)
file_path.rmtree(self.cache_dir)
fs.makedirs(self.cache_dir)
self._lru = lru.LRUDict()
with self._lock:
self._try_upgrade()
if time_fn:
self._lru.time_fn = time_fn
def test_timestamp(self):
lru_dict = lru.LRUDict()
now = 0
lru_dict.time_fn = lambda: now
lru_dict.add('ka', 'va')
now += 1
lru_dict.add('kb', 'vb')
now += 1
self.assertEqual(lru_dict.get_oldest(), ('ka', ('va', 0)))
self.assertEqual(lru_dict.pop_oldest(), ('ka', ('va', 0)))
self.assertEqual(lru_dict.get_oldest(), ('kb', ('vb', 1)))
self.assertEqual(lru_dict.pop_oldest(), ('kb', ('vb', 1)))
def test_load_save(self):
def save_and_load(lru_dict):
handle, tmp_name = tempfile.mkstemp(prefix=u'lru_test')
os.close(handle)
try:
lru_dict.save(tmp_name)
return lru.LRUDict.load(tmp_name)
finally:
try:
os.unlink(tmp_name)
except OSError:
pass
data = [1, 2, 3]
# Edge case.
empty = save_and_load(lru.LRUDict())
self.assertFalse(empty)
# Normal flow.
lru_dict = save_and_load(self.prepare_lru_dict(data))
self.assert_order(lru_dict, data)
# After touches.
lru_dict = self.prepare_lru_dict(data)
lru_dict.touch(2)
lru_dict = save_and_load(lru_dict)
self.assert_order(lru_dict, [1, 3, 2])
# After pop.
lru_dict = self.prepare_lru_dict(data)
lru_dict.pop(2)
lru_dict = save_and_load(lru_dict)
self.assert_order(lru_dict, [1, 3])
# After add.
lru_dict = self.prepare_lru_dict(data)
lru_dict.add(4, 4)
lru_dict.batch_insert_oldest([(5, 5), (6, 6)])
lru_dict = save_and_load(lru_dict)
self.assert_order(lru_dict, [5, 6] + data + [4])
def prepare_lru_dict(keys):
"""Returns new LRUDict with given |keys| added one by one."""
lru_dict = lru.LRUDict()
for key in keys:
lru_dict.add(key, None)
return lru_dict
def test_load_save_empty(self): self.assertFalse(_save_and_load(lru.LRUDict()))
def test_transform(self):
lru_dict = lru.LRUDict()
lru_dict.add('ka', 'va')
lru_dict.add('kb', 'vb')
lru_dict.transform(lambda k, v: v + '*')
self.assert_same_data([('ka', 'va*'), ('kb', 'vb*')], lru_dict)
def _prepare_lru_dict(data):
"""Returns new LRUDict with given |keys| added one by one."""
lru_dict = lru.LRUDict()
for key, val in data:
lru_dict.add(key, val)
return lru_dict
def test_empty(self): self.assert_same_data([], lru.LRUDict())