def test_memory_reduce_size(tmpdir):
memory, _, _ = _setup_toy_cache(tmpdir)
cachedir = memory.cachedir
ref_cache_items = _get_cache_items(cachedir)
# By default memory.bytes_limit is None and reduce_size is a noop
memory.reduce_size()
cache_items = _get_cache_items(cachedir)
assert sorted(ref_cache_items) == sorted(cache_items)
# No cache items deleted if bytes_limit greater than the size of
# the cache
memory.bytes_limit = '1M'
memory.reduce_size()
cache_items = _get_cache_items(cachedir)
assert sorted(ref_cache_items) == sorted(cache_items)
# bytes_limit is set so that only two cache items are kept
memory.bytes_limit = '3K'
memory.reduce_size()
cache_items = _get_cache_items(cachedir)
assert set.issubset(set(cache_items), set(ref_cache_items))
assert len(cache_items) == 2
# bytes_limit set so that no cache item is kept
bytes_limit_too_small = 500
memory.bytes_limit = bytes_limit_too_small
memory.reduce_size()
cache_items = _get_cache_items(cachedir)
assert cache_items == []
def test_memory_reduce_size(tmpdir):
mem, _, _ = _setup_temporary_cache_folder(tmpdir)
cachedir = mem.cachedir
ref_cache_items = _get_cache_items(cachedir)
# By default mem.bytes_limit is None and reduce_size is a noop
mem.reduce_size()
cache_items = _get_cache_items(cachedir)
assert sorted(ref_cache_items) == sorted(cache_items)
# No cache items deleted if bytes_limit greater than the size of
# the cache
mem.bytes_limit = '1M'
mem.reduce_size()
cache_items = _get_cache_items(cachedir)
assert sorted(ref_cache_items) == sorted(cache_items)
# bytes_limit is set so that only two cache items are kept
mem.bytes_limit = '3K'
mem.reduce_size()
cache_items = _get_cache_items(cachedir)
assert set.issubset(set(cache_items), set(ref_cache_items))
assert len(cache_items) == 2
# bytes_limit set so that no cache item is kept
bytes_limit_too_small = 500
mem.bytes_limit = bytes_limit_too_small
mem.reduce_size()
cache_items = _get_cache_items(cachedir)
assert cache_items == []
def test_memory_reduce_size():
mem, _, _ = _setup_temporary_cache_folder()
cachedir = mem.cachedir
ref_cache_items = _get_cache_items(cachedir)
# By default mem.bytes_limit is None and reduce_size is a noop
mem.reduce_size()
cache_items = _get_cache_items(cachedir)
nose.tools.assert_equal(sorted(ref_cache_items), sorted(cache_items))
# No cache items deleted if bytes_limit greater than the size of
# the cache
mem.bytes_limit = '1M'
mem.reduce_size()
cache_items = _get_cache_items(cachedir)
nose.tools.assert_equal(sorted(ref_cache_items), sorted(cache_items))
# bytes_limit is set so that only two cache items are kept
mem.bytes_limit = '3K'
mem.reduce_size()
cache_items = _get_cache_items(cachedir)
nose.tools.assert_true(set.issubset(set(cache_items),
set(ref_cache_items)))
nose.tools.assert_equal(len(cache_items), 2)
# bytes_limit set so that no cache item is kept
bytes_limit_too_small = 500
mem.bytes_limit = bytes_limit_too_small
mem.reduce_size()
cache_items = _get_cache_items(cachedir)
nose.tools.assert_equal(cache_items, [])
def test_memory_reduce_size():
mem, _, _ = _setup_temporary_cache_folder()
cachedir = mem.cachedir
ref_cache_items = _get_cache_items(cachedir)
# By default mem.bytes_limit is None and reduce_size is a noop
mem.reduce_size()
cache_items = _get_cache_items(cachedir)
nose.tools.assert_equal(sorted(ref_cache_items),
sorted(cache_items))
# No cache items deleted if bytes_limit greater than the size of
# the cache
mem.bytes_limit = '1M'
mem.reduce_size()
cache_items = _get_cache_items(cachedir)
nose.tools.assert_equal(sorted(ref_cache_items),
sorted(cache_items))
# bytes_limit is set so that only two cache items are kept
mem.bytes_limit = '3K'
mem.reduce_size()
cache_items = _get_cache_items(cachedir)
nose.tools.assert_true(set.issubset(set(cache_items),
set(ref_cache_items)))
nose.tools.assert_equal(len(cache_items), 2)
# bytes_limit set so that no cache item is kept
bytes_limit_too_small = 500
mem.bytes_limit = bytes_limit_too_small
mem.reduce_size()
cache_items = _get_cache_items(cachedir)
nose.tools.assert_equal(cache_items, [])
def test__get_cache_items_to_delete(tmpdir):
mem, expected_hash_cachedirs, _ = _setup_temporary_cache_folder(tmpdir)
cachedir = mem.cachedir
cache_items = _get_cache_items(cachedir)
# bytes_limit set to keep only one cache item (each hash cache
# folder is about 1000 bytes + metadata)
cache_items_to_delete = _get_cache_items_to_delete(cachedir, '2K')
nb_hashes = len(expected_hash_cachedirs)
assert set.issubset(set(cache_items_to_delete), set(cache_items))
assert len(cache_items_to_delete) == nb_hashes - 1
# Sanity check bytes_limit=2048 is the same as bytes_limit='2K'
cache_items_to_delete_2048b = _get_cache_items_to_delete(cachedir, 2048)
assert sorted(cache_items_to_delete) == sorted(cache_items_to_delete_2048b)
# bytes_limit greater than the size of the cache
cache_items_to_delete_empty = _get_cache_items_to_delete(cachedir, '1M')
assert cache_items_to_delete_empty == []
# All the cache items need to be deleted
bytes_limit_too_small = 500
cache_items_to_delete_500b = _get_cache_items_to_delete(
cachedir, bytes_limit_too_small)
assert set(cache_items_to_delete_500b), set(cache_items)
# Test LRU property: surviving cache items should all have a more
# recent last_access that the ones that have been deleted
cache_items_to_delete_6000b = _get_cache_items_to_delete(cachedir, 6000)
surviving_cache_items = set(cache_items).difference(
cache_items_to_delete_6000b)
assert (max(ci.last_access for ci in cache_items_to_delete_6000b) <=
min(ci.last_access for ci in surviving_cache_items))
def test__get_cache_items(tmpdir):
mem, expected_hash_cachedirs, _ = _setup_temporary_cache_folder(tmpdir)
cachedir = mem.cachedir
cache_items = _get_cache_items(cachedir)
hash_cachedirs = [ci.path for ci in cache_items]
assert set(hash_cachedirs) == set(expected_hash_cachedirs)
def get_files_size(directory):
full_paths = [os.path.join(directory, fn)
for fn in os.listdir(directory)]
return sum(os.path.getsize(fp) for fp in full_paths)
expected_hash_cache_sizes = [get_files_size(hash_dir)
for hash_dir in hash_cachedirs]
hash_cache_sizes = [ci.size for ci in cache_items]
assert hash_cache_sizes == expected_hash_cache_sizes
output_filenames = [os.path.join(hash_dir, 'output.pkl')
for hash_dir in hash_cachedirs]
expected_last_accesses = [
datetime.datetime.fromtimestamp(os.path.getatime(fn))
for fn in output_filenames]
last_accesses = [ci.last_access for ci in cache_items]
assert last_accesses == expected_last_accesses
def test__get_cache_items_to_delete(tmpdir):
memory, expected_hash_cachedirs, _ = _setup_toy_cache(tmpdir)
cachedir = memory.cachedir
cache_items = _get_cache_items(cachedir)
# bytes_limit set to keep only one cache item (each hash cache
# folder is about 1000 bytes + metadata)
cache_items_to_delete = _get_cache_items_to_delete(cachedir, '2K')
nb_hashes = len(expected_hash_cachedirs)
assert set.issubset(set(cache_items_to_delete), set(cache_items))
assert len(cache_items_to_delete) == nb_hashes - 1
# Sanity check bytes_limit=2048 is the same as bytes_limit='2K'
cache_items_to_delete_2048b = _get_cache_items_to_delete(cachedir, 2048)
assert sorted(cache_items_to_delete) == sorted(cache_items_to_delete_2048b)
# bytes_limit greater than the size of the cache
cache_items_to_delete_empty = _get_cache_items_to_delete(cachedir, '1M')
assert cache_items_to_delete_empty == []
# All the cache items need to be deleted
bytes_limit_too_small = 500
cache_items_to_delete_500b = _get_cache_items_to_delete(
cachedir, bytes_limit_too_small)
assert set(cache_items_to_delete_500b), set(cache_items)
# Test LRU property: surviving cache items should all have a more
# recent last_access that the ones that have been deleted
cache_items_to_delete_6000b = _get_cache_items_to_delete(cachedir, 6000)
surviving_cache_items = set(cache_items).difference(
cache_items_to_delete_6000b)
assert (max(ci.last_access for ci in cache_items_to_delete_6000b) <= min(
ci.last_access for ci in surviving_cache_items))
def test__get_cache_items(tmpdir):
memory, expected_hash_cachedirs, _ = _setup_toy_cache(tmpdir)
cachedir = memory.cachedir
cache_items = _get_cache_items(cachedir)
hash_cachedirs = [ci.path for ci in cache_items]
assert set(hash_cachedirs) == set(expected_hash_cachedirs)
def get_files_size(directory):
full_paths = [
os.path.join(directory, fn) for fn in os.listdir(directory)
]
return sum(os.path.getsize(fp) for fp in full_paths)
expected_hash_cache_sizes = [
get_files_size(hash_dir) for hash_dir in hash_cachedirs
]
hash_cache_sizes = [ci.size for ci in cache_items]
assert hash_cache_sizes == expected_hash_cache_sizes
output_filenames = [
os.path.join(hash_dir, 'output.pkl') for hash_dir in hash_cachedirs
]
expected_last_accesses = [
datetime.datetime.fromtimestamp(os.path.getatime(fn))
for fn in output_filenames
]
last_accesses = [ci.last_access for ci in cache_items]
assert last_accesses == expected_last_accesses
def test__get_cache_items():
mem, expected_hash_cachedirs, _ = _setup_temporary_cache_folder()
cachedir = mem.cachedir
cache_items = _get_cache_items(cachedir)
hash_cachedirs = [ci.path for ci in cache_items]
nose.tools.assert_equal(set(hash_cachedirs), set(expected_hash_cachedirs))
def get_files_size(directory):
full_paths = [
os.path.join(directory, fn) for fn in os.listdir(directory)
]
return sum(os.path.getsize(fp) for fp in full_paths)
expected_hash_cache_sizes = [
get_files_size(hash_dir) for hash_dir in hash_cachedirs
]
hash_cache_sizes = [ci.size for ci in cache_items]
nose.tools.assert_equal(hash_cache_sizes, expected_hash_cache_sizes)
output_filenames = [
os.path.join(hash_dir, 'output.pkl') for hash_dir in hash_cachedirs
]
expected_last_accesses = [
datetime.datetime.fromtimestamp(os.path.getatime(fn))
for fn in output_filenames
]
last_accesses = [ci.last_access for ci in cache_items]
nose.tools.assert_equal(last_accesses, expected_last_accesses)
def test_memory_recomputes_after_an_error_why_loading_results(tmpdir,
monkeypatch):
memory = Memory(tmpdir.strpath)
def func(arg):
# This makes sure that the timestamp returned by two calls of
# func are different. This is needed on Windows where
# time.time resolution may not be accurate enough
time.sleep(0.01)
return arg, time.time()
cached_func = memory.cache(func)
input_arg = 'arg'
arg, timestamp = cached_func(input_arg)
# Make sure the function is correctly cached
assert arg == input_arg
# Corrupting output.pkl to make sure that an error happens when
# loading the cached result
single_cache_item, = _get_cache_items(memory.cachedir)
output_filename = os.path.join(single_cache_item.path, 'output.pkl')
with open(output_filename, 'w') as f:
f.write('garbage')
recorded_warnings = []
def append_to_record(item):
recorded_warnings.append(item)
# Make sure that corrupting the file causes recomputation and that
# a warning is issued. Need monkeypatch because pytest does not
# capture stdlib logging output (see
# https://github.com/pytest-dev/pytest/issues/2079)
monkeypatch.setattr(cached_func, 'warn', append_to_record)
recomputed_arg, recomputed_timestamp = cached_func(arg)
assert len(recorded_warnings) == 1
exception_msg = 'Exception while loading results'
assert exception_msg in recorded_warnings[0]
assert recomputed_arg == arg
assert recomputed_timestamp > timestamp
def test__get_cache_items_to_delete():
mem, expected_hash_cachedirs, _ = _setup_temporary_cache_folder()
cachedir = mem.cachedir
cache_items = _get_cache_items(cachedir)
# bytes_limit set to keep only one cache item (each hash cache
# folder is about 1000 bytes + metadata)
cache_items_to_delete = _get_cache_items_to_delete(cachedir, '2K')
nb_hashes = len(expected_hash_cachedirs)
nose.tools.assert_true(
set.issubset(set(cache_items_to_delete), set(cache_items)))
nose.tools.assert_equal(len(cache_items_to_delete), nb_hashes - 1)
# Sanity check bytes_limit=2048 is the same as bytes_limit='2K'
cache_items_to_delete_2048b = _get_cache_items_to_delete(cachedir, 2048)
nose.tools.assert_equal(sorted(cache_items_to_delete),
sorted(cache_items_to_delete_2048b))
# bytes_limit greater than the size of the cache
cache_items_to_delete_empty = _get_cache_items_to_delete(cachedir, '1M')
nose.tools.assert_equal(cache_items_to_delete_empty, [])
# All the cache items need to be deleted
bytes_limit_too_small = 500
cache_items_to_delete_500b = _get_cache_items_to_delete(
cachedir, bytes_limit_too_small)
nose.tools.assert_true(set(cache_items_to_delete_500b), set(cache_items))
# Test LRU property: surviving cache items should all have a more
# recent last_access that the ones that have been deleted
cache_items_to_delete_6000b = _get_cache_items_to_delete(cachedir, 6000)
surviving_cache_items = set(cache_items).difference(
cache_items_to_delete_6000b)
nose.tools.assert_true(
max(ci.last_access for ci in cache_items_to_delete_6000b) <= min(
ci.last_access for ci in surviving_cache_items))