def test_decoder_cache_shrink_threadsafe(monkeypatch, tmpdir):
"""Tests that shrink handles files deleted by other processes."""
cache_dir = str(tmpdir)
solver_mock = SolverMock()
another_solver = SolverMock('another_solver')
cache = DecoderCache(cache_dir=cache_dir)
cache.wrap_solver(solver_mock)(**get_solver_test_args())
limit = cache.get_size()
# Ensure differing time stamps (depending on the file system the timestamp
# resolution might be as bad as 1 day).
for filename in os.listdir(cache.cache_dir):
path = os.path.join(cache.cache_dir, filename)
timestamp = os.stat(path).st_atime
timestamp -= 60 * 60 * 24 * 2 # 2 days
os.utime(path, (timestamp, timestamp))
cache.wrap_solver(another_solver)(**get_solver_test_args())
cache_size = cache.get_size_in_bytes()
assert cache_size > 0
def raise_file_not_found(*args, **kwargs):
raise OSError(errno.ENOENT, "File not found.")
monkeypatch.setattr(cache, 'get_size_in_bytes', lambda: cache_size)
monkeypatch.setattr('os.stat', raise_file_not_found)
monkeypatch.setattr('os.remove', raise_file_not_found)
monkeypatch.setattr('os.unlink', raise_file_not_found)
cache.shrink(limit)
def test_shrink_does_not_fail_if_lock_cannot_be_acquired(tmpdir):
cache = DecoderCache(cache_dir=str(tmpdir))
cache._index._lock.timeout = 1.
with cache:
cache.wrap_solver(SolverMock())(**get_solver_test_args())
with cache._index._lock:
cache.shrink(limit=0)
def test_decoder_cache_shrinking(tmpdir):
cache_dir = str(tmpdir)
solver_mock = SolverMock()
another_solver = SolverMock('another_solver')
cache = DecoderCache(cache_dir=cache_dir)
cache.wrap_solver(solver_mock)(**get_solver_test_args())
# Ensure differing time stamps (depending on the file system the timestamp
# resolution might be as bad as 1 day).
for path in cache.get_files():
timestamp = os.stat(path).st_atime
timestamp -= 60 * 60 * 24 * 2 # 2 days
os.utime(path, (timestamp, timestamp))
cache.wrap_solver(another_solver)(**get_solver_test_args())
cache_size = cache.get_size_in_bytes()
assert cache_size > 0
cache.shrink(cache_size - 1)
# check that older cached result was removed
assert SolverMock.n_calls[solver_mock] == 1
cache.wrap_solver(another_solver)(**get_solver_test_args())
cache.wrap_solver(solver_mock)(**get_solver_test_args())
assert SolverMock.n_calls[solver_mock] == 2
assert SolverMock.n_calls[another_solver] == 1
def test_shrink_does_not_fail_if_lock_cannot_be_acquired(tmpdir):
cache = DecoderCache(cache_dir=str(tmpdir))
cache._index._lock.timeout = 1.
with cache:
cache.wrap_solver(SolverMock())(**get_solver_test_args())
with cache._index._lock:
cache.shrink(limit=0)
def test_warns_out_of_context(tmpdir):
cache_dir = str(tmpdir)
cache = DecoderCache(cache_dir=cache_dir)
with warns(UserWarning):
cache.shrink()
solver_mock = SolverMock()
solver = cache.wrap_solver(solver_mock)
with warns(UserWarning):
solver(**get_solver_test_args())
assert SolverMock.n_calls[solver_mock] == 1
def test_decoder_cache_shrinking(tmpdir):
cache_dir = str(tmpdir)
solver_mock = SolverMock()
another_solver = SolverMock('another_solver')
M = 100
N = 10
D = 2
activities = np.ones((M, D))
targets = np.ones((M, N))
rng = np.random.RandomState(42)
cache = DecoderCache(cache_dir=cache_dir)
cache.wrap_solver(solver_mock)(activities, targets, rng)
limit = cache.get_size()
# Ensure differing time stamps (depending on the file system the timestamp
# resolution might be as bad as 1 day).
for filename in os.listdir(cache.cache_dir):
path = os.path.join(cache.cache_dir, filename)
timestamp = os.stat(path).st_atime
timestamp -= 60 * 60 * 24 * 2 # 2 days
os.utime(path, (timestamp, timestamp))
cache.wrap_solver(another_solver)(activities, targets, rng)
assert cache.get_size() > 0
cache.shrink(limit)
# check that older cached result was removed
assert SolverMock.n_calls[solver_mock] == 1
cache.wrap_solver(another_solver)(activities, targets, rng)
cache.wrap_solver(solver_mock)(activities, targets, rng)
assert SolverMock.n_calls[solver_mock] == 2
assert SolverMock.n_calls[another_solver] == 1
def test_decoder_cache_shrinking(tmpdir):
cache_dir = str(tmpdir)
solver_mock = SolverMock()
another_solver = SolverMock('another_solver')
M = 100
N = 10
D = 2
activities = np.ones((M, D))
targets = np.ones((M, N))
rng = np.random.RandomState(42)
cache = DecoderCache(cache_dir=cache_dir)
cache.wrap_solver(solver_mock)(activities, targets, rng)
limit = cache.get_size()
# Ensure differing time stamps (depending on the file system the timestamp
# resolution might be as bad as 1 day).
for filename in os.listdir(cache.cache_dir):
path = os.path.join(cache.cache_dir, filename)
timestamp = os.stat(path).st_atime
timestamp -= 60 * 60 * 24 * 2 # 2 days
os.utime(path, (timestamp, timestamp))
cache.wrap_solver(another_solver)(activities, targets, rng)
assert cache.get_size() > 0
cache.shrink(limit)
# check that older cached result was removed
assert SolverMock.n_calls[solver_mock] == 1
cache.wrap_solver(another_solver)(activities, targets, rng)
cache.wrap_solver(solver_mock)(activities, targets, rng)
assert SolverMock.n_calls[solver_mock] == 2
assert SolverMock.n_calls[another_solver] == 1
