def test_fs_backend_load_store_loads_from_fallback_dir(tmpdir): run_dir = tmpdir.mkdir('run_dir') fb_dir = tmpdir.mkdir('fb_dir') fallback = _FilesystemBackend(key_prefixes=['prefix'], runtime_dirs=[fb_dir]) fallback.store('key', 42) store = _FilesystemBackend(key_prefixes=['prefix'], runtime_dirs=[run_dir, fb_dir]) assert store.load_store('key', lambda x: x + 1) == (42, 43) assert fallback.load('key') == 42, 'content in fallback location changed'
def test_fs_backend_loads_from_fallback_dir(tmpdir): run_dir = tmpdir.mkdir('run_dir') fb_dir = tmpdir.mkdir('fb_dir') fallback = _FilesystemBackend(key_prefixes=['prefix'], runtime_dirs=[fb_dir]) fallback.store('key', 42) store = _FilesystemBackend(key_prefixes=['prefix'], runtime_dirs=[run_dir, fb_dir]) assert store.load('key') == 42 store.store('key', -1) assert store.load('key') == -1 assert fallback.load('key') == 42, 'fallback location was changed'
def test_fs_backend_load_store_returns_old_and_new_values(tmpdir): run_dir = tmpdir.mkdir('run_dir') store = _FilesystemBackend(key_prefixes=['prefix'], runtime_dirs=[run_dir]) assert store.load_store('key', lambda _: 42) == (None, 42) assert store.load_store('key', lambda x: x + 1) == (42, 43)
def test_fs_backend_stores_honor_load_store_locking(tmpdir): run_dir = tmpdir.mkdir('run_dir') store = _FilesystemBackend(key_prefixes=['prefix'], runtime_dirs=[run_dir]) store.store('key', 42) ps = [ Process(target=_fs_mp_increment_key, args=(run_dir, 'prefix', 'key', .2)), Process(target=_fs_mp_store_key, args=(run_dir, 'prefix', 'key', -1)), ] start_time = time.monotonic() ps[0].start() time.sleep(.1) ps[1].start() # join second process first ps[1].join() elapsed = (time.monotonic() - start_time) assert elapsed >= .2 ps[0].join() assert store.load('key') == -1
def test_fs_backend_load_store_is_atomic(tmpdir): run_dir = tmpdir.mkdir('run_dir') store = _FilesystemBackend(key_prefixes=['prefix'], runtime_dirs=[run_dir]) store.store('key', 42) ps = [ Process(target=_fs_mp_increment_key, args=(run_dir, 'prefix', 'key', .2)), Process(target=_fs_mp_increment_key, args=(run_dir, 'prefix', 'key', .2)), Process(target=_fs_mp_increment_key, args=(run_dir, 'prefix', 'key', .2)), ] start_time = time.monotonic() for p in ps: p.start() for p in ps: p.join() elapsed = (time.monotonic() - start_time) assert store.load('key') == 45 assert elapsed >= .2 * len(ps)
def _fs_mp_store_key(run_dir, prefix, key, new_value): """Open a _FilesystemBackend and store `new_value` for `key`. For the `multiprocessing` tests. Opens the storage on `run_dir` and with `prefix`. """ store = _FilesystemBackend(key_prefixes=['prefix'], runtime_dirs=[run_dir]) store.store(key, new_value)
def _fs_mp_check_key(run_dir, prefix, key, expected): """Open a _FilesystemBackend and check `key` value against `expected`. For the `multiprocessing` tests. Opens the storage on `run_dir` and with `prefix`. """ store = _FilesystemBackend(key_prefixes=['prefix'], runtime_dirs=[run_dir]) assert store.load(key) == expected
def test_fs_backend_load_store_loads_from_fallback_dir_that_is_symlink(tmpdir): # should deadlock if there is a problem with the lock type or with the # handling of fallback paths that point to the same principal/write # directory run_dir = tmpdir.mkdir('run_dir') fb_dir = os.path.join(run_dir, 'symlink') os.symlink(run_dir, fb_dir, target_is_directory=True) # don't store any initial value so that the fallback location is checked store = _FilesystemBackend(key_prefixes=['prefix'], runtime_dirs=[run_dir, fb_dir]) assert store.load_store('key', lambda x: 42) == (None, 42) fallback = _FilesystemBackend(key_prefixes=['prefix'], runtime_dirs=[fb_dir]) assert fallback.load( 'key') == 42, 'content in fallback symlink did not change'
def test_fs_backend_stores_truncate_appropriately(tmpdir): run_dir = tmpdir.mkdir('run_dir') # use a separate reader to prevent caching from masking issues writer = _FilesystemBackend(key_prefixes=['prefix'], runtime_dirs=[run_dir]) reader = _FilesystemBackend(key_prefixes=['prefix'], runtime_dirs=[run_dir]) writer.store('key', 42) assert reader.load('key') == 42 writer.store('key', 1) assert reader.load('key') == 1 writer.load_store('key', lambda _: 42) assert reader.load('key') == 42 writer.load_store('key', lambda _: 1) assert reader.load('key') == 1
def test_fs_backend_handles_values_corupted_with_nulls(tmpdir, caplog): run_dir = tmpdir.mkdir('run_dir') store = _FilesystemBackend(key_prefixes=['prefix'], runtime_dirs=[run_dir]) store.store('key', 42) key_file = Path(run_dir).joinpath('prefix', 'key') assert key_file.read_bytes() == b'42', 'unit test is unsound' key_file.write_bytes(b'\x00') val = store.load('key') assert val is None assert 'was corrupted' in caplog.text
def _fs_mp_increment_key(run_dir, prefix, key, sleep): """Open a _FilesystemBackend and increment `key`. For the `multiprocessing` tests. Opens the storage on `run_dir` and with `prefix`. Sleeps for `sleep` seconds within the increment closure. """ def l(x): time.sleep(sleep) return x + 1 store = _FilesystemBackend(key_prefixes=[prefix], runtime_dirs=[run_dir]) store.load_store(key, l)
def test_fs_backend_loads_honor_load_store_locking(tmpdir): run_dir = tmpdir.mkdir('run_dir') store = _FilesystemBackend(key_prefixes=['prefix'], runtime_dirs=[run_dir]) store.store('key', 42) ps = [ Process(target=_fs_mp_increment_key, args=(run_dir, 'prefix', 'key', .2)), Process(target=_fs_mp_check_key, args=(run_dir, 'prefix', 'key', 43)), ] ps[0].start() time.sleep(.1) ps[1].start() for p in ps: p.join()
def test_fs_backend_releases_locks(tmpdir): # should deadlock if any method does not properly release its lock run_dir = tmpdir.mkdir('run_dir') store = _FilesystemBackend(key_prefixes=['prefix'], runtime_dirs=[run_dir]) def incr_from_other_process(): other = Process(target=_fs_mp_increment_key, args=(run_dir, 'prefix', 'key', 0.)) other.start() other.join() store.store('key', 42) incr_from_other_process() assert store.load('key') == 43 store.load_store('key', lambda _: -1) incr_from_other_process() assert store.load('key') == 0 incr_from_other_process() assert store.load('key') == 1
def __init__(self, key_prefixes, backend=None): if not backend: run_dir = mkdtemp('run_dir') backend = _FilesystemBackend(key_prefixes, runtime_dirs=[run_dir]) super().__init__(key_prefixes, backend)
def tmpstore(tmpdir): run_dir = tmpdir.mkdir('run_dir') prefixes = ['prefix'] backend = _FilesystemBackend(key_prefixes=prefixes, runtime_dirs=[run_dir]) return RuntimeStorage(prefixes, backend=backend)