def test_cache_entry_newer_than(): entry_a = CacheEntry("a", LocalCacheService().get_metadata("/")) entry_b = CacheEntry("b", LocalCacheService().get_metadata("/")) assert entry_b.newer_than(entry_a) entry_a.last_update = time.time() assert not entry_b.newer_than(entry_a)
def test_concurrent_cache_content_cleanup(tmp_path): (tmp_path / "cache").mkdir() cache = create_cache(tmp_path, max_size=3) for x in ["a", "b", "c", "d"]: (tmp_path / x).write_text("123") for x in ["a", "b", "c", "d"]: fd = cache.open_contents(str(tmp_path / x), os.O_RDONLY) os.close(fd) cache.save() assert len(os.listdir(tmp_path / "cache" / "contents")) == 1 cache = RemoteCache( str(tmp_path / "cache"), "machine", LocalCacheService(), prefetch=False, max_entries=1024, max_size=1024 * 1024, cacheable_paths=["/"], ) cache.save(merge_disk_cache=False) assert len(os.listdir(tmp_path / "cache" / "contents")) == 0
def test_concurrent_cache_disk_merge(tmp_path): (tmp_path / "foo").touch() (tmp_path / "bar").touch() cache_a = create_cache(tmp_path) cache_b = create_cache(tmp_path) cache_a.get_metadata(str(tmp_path / "foo")) cache_b.get_metadata(str(tmp_path / "bar")) cache_a.save() cache_b.save() cache_c = RemoteCache( str(tmp_path / "cache"), "machine", LocalCacheService(), prefetch=False, max_entries=1024, max_size=1024 * 1024, cacheable_paths=["/"], ) cache_c.load() assert cache_c.count() == 2
def test_cached_readlink(tmp_path): (tmp_path / "cache").mkdir() (tmp_path / "cached").mkdir() os.symlink("a", tmp_path / "cached/b") mock_client = mock.Mock() mock_client.get_metadata.side_effect = LocalCacheService().get_metadata fs = create_remote_file_system( tmp_path, client=mock_client, cache=create_cache(tmp_path, client=mock_client, cacheable_paths=[str(tmp_path / "cached")]), ) with pytest.raises(FileNotFoundError): fs.getattr(str(tmp_path / "cached" / "a"), None) with pytest.raises(FileNotFoundError): fs.readlink(str(tmp_path / "cached" / "a")) assert mock_client.get_metadata.call_count == 1 with pytest.raises(OSError) as e: fs.readlink(str(tmp_path / "cached")) assert e.value.args == (errno.EINVAL, ) assert fs.readlink(str(tmp_path / "cached/b")) == "a"
def test_cacheable_paths(tmp_path): (tmp_path / "cache").mkdir() (tmp_path / "cached").mkdir() (tmp_path / "notcached").mkdir() mock_client = mock.Mock() mock_client.get_metadata.return_value = LocalCacheService().get_metadata( "/") fs = create_remote_file_system( tmp_path, client=mock_client, cache=create_cache(tmp_path, client=mock_client, cacheable_paths=[str(tmp_path / "cached")]), ) fs.getattr(str(tmp_path / "cached" / "a"), None) fs.getattr(str(tmp_path / "cached" / "b"), None) fs.getattr(str(tmp_path / "cached" / "a"), None) assert mock_client.get_metadata.call_count == 2 # Should not even be retrieved through cache fs.getattr(str(tmp_path / "notcached" / "a"), 123) fs.getattr(str(tmp_path / "notcached" / "b"), 456) fs.getattr(str(tmp_path / "notcached" / "a"), 789) assert mock_client.get_metadata.call_count == 2
def test_readfile_prefetch_executable_with_previously_fetched_contents(): sh_path = shutil.which("ssh") service = LocalCacheService() _metadata, prefetches = service.readfile_prefetch(sh_path) assert any(p.contents for p in prefetches) service = LocalCacheService() service.mark_previously_fetched_contents([p.path for p in prefetches]) _metadata, prefetches = service.readfile_prefetch(sh_path) assert not any(p.contents for p in prefetches)
def test_concurrent_cache_get_metadata(tmp_path): meta = LocalCacheService().get_metadata("/") meta = dataclasses.replace(meta, attr=meta.attr.as_readonly()) mock_client = mock.Mock() mock_client.get_metadata.return_value = meta (tmp_path / "cache").mkdir() cache = create_cache(tmp_path, client=mock_client) for _ in range(10): assert cache.get_metadata("/") == meta assert mock_client.get_metadata.call_count == 1
def create_cache(tmp_path, **override_args): base_args = dict( base_path=str(tmp_path / "cache"), machine_id="machine", client=LocalCacheService(), prefetch=False, max_entries=1024, max_size=1024 * 1024, cacheable_paths=["/"], ) final_args = {**base_args, **override_args} return RemoteCache(**final_args)
def test_concurrent_cache_load_save(tmp_path): meta = LocalCacheService().get_metadata("/") meta = dataclasses.replace(meta, attr=meta.attr.as_readonly()) mock_client = mock.Mock() mock_client.get_metadata.return_value = meta (tmp_path / "cache").mkdir() cache_a = create_cache(tmp_path, client=mock_client) assert cache_a.get_metadata("/") == meta cache_a.save() cache_b = create_cache(tmp_path, client=mock_client) cache_b.load() assert cache_b.get_metadata("/") == meta assert mock_client.get_metadata.call_count == 1
def test_concurrent_cache_open_content(tmp_path): fs = LocalCacheService() (tmp_path / "cache").mkdir() (tmp_path / "hello").write_text("world") mock_client = mock.Mock() mock_client.readfile.side_effect = fs.readfile cache = create_cache(tmp_path, client=mock_client) for _ in range(10): fd = cache.open_contents(str(tmp_path / "hello"), os.O_RDONLY) try: os.lseek(fd, 0, 0) assert os.read(fd, 1024) == b"world" finally: os.close(fd) assert mock_client.readfile.call_count == 1
def service(): return LocalCacheService()