def test_cache_entry_newer_than():
entry_a = CacheEntry("a", LocalCacheService().get_metadata("/"))
entry_b = CacheEntry("b", LocalCacheService().get_metadata("/"))
assert entry_b.newer_than(entry_a)
entry_a.last_update = time.time()
assert not entry_b.newer_than(entry_a)
def test_concurrent_cache_content_cleanup(tmp_path):
(tmp_path / "cache").mkdir()
cache = create_cache(tmp_path, max_size=3)
for x in ["a", "b", "c", "d"]:
(tmp_path / x).write_text("123")
for x in ["a", "b", "c", "d"]:
fd = cache.open_contents(str(tmp_path / x), os.O_RDONLY)
os.close(fd)
cache.save()
assert len(os.listdir(tmp_path / "cache" / "contents")) == 1
cache = RemoteCache(
str(tmp_path / "cache"),
"machine",
LocalCacheService(),
prefetch=False,
max_entries=1024,
max_size=1024 * 1024,
cacheable_paths=["/"],
)
cache.save(merge_disk_cache=False)
assert len(os.listdir(tmp_path / "cache" / "contents")) == 0
def test_concurrent_cache_disk_merge(tmp_path):
(tmp_path / "foo").touch()
(tmp_path / "bar").touch()
cache_a = create_cache(tmp_path)
cache_b = create_cache(tmp_path)
cache_a.get_metadata(str(tmp_path / "foo"))
cache_b.get_metadata(str(tmp_path / "bar"))
cache_a.save()
cache_b.save()
cache_c = RemoteCache(
str(tmp_path / "cache"),
"machine",
LocalCacheService(),
prefetch=False,
max_entries=1024,
max_size=1024 * 1024,
cacheable_paths=["/"],
)
cache_c.load()
assert cache_c.count() == 2
def test_cached_readlink(tmp_path):
(tmp_path / "cache").mkdir()
(tmp_path / "cached").mkdir()
os.symlink("a", tmp_path / "cached/b")
mock_client = mock.Mock()
mock_client.get_metadata.side_effect = LocalCacheService().get_metadata
fs = create_remote_file_system(
tmp_path,
client=mock_client,
cache=create_cache(tmp_path,
client=mock_client,
cacheable_paths=[str(tmp_path / "cached")]),
)
with pytest.raises(FileNotFoundError):
fs.getattr(str(tmp_path / "cached" / "a"), None)
with pytest.raises(FileNotFoundError):
fs.readlink(str(tmp_path / "cached" / "a"))
assert mock_client.get_metadata.call_count == 1
with pytest.raises(OSError) as e:
fs.readlink(str(tmp_path / "cached"))
assert e.value.args == (errno.EINVAL, )
assert fs.readlink(str(tmp_path / "cached/b")) == "a"
def test_cacheable_paths(tmp_path):
(tmp_path / "cache").mkdir()
(tmp_path / "cached").mkdir()
(tmp_path / "notcached").mkdir()
mock_client = mock.Mock()
mock_client.get_metadata.return_value = LocalCacheService().get_metadata(
"/")
fs = create_remote_file_system(
tmp_path,
client=mock_client,
cache=create_cache(tmp_path,
client=mock_client,
cacheable_paths=[str(tmp_path / "cached")]),
)
fs.getattr(str(tmp_path / "cached" / "a"), None)
fs.getattr(str(tmp_path / "cached" / "b"), None)
fs.getattr(str(tmp_path / "cached" / "a"), None)
assert mock_client.get_metadata.call_count == 2
# Should not even be retrieved through cache
fs.getattr(str(tmp_path / "notcached" / "a"), 123)
fs.getattr(str(tmp_path / "notcached" / "b"), 456)
fs.getattr(str(tmp_path / "notcached" / "a"), 789)
assert mock_client.get_metadata.call_count == 2
def test_readfile_prefetch_executable_with_previously_fetched_contents():
sh_path = shutil.which("ssh")
service = LocalCacheService()
_metadata, prefetches = service.readfile_prefetch(sh_path)
assert any(p.contents for p in prefetches)
service = LocalCacheService()
service.mark_previously_fetched_contents([p.path for p in prefetches])
_metadata, prefetches = service.readfile_prefetch(sh_path)
assert not any(p.contents for p in prefetches)
def test_concurrent_cache_get_metadata(tmp_path):
meta = LocalCacheService().get_metadata("/")
meta = dataclasses.replace(meta, attr=meta.attr.as_readonly())
mock_client = mock.Mock()
mock_client.get_metadata.return_value = meta
(tmp_path / "cache").mkdir()
cache = create_cache(tmp_path, client=mock_client)
for _ in range(10):
assert cache.get_metadata("/") == meta
assert mock_client.get_metadata.call_count == 1
def create_cache(tmp_path, **override_args):
base_args = dict(
base_path=str(tmp_path / "cache"),
machine_id="machine",
client=LocalCacheService(),
prefetch=False,
max_entries=1024,
max_size=1024 * 1024,
cacheable_paths=["/"],
)
final_args = {**base_args, **override_args}
return RemoteCache(**final_args)
def test_concurrent_cache_load_save(tmp_path):
meta = LocalCacheService().get_metadata("/")
meta = dataclasses.replace(meta, attr=meta.attr.as_readonly())
mock_client = mock.Mock()
mock_client.get_metadata.return_value = meta
(tmp_path / "cache").mkdir()
cache_a = create_cache(tmp_path, client=mock_client)
assert cache_a.get_metadata("/") == meta
cache_a.save()
cache_b = create_cache(tmp_path, client=mock_client)
cache_b.load()
assert cache_b.get_metadata("/") == meta
assert mock_client.get_metadata.call_count == 1
def test_concurrent_cache_open_content(tmp_path):
fs = LocalCacheService()
(tmp_path / "cache").mkdir()
(tmp_path / "hello").write_text("world")
mock_client = mock.Mock()
mock_client.readfile.side_effect = fs.readfile
cache = create_cache(tmp_path, client=mock_client)
for _ in range(10):
fd = cache.open_contents(str(tmp_path / "hello"), os.O_RDONLY)
try:
os.lseek(fd, 0, 0)
assert os.read(fd, 1024) == b"world"
finally:
os.close(fd)
assert mock_client.readfile.call_count == 1
def service():
return LocalCacheService()