def test_cache_exists(path_to_checksum, object_exists, traverse):
remote = RemoteBASE(None, {})
# remote does not support traverse
remote.CAN_TRAVERSE = False
with mock.patch.object(remote,
"list_cache_paths",
return_value=list(range(256))):
checksums = list(range(1000))
remote.cache_exists(checksums)
object_exists.assert_called_with(checksums, None, None)
traverse.assert_not_called()
remote.CAN_TRAVERSE = True
# large remote, small local
object_exists.reset_mock()
traverse.reset_mock()
with mock.patch.object(remote,
"list_cache_paths",
return_value=list(range(256))):
checksums = list(range(1000))
remote.cache_exists(checksums)
# verify that _cache_paths_with_max() short circuits
# before returning all 256 remote checksums
max_checksums = math.ceil(
remote._max_estimation_size(checksums) /
pow(16, remote.TRAVERSE_PREFIX_LEN))
assert max_checksums < 256
object_exists.assert_called_with(frozenset(range(max_checksums, 1000)),
None, None)
traverse.assert_not_called()
# large remote, large local
object_exists.reset_mock()
traverse.reset_mock()
remote.JOBS = 16
with mock.patch.object(remote,
"list_cache_paths",
return_value=list(range(256))):
checksums = list(range(1000000))
remote.cache_exists(checksums)
object_exists.assert_not_called()
traverse.assert_called_with(
frozenset(checksums),
set(range(256)),
256 * pow(16, remote.TRAVERSE_PREFIX_LEN),
None,
None,
)
# default traverse
object_exists.reset_mock()
traverse.reset_mock()
remote.TRAVERSE_WEIGHT_MULTIPLIER = 1
with mock.patch.object(remote, "list_cache_paths", return_value=[0]):
checksums = set(range(1000000))
remote.cache_exists(checksums)
traverse.assert_not_called()
object_exists.assert_not_called()
def test(self):
config = {
"url": "base://example/prefix",
"connection_string": "1234567",
}
remote = RemoteBASE(None, config)
remote.PARAM_CHECKSUM = "checksum"
remote.path_info = remote.path_cls(config["url"])
remote.url = ""
remote.prefix = ""
path_info = remote.path_info / "example"
checksum_info = {remote.PARAM_CHECKSUM: "1234567890"}
with mock.patch.object(remote, "_checkout") as mock_checkout:
with mock.patch.object(remote, "_save") as mock_save:
with mock.patch.object(
remote, "changed_cache", return_value=True
):
remote.save(path_info, checksum_info)
mock_save.assert_called_once()
mock_checkout.assert_not_called()
with mock.patch.object(remote, "_checkout") as mock_checkout:
with mock.patch.object(remote, "_save") as mock_save:
with mock.patch.object(
remote, "changed_cache", return_value=False
):
remote.save(path_info, checksum_info)
mock_save.assert_not_called()
mock_checkout.assert_called_once()
def test_cache_exists_traverse(path_to_checksum, list_cache_paths):
remote = RemoteBASE(None, {})
remote.path_info = PathInfo("foo")
remote._cache_exists_traverse({0}, set())
for i in range(1, 16):
list_cache_paths.assert_any_call(prefix="{:03x}".format(i))
for i in range(1, 256):
list_cache_paths.assert_any_call(prefix="{:02x}".format(i))
def test_cache_checksums():
remote = RemoteBASE(None, {})
remote.path_info = PathInfo("foo")
with mock.patch.object(remote,
"list_cache_paths",
return_value=["12/3456", "bar"]):
checksums = list(remote.cache_checksums())
assert checksums == ["123456"]
def test_cache_exists(path_to_checksum, object_exists, traverse):
remote = RemoteBASE(None, {})
# remote does not support traverse
remote.CAN_TRAVERSE = False
with mock.patch.object(
remote, "list_cache_paths", return_value=list(range(256))
):
checksums = list(range(1000))
remote.cache_exists(checksums)
object_exists.assert_called_with(checksums, None, None)
traverse.assert_not_called()
remote.CAN_TRAVERSE = True
# large remote, small local
object_exists.reset_mock()
traverse.reset_mock()
with mock.patch.object(
remote, "list_cache_paths", return_value=list(range(256))
):
checksums = list(range(1000))
remote.cache_exists(checksums)
object_exists.assert_called_with(
frozenset(range(256, 1000)), None, None
)
traverse.assert_not_called()
# large remote, large local
object_exists.reset_mock()
traverse.reset_mock()
remote.JOBS = 16
with mock.patch.object(
remote, "list_cache_paths", return_value=list(range(256))
):
checksums = list(range(1000000))
remote.cache_exists(checksums)
object_exists.assert_not_called()
traverse.assert_called_with(
frozenset(checksums),
set(range(256)),
256 * pow(16, remote.TRAVERSE_PREFIX_LEN),
None,
None,
)
# default traverse
object_exists.reset_mock()
traverse.reset_mock()
remote.TRAVERSE_WEIGHT_MULTIPLIER = 1
with mock.patch.object(remote, "list_cache_paths", return_value=[0]):
checksums = set(range(1000000))
remote.cache_exists(checksums)
traverse.assert_not_called()
object_exists.assert_not_called()
def test_cache_checksums_traverse(path_to_checksum, cache_checksums):
remote = RemoteBASE(None, {})
remote.path_info = PathInfo("foo")
# parallel traverse
size = 256 / remote.JOBS * remote.LIST_OBJECT_PAGE_SIZE
list(remote._cache_checksums_traverse(size, {0}))
for i in range(1, 16):
cache_checksums.assert_any_call(prefix="{:03x}".format(i),
progress_callback=CallableOrNone)
for i in range(1, 256):
cache_checksums.assert_any_call(prefix="{:02x}".format(i),
progress_callback=CallableOrNone)
# default traverse (small remote)
size -= 1
cache_checksums.reset_mock()
list(remote._cache_checksums_traverse(size - 1, {0}))
cache_checksums.assert_called_with(prefix=None,
progress_callback=CallableOrNone)