예제 #1
0
def test_cache_exists(path_to_checksum, object_exists, traverse):
    remote = RemoteBASE(None, {})

    # remote does not support traverse
    remote.CAN_TRAVERSE = False
    with mock.patch.object(remote,
                           "list_cache_paths",
                           return_value=list(range(256))):
        checksums = list(range(1000))
        remote.cache_exists(checksums)
        object_exists.assert_called_with(checksums, None, None)
        traverse.assert_not_called()

    remote.CAN_TRAVERSE = True

    # large remote, small local
    object_exists.reset_mock()
    traverse.reset_mock()
    with mock.patch.object(remote,
                           "list_cache_paths",
                           return_value=list(range(256))):
        checksums = list(range(1000))
        remote.cache_exists(checksums)
        # verify that _cache_paths_with_max() short circuits
        # before returning all 256 remote checksums
        max_checksums = math.ceil(
            remote._max_estimation_size(checksums) /
            pow(16, remote.TRAVERSE_PREFIX_LEN))
        assert max_checksums < 256
        object_exists.assert_called_with(frozenset(range(max_checksums, 1000)),
                                         None, None)
        traverse.assert_not_called()

    # large remote, large local
    object_exists.reset_mock()
    traverse.reset_mock()
    remote.JOBS = 16
    with mock.patch.object(remote,
                           "list_cache_paths",
                           return_value=list(range(256))):
        checksums = list(range(1000000))
        remote.cache_exists(checksums)
        object_exists.assert_not_called()
        traverse.assert_called_with(
            frozenset(checksums),
            set(range(256)),
            256 * pow(16, remote.TRAVERSE_PREFIX_LEN),
            None,
            None,
        )

    # default traverse
    object_exists.reset_mock()
    traverse.reset_mock()
    remote.TRAVERSE_WEIGHT_MULTIPLIER = 1
    with mock.patch.object(remote, "list_cache_paths", return_value=[0]):
        checksums = set(range(1000000))
        remote.cache_exists(checksums)
        traverse.assert_not_called()
        object_exists.assert_not_called()
예제 #2
0
파일: test_base.py 프로젝트: yk/dvc
    def test(self):
        config = {
            "url": "base://example/prefix",
            "connection_string": "1234567",
        }
        remote = RemoteBASE(None, config)

        remote.PARAM_CHECKSUM = "checksum"
        remote.path_info = remote.path_cls(config["url"])
        remote.url = ""
        remote.prefix = ""
        path_info = remote.path_info / "example"
        checksum_info = {remote.PARAM_CHECKSUM: "1234567890"}

        with mock.patch.object(remote, "_checkout") as mock_checkout:
            with mock.patch.object(remote, "_save") as mock_save:
                with mock.patch.object(
                    remote, "changed_cache", return_value=True
                ):
                    remote.save(path_info, checksum_info)
                    mock_save.assert_called_once()
                    mock_checkout.assert_not_called()

        with mock.patch.object(remote, "_checkout") as mock_checkout:
            with mock.patch.object(remote, "_save") as mock_save:
                with mock.patch.object(
                    remote, "changed_cache", return_value=False
                ):
                    remote.save(path_info, checksum_info)
                    mock_save.assert_not_called()
                    mock_checkout.assert_called_once()
예제 #3
0
파일: test_base.py 프로젝트: SdgJlbl/dvc
def test_cache_exists_traverse(path_to_checksum, list_cache_paths):
    remote = RemoteBASE(None, {})
    remote.path_info = PathInfo("foo")
    remote._cache_exists_traverse({0}, set())
    for i in range(1, 16):
        list_cache_paths.assert_any_call(prefix="{:03x}".format(i))
    for i in range(1, 256):
        list_cache_paths.assert_any_call(prefix="{:02x}".format(i))
예제 #4
0
파일: test_base.py 프로젝트: refaqtor/dvc
def test_cache_checksums():
    remote = RemoteBASE(None, {})
    remote.path_info = PathInfo("foo")

    with mock.patch.object(remote,
                           "list_cache_paths",
                           return_value=["12/3456", "bar"]):
        checksums = list(remote.cache_checksums())
        assert checksums == ["123456"]
예제 #5
0
def test_cache_exists(path_to_checksum, object_exists, traverse):
    remote = RemoteBASE(None, {})

    # remote does not support traverse
    remote.CAN_TRAVERSE = False
    with mock.patch.object(
        remote, "list_cache_paths", return_value=list(range(256))
    ):
        checksums = list(range(1000))
        remote.cache_exists(checksums)
        object_exists.assert_called_with(checksums, None, None)
        traverse.assert_not_called()

    remote.CAN_TRAVERSE = True

    # large remote, small local
    object_exists.reset_mock()
    traverse.reset_mock()
    with mock.patch.object(
        remote, "list_cache_paths", return_value=list(range(256))
    ):
        checksums = list(range(1000))
        remote.cache_exists(checksums)
        object_exists.assert_called_with(
            frozenset(range(256, 1000)), None, None
        )
        traverse.assert_not_called()

    # large remote, large local
    object_exists.reset_mock()
    traverse.reset_mock()
    remote.JOBS = 16
    with mock.patch.object(
        remote, "list_cache_paths", return_value=list(range(256))
    ):
        checksums = list(range(1000000))
        remote.cache_exists(checksums)
        object_exists.assert_not_called()
        traverse.assert_called_with(
            frozenset(checksums),
            set(range(256)),
            256 * pow(16, remote.TRAVERSE_PREFIX_LEN),
            None,
            None,
        )

    # default traverse
    object_exists.reset_mock()
    traverse.reset_mock()
    remote.TRAVERSE_WEIGHT_MULTIPLIER = 1
    with mock.patch.object(remote, "list_cache_paths", return_value=[0]):
        checksums = set(range(1000000))
        remote.cache_exists(checksums)
        traverse.assert_not_called()
        object_exists.assert_not_called()
예제 #6
0
파일: test_base.py 프로젝트: refaqtor/dvc
def test_cache_checksums_traverse(path_to_checksum, cache_checksums):
    remote = RemoteBASE(None, {})
    remote.path_info = PathInfo("foo")

    # parallel traverse
    size = 256 / remote.JOBS * remote.LIST_OBJECT_PAGE_SIZE
    list(remote._cache_checksums_traverse(size, {0}))
    for i in range(1, 16):
        cache_checksums.assert_any_call(prefix="{:03x}".format(i),
                                        progress_callback=CallableOrNone)
    for i in range(1, 256):
        cache_checksums.assert_any_call(prefix="{:02x}".format(i),
                                        progress_callback=CallableOrNone)

    # default traverse (small remote)
    size -= 1
    cache_checksums.reset_mock()
    list(remote._cache_checksums_traverse(size - 1, {0}))
    cache_checksums.assert_called_with(prefix=None,
                                       progress_callback=CallableOrNone)