def test_list_hashes(dvc):
tree = BaseTree(dvc, {})
tree.path_info = PathInfo("foo")
with mock.patch.object(tree, "list_paths", return_value=["12/3456",
"bar"]):
hashes = list(tree.list_hashes())
assert hashes == ["123456"]
def test_list_paths(dvc):
tree = BaseTree(dvc, {})
tree.path_info = PathInfo("foo")
with mock.patch.object(tree, "walk_files", return_value=[]) as walk_mock:
for _ in tree.list_paths():
pass
walk_mock.assert_called_with(tree.path_info, prefix=False)
for _ in tree.list_paths(prefix="000"):
pass
walk_mock.assert_called_with(tree.path_info / "00" / "0", prefix=True)
def __init__(self, src_tree: BaseTree, baseline_rev: str, **kwargs):
dvc_dir = kwargs.pop("dvc_dir")
cache_dir = kwargs.pop("cache_dir")
super().__init__(src_tree, baseline_rev, **kwargs)
self.tmp_dir = TemporaryDirectory()
logger.debug("Init local executor in dir '%s'.", self.tmp_dir)
self.dvc_dir = os.path.join(self.tmp_dir.name, dvc_dir)
try:
for fname in src_tree.walk_files(src_tree.tree_root):
dest = self.path_info / relpath(fname, src_tree.tree_root)
if not os.path.exists(dest.parent):
makedirs(dest.parent)
with src_tree.open(fname, "rb") as fobj:
copy_fobj_to_file(fobj, dest)
except Exception:
self.tmp_dir.cleanup()
raise
self._config(cache_dir)
def test_list_hashes(dvc):
cache = CloudCache(BaseTree(dvc, {}))
cache.tree.path_info = PathInfo("foo")
with mock.patch.object(cache,
"_list_paths",
return_value=["12/3456", "bar"]):
hashes = list(cache.list_hashes())
assert hashes == ["123456"]
def test_list_hashes_traverse(_path_to_hash, list_hashes, dvc):
tree = BaseTree(dvc, {})
tree.path_info = PathInfo("foo")
# parallel traverse
size = 256 / tree.JOBS * tree.LIST_OBJECT_PAGE_SIZE
list(tree.list_hashes_traverse(size, {0}))
for i in range(1, 16):
list_hashes.assert_any_call(prefix=f"{i:03x}",
progress_callback=CallableOrNone)
for i in range(1, 256):
list_hashes.assert_any_call(prefix=f"{i:02x}",
progress_callback=CallableOrNone)
# default traverse (small remote)
size -= 1
list_hashes.reset_mock()
list(tree.list_hashes_traverse(size - 1, {0}))
list_hashes.assert_called_with(prefix=None,
progress_callback=CallableOrNone)
def test_cmd_error(dvc):
config = {}
cmd = "sed 'hello'"
ret = "1"
err = "sed: expression #1, char 2: extra characters after command"
with mock.patch.object(
BaseTree,
"remove",
side_effect=RemoteCmdError("base", cmd, ret, err),
):
with pytest.raises(RemoteCmdError):
BaseTree(dvc, config).remove("file")
def test_hashes_exist(object_exists, traverse, dvc):
remote = Remote(BaseTree(dvc, {}))
# remote does not support traverse
remote.tree.CAN_TRAVERSE = False
with mock.patch.object(remote.tree,
"list_hashes",
return_value=list(range(256))):
hashes = set(range(1000))
remote.hashes_exist(hashes)
object_exists.assert_called_with(hashes, None, None)
traverse.assert_not_called()
remote.tree.CAN_TRAVERSE = True
# large remote, small local
object_exists.reset_mock()
traverse.reset_mock()
with mock.patch.object(remote.tree,
"list_hashes",
return_value=list(range(256))):
hashes = list(range(1000))
remote.hashes_exist(hashes)
# verify that _cache_paths_with_max() short circuits
# before returning all 256 remote hashes
max_hashes = math.ceil(
remote.tree._max_estimation_size(hashes) /
pow(16, remote.tree.TRAVERSE_PREFIX_LEN))
assert max_hashes < 256
object_exists.assert_called_with(frozenset(range(max_hashes, 1000)),
None, None)
traverse.assert_not_called()
# large remote, large local
object_exists.reset_mock()
traverse.reset_mock()
remote.tree.JOBS = 16
with mock.patch.object(remote.tree,
"list_hashes",
return_value=list(range(256))):
hashes = list(range(1000000))
remote.hashes_exist(hashes)
object_exists.assert_not_called()
traverse.assert_called_with(
256 * pow(16, remote.tree.TRAVERSE_PREFIX_LEN),
set(range(256)),
None,
None,
)
def test_missing_deps(pkg, msg, mocker):
requires = {"missing": "missing"}
mocker.patch.object(BaseTree, "REQUIRES", requires)
mocker.patch("dvc.utils.pkg.PKG", pkg)
with pytest.raises(RemoteMissingDepsError, match=msg):
BaseTree(None, {})
def test_missing_deps(dvc):
requires = {"missing": "missing"}
with mock.patch.object(BaseTree, "REQUIRES", requires):
with pytest.raises(RemoteMissingDepsError):
BaseTree(dvc, {})
def test_is_dir_hash(hash_, result):
assert BaseTree.is_dir_hash(hash_) == result