def _checkout(self,
target=None,
with_deps=False,
force=False,
recursive=False):
from dvc.stage import StageFileDoesNotExistError, StageFileBadNameError
try:
stages = self.collect(target, with_deps=with_deps, recursive=recursive)
except (StageFileDoesNotExistError, StageFileBadNameError) as exc:
if not target:
raise
raise CheckoutErrorSuggestGit(target, exc)
_cleanup_unused_links(self, self.stages)
total = get_all_files_numbers(stages)
if total == 0:
logger.info("Nothing to do")
failed = []
with Tqdm(total=total, unit="file", desc="Checkout",
disable=total == 0) as pbar:
for stage in stages:
if stage.locked:
logger.warning(
"DVC-file '{path}' is locked. Its dependencies are"
" not going to be checked out.".format(path=stage.relpath))
failed.extend(
stage.checkout(force=force,
progress_callback=pbar.update_desc))
if failed:
raise CheckoutError(failed)
def checkout(
self,
path_info,
checksum_info,
force=False,
progress_callback=None,
relink=False,
filter_info=None,
):
if path_info.scheme not in ["local", self.scheme]:
raise NotImplementedError
checksum = checksum_info.get(self.PARAM_CHECKSUM)
failed = None
skip = False
if not checksum:
logger.warning(
"No file hash info found for '%s'. "
"It won't be created.",
path_info,
)
self.safe_remove(path_info, force=force)
failed = path_info
elif not relink and not self.changed(path_info, checksum_info):
logger.debug("Data '%s' didn't change.", path_info)
skip = True
elif self.changed_cache(checksum,
path_info=path_info,
filter_info=filter_info):
logger.warning(
"Cache '%s' not found. File '%s' won't be created.",
checksum,
path_info,
)
self.safe_remove(path_info, force=force)
failed = path_info
if failed or skip:
if progress_callback:
progress_callback(
str(path_info),
self.get_files_number(self.path_info, checksum,
filter_info),
)
if failed:
raise CheckoutError([failed])
return
logger.debug("Checking out '%s' with cache '%s'.", path_info, checksum)
return self._checkout(
path_info,
checksum,
force,
progress_callback,
relink,
filter_info,
)
def _checkout(self,
targets=None,
with_deps=False,
force=False,
recursive=False):
from dvc.stage import StageFileDoesNotExistError, StageFileBadNameError
stages = set()
targets = targets or [None]
for target in targets:
try:
new = self.collect(target,
with_deps=with_deps,
recursive=recursive)
stages.update(new)
except (StageFileDoesNotExistError, StageFileBadNameError) as exc:
if not target:
raise
raise CheckoutErrorSuggestGit(target, exc)
_cleanup_unused_links(self, self.stages)
total = get_all_files_numbers(stages)
if total == 0:
logger.info("Nothing to do")
failed = []
with Tqdm(total=total, unit="file", desc="Checkout",
disable=total == 0) as pbar:
for stage in stages:
failed.extend(
stage.checkout(force=force,
progress_callback=pbar.update_desc))
if failed:
raise CheckoutError(failed)
def _checkout(
self,
targets=None,
with_deps=False,
force=False,
relink=False,
recursive=False,
):
from dvc.stage import StageFileDoesNotExistError, StageFileBadNameError
unused = []
stats = {
"added": [],
"deleted": [],
"modified": [],
"failed": [],
}
if not targets:
targets = [None]
unused = _get_unused_links(self)
stats["deleted"] = [_fspath_dir(u, self.root_dir) for u in unused]
self.state.remove_links(unused)
pairs = set()
for target in targets:
try:
pairs.update(
self.collect_granular(
target, with_deps=with_deps, recursive=recursive
)
)
except (StageFileDoesNotExistError, StageFileBadNameError) as exc:
if not target:
raise
raise CheckoutErrorSuggestGit(target) from exc
total = get_all_files_numbers(pairs)
with Tqdm(
total=total, unit="file", desc="Checkout", disable=total == 0
) as pbar:
for stage, filter_info in pairs:
result = stage.checkout(
force=force,
progress_callback=pbar.update_desc,
relink=relink,
filter_info=filter_info,
)
for data in ["failed", "added", "modified"]:
stats[data].extend(
_fspath_dir(path, self.root_dir) for path in result[data]
)
if stats.get("failed"):
raise CheckoutError(stats["failed"], stats)
del stats["failed"]
return stats
def checkout(
path_info,
fs,
obj,
cache,
force=False,
progress_callback=None,
relink=False,
quiet=False,
):
if path_info.scheme not in ["local", cache.fs.scheme]:
raise NotImplementedError
failed = None
skip = False
if not obj:
if not quiet:
logger.warning(
"No file hash info found for '%s'. It won't be created.",
path_info,
)
_remove(path_info, fs, cache, force=force)
failed = path_info
elif not relink and not _changed(path_info, fs, obj, cache):
logger.trace("Data '%s' didn't change.", path_info)
skip = True
else:
try:
check(cache, obj)
except (FileNotFoundError, ObjectFormatError):
if not quiet:
logger.warning(
"Cache '%s' not found. File '%s' won't be created.",
obj.hash_info,
path_info,
)
_remove(path_info, fs, cache, force=force)
failed = path_info
if failed or skip:
if progress_callback and obj:
progress_callback(
str(path_info), len(obj),
)
if failed:
raise CheckoutError([failed])
return
logger.debug("Checking out '%s' with cache '%s'.", path_info, obj)
return _checkout(
path_info, fs, obj, cache, force, progress_callback, relink,
)
def _checkout(
diff,
fs_path,
fs,
cache,
force=False,
progress_callback=None,
relink=False,
state=None,
):
if not diff:
return
links = test_links(cache.cache_types, cache.fs, cache.fs_path, fs, fs_path)
if not links:
raise CacheLinkError([fs_path])
link = Link(links)
for change in diff.deleted:
entry_path = (
fs.path.join(fs_path, *change.old.key)
if change.old.key != ROOT
else fs_path
)
_remove(entry_path, fs, change.old.in_cache, force=force)
failed = []
for change in chain(diff.added, diff.modified):
entry_path = (
fs.path.join(fs_path, *change.new.key)
if change.new.key != ROOT
else fs_path
)
if change.new.oid.isdir:
fs.makedirs(entry_path)
continue
try:
_checkout_file(
link,
entry_path,
fs,
change,
cache,
force,
progress_callback,
relink,
state=state,
)
except CheckoutError as exc:
failed.extend(exc.target_infos)
if failed:
raise CheckoutError(failed)
def __call__(self, cache, from_path, to_fs, to_path):
if to_fs.exists(to_path):
to_fs.remove(to_path) # broken symlink
cache.makedirs(cache.fs.path.parent(to_path))
try:
transfer(cache.fs, from_path, to_fs, to_path, links=self._links)
except FileNotFoundError as exc:
raise CheckoutError([to_path]) from exc
except OSError as exc:
raise CacheLinkError([to_path]) from exc
def _checkout_dir(
path_info,
fs,
obj,
cache,
force,
progress_callback=None,
relink=False,
dvcignore: Optional[DvcIgnoreFilter] = None,
state=None,
):
modified = False
# Create dir separately so that dir is created
# even if there are no files in it
if not fs.exists(path_info):
modified = True
fs.makedirs(path_info)
logger.debug("Linking directory '%s'.", path_info)
failed = []
for entry_key, entry_obj in obj:
try:
entry_modified = _checkout_file(
path_info.joinpath(*entry_key),
fs,
entry_obj,
cache,
force,
progress_callback,
relink,
state=None,
)
if entry_modified:
modified = True
except CheckoutError as exc:
failed.extend(exc.target_infos)
modified = (
_remove_redundant_files(
path_info, fs, obj, cache, force, dvcignore=dvcignore
)
or modified
)
if failed:
raise CheckoutError(failed)
if state:
state.save(path_info, fs, obj.hash_info)
# relink is not modified, assume it as nochange
return modified and not relink
def _checkout(self, *args, **kwargs):
from dvc_data.checkout import CheckoutError as _CheckoutError
from dvc_data.checkout import LinkError, PromptError
try:
return checkout(*args, **kwargs)
except PromptError as exc:
raise ConfirmRemoveError(exc.path)
except LinkError as exc:
raise CacheLinkError([exc.path])
except _CheckoutError as exc:
raise CheckoutError(exc.paths)
def checkout(
path_info,
fs,
obj,
cache,
force=False,
progress_callback=None,
relink=False,
quiet=False,
dvcignore: Optional[DvcIgnoreFilter] = None,
state=None,
):
if path_info.scheme not in ["local", cache.fs.scheme]:
raise NotImplementedError
failed = None
skip = False
if not obj:
if not quiet:
logger.warning(
"No file hash info found for '%s'. It won't be created.",
path_info,
)
_remove(path_info, fs, cache, force=force)
failed = path_info
elif not relink and not _changed(path_info, fs, obj, cache, state=state):
logger.trace("Data '%s' didn't change.", path_info) # type: ignore
skip = True
if failed or skip:
if progress_callback and obj:
progress_callback(str(path_info), len(obj))
if failed:
raise CheckoutError([failed])
return
logger.debug("Checking out '%s' with cache '%s'.", path_info, obj)
return _checkout(
path_info,
fs,
obj,
cache,
force,
progress_callback,
relink,
dvcignore=dvcignore,
state=state,
)
def checkout(
self,
targets=None,
with_deps=False,
force=False,
relink=False,
recursive=False,
allow_missing=False,
**kwargs,
):
unused = []
stats = {
"added": [],
"deleted": [],
"modified": [],
"failed": [],
}
if not targets:
targets = [None]
unused = _get_unused_links(self)
stats["deleted"] = [_fspath_dir(u) for u in unused]
self.state.remove_links(unused, self.fs)
if isinstance(targets, str):
targets = [targets]
pairs = _collect_pairs(self, targets, with_deps, recursive)
total = get_all_files_numbers(pairs)
with Tqdm(
total=total, unit="file", desc="Checkout", disable=total == 0
) as pbar:
for stage, filter_info in pairs:
result = stage.checkout(
force=force,
progress_callback=pbar.update_msg,
relink=relink,
filter_info=filter_info,
allow_missing=allow_missing,
**kwargs,
)
for key, items in result.items():
stats[key].extend(_fspath_dir(path) for path in items)
if stats.get("failed"):
raise CheckoutError(stats["failed"], stats)
del stats["failed"]
return stats
def _checkout(
self,
targets=None,
with_deps=False,
force=False,
relink=False,
recursive=False,
):
from dvc.stage import StageFileDoesNotExistError, StageFileBadNameError
if not targets:
targets = [None]
_cleanup_unused_links(self)
pairs = set()
for target in targets:
try:
pairs.update(
self.collect_granular(
target, with_deps=with_deps, recursive=recursive
)
)
except (StageFileDoesNotExistError, StageFileBadNameError) as exc:
if not target:
raise
raise CheckoutErrorSuggestGit(target) from exc
total = get_all_files_numbers(pairs)
if total == 0:
logger.info("Nothing to do")
failed = []
with Tqdm(
total=total, unit="file", desc="Checkout", disable=total == 0
) as pbar:
for stage, filter_info in pairs:
failed.extend(
stage.checkout(
force=force,
progress_callback=pbar.update_desc,
relink=relink,
filter_info=filter_info,
)
)
if failed:
raise CheckoutError(failed)
def _checkout(
diff,
path_info,
fs,
cache,
force=False,
progress_callback=None,
relink=False,
state=None,
):
if not diff:
return
for change in diff.deleted:
entry_path = (path_info.joinpath(
*change.old.key) if change.old.key != ROOT else path_info)
_remove(entry_path, fs, change.old.in_cache, force=force)
failed = []
for change in chain(diff.added, diff.modified):
entry_path = (path_info.joinpath(
*change.new.key) if change.new.key != ROOT else path_info)
if isinstance(change.new.obj, Tree):
fs.makedirs(entry_path)
continue
try:
_checkout_file(
entry_path,
fs,
change,
cache,
force,
progress_callback,
relink,
state=state,
)
except CheckoutError as exc:
failed.extend(exc.target_infos)
if failed:
raise CheckoutError(failed)
def _link(cache, from_info, to_info):
cache.makedirs(to_info.parent)
try:
_try_links(cache, from_info, to_info, cache.cache_types)
except FileNotFoundError as exc:
raise CheckoutError([str(to_info)]) from exc
def checkout(
path_info,
fs,
obj,
cache,
force=False,
progress_callback=None,
relink=False,
quiet=False,
dvcignore: Optional[DvcIgnoreFilter] = None,
state=None,
):
if path_info.scheme not in ["local", cache.fs.scheme]:
raise NotImplementedError
diff = _diff(
path_info,
fs,
obj,
cache,
relink=relink,
dvcignore=dvcignore,
)
failed = []
if not obj:
if not quiet:
logger.warning(
"No file hash info found for '%s'. It won't be created.",
path_info,
)
failed.append(str(path_info))
elif not diff:
logger.trace("Data '%s' didn't change.", path_info) # type: ignore
try:
_checkout(
diff,
path_info,
fs,
cache,
force=force,
progress_callback=progress_callback,
relink=relink,
state=state,
)
except CheckoutError as exc:
failed.extend(exc.target_infos)
if diff and state:
state.save_link(path_info, fs)
if not failed:
state.save(path_info, fs, obj.hash_info)
if failed or not diff:
if progress_callback and obj:
progress_callback(str(path_info), len(obj))
if failed:
raise CheckoutError(failed)
return
return bool(diff) and not relink
def checkout(
path_info,
tree,
hash_info,
cache,
force=False,
progress_callback=None,
relink=False,
filter_info=None,
quiet=False,
):
if path_info.scheme not in ["local", cache.tree.scheme]:
raise NotImplementedError
failed = None
skip = False
if not hash_info:
if not quiet:
logger.warning(
"No file hash info found for '%s'. It won't be created.",
path_info,
)
_remove(path_info, tree, cache, force=force)
failed = path_info
elif not relink and not _changed(
path_info, tree, hash_info, cache, filter_info=filter_info
):
logger.trace("Data '%s' didn't change.", path_info)
skip = True
elif cache.changed_cache(
hash_info, path_info=path_info, filter_info=filter_info
):
if not quiet:
logger.warning(
"Cache '%s' not found. File '%s' won't be created.",
hash_info,
path_info,
)
_remove(path_info, tree, cache, force=force)
failed = path_info
if failed or skip:
if progress_callback:
progress_callback(
str(path_info),
cache.get_files_number(path_info, hash_info, filter_info),
)
if failed:
raise CheckoutError([failed])
return
logger.debug("Checking out '%s' with cache '%s'.", path_info, hash_info)
return _checkout(
path_info,
tree,
hash_info,
cache,
force,
progress_callback,
relink,
filter_info,
)
def checkout(
self,
targets=None,
with_deps=False,
force=False,
relink=False,
recursive=False,
allow_persist_missing=False,
):
from dvc.stage.exceptions import (
StageFileBadNameError,
StageFileDoesNotExistError,
)
unused = []
stats = {
"added": [],
"deleted": [],
"modified": [],
"failed": [],
}
if not targets:
targets = [None]
unused = _get_unused_links(self)
stats["deleted"] = [_fspath_dir(u) for u in unused]
self.state.remove_links(unused)
if isinstance(targets, str):
targets = [targets]
pairs = set()
for target in targets:
try:
pairs.update(
self.collect_granular(
target, with_deps=with_deps, recursive=recursive
)
)
except (
StageFileDoesNotExistError,
StageFileBadNameError,
NoOutputOrStageError,
) as exc:
if not target:
raise
raise CheckoutErrorSuggestGit(target) from exc
total = get_all_files_numbers(pairs)
with Tqdm(
total=total, unit="file", desc="Checkout", disable=total == 0
) as pbar:
for stage, filter_info in pairs:
result = stage.checkout(
force=force,
progress_callback=pbar.update_msg,
relink=relink,
filter_info=filter_info,
allow_persist_missing=allow_persist_missing,
)
for key, items in result.items():
stats[key].extend(_fspath_dir(path) for path in items)
if stats.get("failed"):
raise CheckoutError(stats["failed"], stats)
del stats["failed"]
return stats
