def _checkout(self, target=None, with_deps=False, force=False, recursive=False): from dvc.stage import StageFileDoesNotExistError, StageFileBadNameError try: stages = self.collect(target, with_deps=with_deps, recursive=recursive) except (StageFileDoesNotExistError, StageFileBadNameError) as exc: if not target: raise raise CheckoutErrorSuggestGit(target, exc) _cleanup_unused_links(self, self.stages) total = get_all_files_numbers(stages) if total == 0: logger.info("Nothing to do") failed = [] with Tqdm(total=total, unit="file", desc="Checkout", disable=total == 0) as pbar: for stage in stages: if stage.locked: logger.warning( "DVC-file '{path}' is locked. Its dependencies are" " not going to be checked out.".format(path=stage.relpath)) failed.extend( stage.checkout(force=force, progress_callback=pbar.update_desc)) if failed: raise CheckoutError(failed)
def checkout( self, path_info, checksum_info, force=False, progress_callback=None, relink=False, filter_info=None, ): if path_info.scheme not in ["local", self.scheme]: raise NotImplementedError checksum = checksum_info.get(self.PARAM_CHECKSUM) failed = None skip = False if not checksum: logger.warning( "No file hash info found for '%s'. " "It won't be created.", path_info, ) self.safe_remove(path_info, force=force) failed = path_info elif not relink and not self.changed(path_info, checksum_info): logger.debug("Data '%s' didn't change.", path_info) skip = True elif self.changed_cache(checksum, path_info=path_info, filter_info=filter_info): logger.warning( "Cache '%s' not found. File '%s' won't be created.", checksum, path_info, ) self.safe_remove(path_info, force=force) failed = path_info if failed or skip: if progress_callback: progress_callback( str(path_info), self.get_files_number(self.path_info, checksum, filter_info), ) if failed: raise CheckoutError([failed]) return logger.debug("Checking out '%s' with cache '%s'.", path_info, checksum) return self._checkout( path_info, checksum, force, progress_callback, relink, filter_info, )
def _checkout(self, targets=None, with_deps=False, force=False, recursive=False): from dvc.stage import StageFileDoesNotExistError, StageFileBadNameError stages = set() targets = targets or [None] for target in targets: try: new = self.collect(target, with_deps=with_deps, recursive=recursive) stages.update(new) except (StageFileDoesNotExistError, StageFileBadNameError) as exc: if not target: raise raise CheckoutErrorSuggestGit(target, exc) _cleanup_unused_links(self, self.stages) total = get_all_files_numbers(stages) if total == 0: logger.info("Nothing to do") failed = [] with Tqdm(total=total, unit="file", desc="Checkout", disable=total == 0) as pbar: for stage in stages: failed.extend( stage.checkout(force=force, progress_callback=pbar.update_desc)) if failed: raise CheckoutError(failed)
def _checkout( self, targets=None, with_deps=False, force=False, relink=False, recursive=False, ): from dvc.stage import StageFileDoesNotExistError, StageFileBadNameError unused = [] stats = { "added": [], "deleted": [], "modified": [], "failed": [], } if not targets: targets = [None] unused = _get_unused_links(self) stats["deleted"] = [_fspath_dir(u, self.root_dir) for u in unused] self.state.remove_links(unused) pairs = set() for target in targets: try: pairs.update( self.collect_granular( target, with_deps=with_deps, recursive=recursive ) ) except (StageFileDoesNotExistError, StageFileBadNameError) as exc: if not target: raise raise CheckoutErrorSuggestGit(target) from exc total = get_all_files_numbers(pairs) with Tqdm( total=total, unit="file", desc="Checkout", disable=total == 0 ) as pbar: for stage, filter_info in pairs: result = stage.checkout( force=force, progress_callback=pbar.update_desc, relink=relink, filter_info=filter_info, ) for data in ["failed", "added", "modified"]: stats[data].extend( _fspath_dir(path, self.root_dir) for path in result[data] ) if stats.get("failed"): raise CheckoutError(stats["failed"], stats) del stats["failed"] return stats
def checkout( path_info, fs, obj, cache, force=False, progress_callback=None, relink=False, quiet=False, ): if path_info.scheme not in ["local", cache.fs.scheme]: raise NotImplementedError failed = None skip = False if not obj: if not quiet: logger.warning( "No file hash info found for '%s'. It won't be created.", path_info, ) _remove(path_info, fs, cache, force=force) failed = path_info elif not relink and not _changed(path_info, fs, obj, cache): logger.trace("Data '%s' didn't change.", path_info) skip = True else: try: check(cache, obj) except (FileNotFoundError, ObjectFormatError): if not quiet: logger.warning( "Cache '%s' not found. File '%s' won't be created.", obj.hash_info, path_info, ) _remove(path_info, fs, cache, force=force) failed = path_info if failed or skip: if progress_callback and obj: progress_callback( str(path_info), len(obj), ) if failed: raise CheckoutError([failed]) return logger.debug("Checking out '%s' with cache '%s'.", path_info, obj) return _checkout( path_info, fs, obj, cache, force, progress_callback, relink, )
def _checkout( diff, fs_path, fs, cache, force=False, progress_callback=None, relink=False, state=None, ): if not diff: return links = test_links(cache.cache_types, cache.fs, cache.fs_path, fs, fs_path) if not links: raise CacheLinkError([fs_path]) link = Link(links) for change in diff.deleted: entry_path = ( fs.path.join(fs_path, *change.old.key) if change.old.key != ROOT else fs_path ) _remove(entry_path, fs, change.old.in_cache, force=force) failed = [] for change in chain(diff.added, diff.modified): entry_path = ( fs.path.join(fs_path, *change.new.key) if change.new.key != ROOT else fs_path ) if change.new.oid.isdir: fs.makedirs(entry_path) continue try: _checkout_file( link, entry_path, fs, change, cache, force, progress_callback, relink, state=state, ) except CheckoutError as exc: failed.extend(exc.target_infos) if failed: raise CheckoutError(failed)
def __call__(self, cache, from_path, to_fs, to_path): if to_fs.exists(to_path): to_fs.remove(to_path) # broken symlink cache.makedirs(cache.fs.path.parent(to_path)) try: transfer(cache.fs, from_path, to_fs, to_path, links=self._links) except FileNotFoundError as exc: raise CheckoutError([to_path]) from exc except OSError as exc: raise CacheLinkError([to_path]) from exc
def _checkout_dir( path_info, fs, obj, cache, force, progress_callback=None, relink=False, dvcignore: Optional[DvcIgnoreFilter] = None, state=None, ): modified = False # Create dir separately so that dir is created # even if there are no files in it if not fs.exists(path_info): modified = True fs.makedirs(path_info) logger.debug("Linking directory '%s'.", path_info) failed = [] for entry_key, entry_obj in obj: try: entry_modified = _checkout_file( path_info.joinpath(*entry_key), fs, entry_obj, cache, force, progress_callback, relink, state=None, ) if entry_modified: modified = True except CheckoutError as exc: failed.extend(exc.target_infos) modified = ( _remove_redundant_files( path_info, fs, obj, cache, force, dvcignore=dvcignore ) or modified ) if failed: raise CheckoutError(failed) if state: state.save(path_info, fs, obj.hash_info) # relink is not modified, assume it as nochange return modified and not relink
def _checkout(self, *args, **kwargs): from dvc_data.checkout import CheckoutError as _CheckoutError from dvc_data.checkout import LinkError, PromptError try: return checkout(*args, **kwargs) except PromptError as exc: raise ConfirmRemoveError(exc.path) except LinkError as exc: raise CacheLinkError([exc.path]) except _CheckoutError as exc: raise CheckoutError(exc.paths)
def checkout( path_info, fs, obj, cache, force=False, progress_callback=None, relink=False, quiet=False, dvcignore: Optional[DvcIgnoreFilter] = None, state=None, ): if path_info.scheme not in ["local", cache.fs.scheme]: raise NotImplementedError failed = None skip = False if not obj: if not quiet: logger.warning( "No file hash info found for '%s'. It won't be created.", path_info, ) _remove(path_info, fs, cache, force=force) failed = path_info elif not relink and not _changed(path_info, fs, obj, cache, state=state): logger.trace("Data '%s' didn't change.", path_info) # type: ignore skip = True if failed or skip: if progress_callback and obj: progress_callback(str(path_info), len(obj)) if failed: raise CheckoutError([failed]) return logger.debug("Checking out '%s' with cache '%s'.", path_info, obj) return _checkout( path_info, fs, obj, cache, force, progress_callback, relink, dvcignore=dvcignore, state=state, )
def checkout( self, targets=None, with_deps=False, force=False, relink=False, recursive=False, allow_missing=False, **kwargs, ): unused = [] stats = { "added": [], "deleted": [], "modified": [], "failed": [], } if not targets: targets = [None] unused = _get_unused_links(self) stats["deleted"] = [_fspath_dir(u) for u in unused] self.state.remove_links(unused, self.fs) if isinstance(targets, str): targets = [targets] pairs = _collect_pairs(self, targets, with_deps, recursive) total = get_all_files_numbers(pairs) with Tqdm( total=total, unit="file", desc="Checkout", disable=total == 0 ) as pbar: for stage, filter_info in pairs: result = stage.checkout( force=force, progress_callback=pbar.update_msg, relink=relink, filter_info=filter_info, allow_missing=allow_missing, **kwargs, ) for key, items in result.items(): stats[key].extend(_fspath_dir(path) for path in items) if stats.get("failed"): raise CheckoutError(stats["failed"], stats) del stats["failed"] return stats
def _checkout( self, targets=None, with_deps=False, force=False, relink=False, recursive=False, ): from dvc.stage import StageFileDoesNotExistError, StageFileBadNameError if not targets: targets = [None] _cleanup_unused_links(self) pairs = set() for target in targets: try: pairs.update( self.collect_granular( target, with_deps=with_deps, recursive=recursive ) ) except (StageFileDoesNotExistError, StageFileBadNameError) as exc: if not target: raise raise CheckoutErrorSuggestGit(target) from exc total = get_all_files_numbers(pairs) if total == 0: logger.info("Nothing to do") failed = [] with Tqdm( total=total, unit="file", desc="Checkout", disable=total == 0 ) as pbar: for stage, filter_info in pairs: failed.extend( stage.checkout( force=force, progress_callback=pbar.update_desc, relink=relink, filter_info=filter_info, ) ) if failed: raise CheckoutError(failed)
def _checkout( diff, path_info, fs, cache, force=False, progress_callback=None, relink=False, state=None, ): if not diff: return for change in diff.deleted: entry_path = (path_info.joinpath( *change.old.key) if change.old.key != ROOT else path_info) _remove(entry_path, fs, change.old.in_cache, force=force) failed = [] for change in chain(diff.added, diff.modified): entry_path = (path_info.joinpath( *change.new.key) if change.new.key != ROOT else path_info) if isinstance(change.new.obj, Tree): fs.makedirs(entry_path) continue try: _checkout_file( entry_path, fs, change, cache, force, progress_callback, relink, state=state, ) except CheckoutError as exc: failed.extend(exc.target_infos) if failed: raise CheckoutError(failed)
def _link(cache, from_info, to_info): cache.makedirs(to_info.parent) try: _try_links(cache, from_info, to_info, cache.cache_types) except FileNotFoundError as exc: raise CheckoutError([str(to_info)]) from exc
def checkout( path_info, fs, obj, cache, force=False, progress_callback=None, relink=False, quiet=False, dvcignore: Optional[DvcIgnoreFilter] = None, state=None, ): if path_info.scheme not in ["local", cache.fs.scheme]: raise NotImplementedError diff = _diff( path_info, fs, obj, cache, relink=relink, dvcignore=dvcignore, ) failed = [] if not obj: if not quiet: logger.warning( "No file hash info found for '%s'. It won't be created.", path_info, ) failed.append(str(path_info)) elif not diff: logger.trace("Data '%s' didn't change.", path_info) # type: ignore try: _checkout( diff, path_info, fs, cache, force=force, progress_callback=progress_callback, relink=relink, state=state, ) except CheckoutError as exc: failed.extend(exc.target_infos) if diff and state: state.save_link(path_info, fs) if not failed: state.save(path_info, fs, obj.hash_info) if failed or not diff: if progress_callback and obj: progress_callback(str(path_info), len(obj)) if failed: raise CheckoutError(failed) return return bool(diff) and not relink
def checkout( path_info, tree, hash_info, cache, force=False, progress_callback=None, relink=False, filter_info=None, quiet=False, ): if path_info.scheme not in ["local", cache.tree.scheme]: raise NotImplementedError failed = None skip = False if not hash_info: if not quiet: logger.warning( "No file hash info found for '%s'. It won't be created.", path_info, ) _remove(path_info, tree, cache, force=force) failed = path_info elif not relink and not _changed( path_info, tree, hash_info, cache, filter_info=filter_info ): logger.trace("Data '%s' didn't change.", path_info) skip = True elif cache.changed_cache( hash_info, path_info=path_info, filter_info=filter_info ): if not quiet: logger.warning( "Cache '%s' not found. File '%s' won't be created.", hash_info, path_info, ) _remove(path_info, tree, cache, force=force) failed = path_info if failed or skip: if progress_callback: progress_callback( str(path_info), cache.get_files_number(path_info, hash_info, filter_info), ) if failed: raise CheckoutError([failed]) return logger.debug("Checking out '%s' with cache '%s'.", path_info, hash_info) return _checkout( path_info, tree, hash_info, cache, force, progress_callback, relink, filter_info, )
def checkout( self, targets=None, with_deps=False, force=False, relink=False, recursive=False, allow_persist_missing=False, ): from dvc.stage.exceptions import ( StageFileBadNameError, StageFileDoesNotExistError, ) unused = [] stats = { "added": [], "deleted": [], "modified": [], "failed": [], } if not targets: targets = [None] unused = _get_unused_links(self) stats["deleted"] = [_fspath_dir(u) for u in unused] self.state.remove_links(unused) if isinstance(targets, str): targets = [targets] pairs = set() for target in targets: try: pairs.update( self.collect_granular( target, with_deps=with_deps, recursive=recursive ) ) except ( StageFileDoesNotExistError, StageFileBadNameError, NoOutputOrStageError, ) as exc: if not target: raise raise CheckoutErrorSuggestGit(target) from exc total = get_all_files_numbers(pairs) with Tqdm( total=total, unit="file", desc="Checkout", disable=total == 0 ) as pbar: for stage, filter_info in pairs: result = stage.checkout( force=force, progress_callback=pbar.update_msg, relink=relink, filter_info=filter_info, allow_persist_missing=allow_persist_missing, ) for key, items in result.items(): stats[key].extend(_fspath_dir(path) for path in items) if stats.get("failed"): raise CheckoutError(stats["failed"], stats) del stats["failed"] return stats