class Repo: DVC_DIR = ".dvc" from dvc.repo.add import add from dvc.repo.checkout import checkout from dvc.repo.commit import commit from dvc.repo.destroy import destroy from dvc.repo.diff import diff from dvc.repo.fetch import fetch from dvc.repo.freeze import freeze, unfreeze from dvc.repo.gc import gc from dvc.repo.get import get as _get from dvc.repo.get_url import get_url as _get_url from dvc.repo.imp import imp from dvc.repo.imp_url import imp_url from dvc.repo.install import install from dvc.repo.ls import ls as _ls from dvc.repo.move import move from dvc.repo.pull import pull from dvc.repo.push import push from dvc.repo.remove import remove from dvc.repo.reproduce import reproduce from dvc.repo.run import run from dvc.repo.status import status from dvc.repo.update import update ls = staticmethod(_ls) get = staticmethod(_get) get_url = staticmethod(_get_url) def _get_repo_dirs( self, root_dir: str = None, scm: "Base" = None, rev: str = None, uninitialized: bool = False, ): assert bool(scm) == bool(rev) from dvc.scm import SCM from dvc.scm.base import SCMError from dvc.scm.git import Git from dvc.utils.fs import makedirs dvc_dir = None tmp_dir = None try: fs = scm.get_fs(rev) if isinstance(scm, Git) and rev else None root_dir = self.find_root(root_dir, fs) dvc_dir = os.path.join(root_dir, self.DVC_DIR) tmp_dir = os.path.join(dvc_dir, "tmp") makedirs(tmp_dir, exist_ok=True) except NotDvcRepoError: if not uninitialized: raise try: scm = SCM(root_dir or os.curdir) except SCMError: scm = SCM(os.curdir, no_scm=True) from dvc.scm import Base assert isinstance(scm, Base) root_dir = scm.root_dir return root_dir, dvc_dir, tmp_dir def __init__( self, root_dir=None, scm=None, rev=None, subrepos=False, uninitialized=False, config=None, url=None, repo_factory=None, ): from dvc.config import Config from dvc.data_cloud import DataCloud from dvc.fs.local import LocalFileSystem from dvc.lock import LockNoop, make_lock from dvc.machine import MachineManager from dvc.objects.db import ODBManager from dvc.repo.live import Live from dvc.repo.metrics import Metrics from dvc.repo.params import Params from dvc.repo.plots import Plots from dvc.repo.stage import StageLoad from dvc.scm import SCM from dvc.stage.cache import StageCache from dvc.state import State, StateNoop self.url = url self._fs_conf = {"repo_factory": repo_factory} if rev and not scm: scm = SCM(root_dir or os.curdir) self.root_dir, self.dvc_dir, self.tmp_dir = self._get_repo_dirs( root_dir=root_dir, scm=scm, rev=rev, uninitialized=uninitialized ) if scm: self._fs = scm.get_fs(rev) else: self._fs = LocalFileSystem(url=self.root_dir) self.config = Config(self.dvc_dir, fs=self.fs, config=config) self._uninitialized = uninitialized self._scm = scm # used by RepoFileSystem to determine if it should traverse subrepos self.subrepos = subrepos self.cloud = DataCloud(self) self.stage = StageLoad(self) if scm or not self.dvc_dir: self.lock = LockNoop() self.state = StateNoop() self.odb = ODBManager(self) else: self.lock = make_lock( os.path.join(self.tmp_dir, "lock"), tmp_dir=self.tmp_dir, hardlink_lock=self.config["core"].get("hardlink_lock", False), friendly=True, ) # NOTE: storing state and link_state in the repository itself to # avoid any possible state corruption in 'shared cache dir' # scenario. self.state = State(self.root_dir, self.tmp_dir, self.dvcignore) self.odb = ODBManager(self) self.stage_cache = StageCache(self) self._ignore() self.metrics = Metrics(self) self.plots = Plots(self) self.params = Params(self) self.live = Live(self) if self.tmp_dir and ( self.config["feature"].get("machine", False) or env2bool("DVC_TEST") ): self.machine = MachineManager(self) else: self.machine = None self.stage_collection_error_handler: Optional[ Callable[[str, Exception], None] ] = None self._lock_depth = 0 def __str__(self): return self.url or self.root_dir @cached_property def index(self): from dvc.repo.index import Index return Index(self) @staticmethod def open(url, *args, **kwargs): if url is None: url = os.getcwd() if os.path.exists(url): try: return Repo(url, *args, **kwargs) except NotDvcRepoError: pass # fallthrough to external_repo from dvc.external_repo import external_repo return external_repo(url, *args, **kwargs) @cached_property def scm(self): from dvc.scm import SCM from dvc.scm.base import SCMError if self._scm: return self._scm no_scm = self.config["core"].get("no_scm", False) try: return SCM(self.root_dir, no_scm=no_scm) except SCMError: if self._uninitialized: # might not be a git/dvc repo at all # used in `params/metrics/plots/live` targets return SCM(self.root_dir, no_scm=True) raise @cached_property def dvcignore(self) -> DvcIgnoreFilter: return DvcIgnoreFilter(self.fs, self.root_dir) def get_rev(self): from dvc.fs.local import LocalFileSystem assert self.scm if isinstance(self.fs, LocalFileSystem): return self.scm.get_rev() return self.fs.rev @cached_property def experiments(self): from dvc.repo.experiments import Experiments return Experiments(self) @property def fs(self) -> "BaseFileSystem": return self._fs @fs.setter def fs(self, fs: "BaseFileSystem"): self._fs = fs # Our graph cache is no longer valid, as it was based on the previous # fs. self._reset() def __repr__(self): return f"{self.__class__.__name__}: '{self.root_dir}'" @classmethod def find_root(cls, root=None, fs=None) -> str: root_dir = os.path.realpath(root or os.curdir) if fs: if fs.isdir(os.path.join(root_dir, cls.DVC_DIR)): return root_dir raise NotDvcRepoError(f"'{root}' does not contain DVC directory") if not os.path.isdir(root_dir): raise NotDvcRepoError(f"directory '{root}' does not exist") while True: dvc_dir = os.path.join(root_dir, cls.DVC_DIR) if os.path.isdir(dvc_dir): return root_dir if os.path.ismount(root_dir): break root_dir = os.path.dirname(root_dir) message = ( "you are not inside of a DVC repository " "(checked up to mount point '{}')" ).format(root_dir) raise NotDvcRepoError(message) @classmethod def find_dvc_dir(cls, root=None): root_dir = cls.find_root(root) return os.path.join(root_dir, cls.DVC_DIR) @staticmethod def init(root_dir=os.curdir, no_scm=False, force=False, subdir=False): from dvc.repo.init import init return init( root_dir=root_dir, no_scm=no_scm, force=force, subdir=subdir ) def unprotect(self, target): return self.odb.local.unprotect(PathInfo(target)) def _ignore(self): flist = [self.config.files["local"], self.tmp_dir] if path_isin(self.odb.local.cache_dir, self.root_dir): flist += [self.odb.local.cache_dir] self.scm.ignore_list(flist) def brancher(self, *args, **kwargs): from dvc.repo.brancher import brancher return brancher(self, *args, **kwargs) def used_objs( self, targets=None, all_branches=False, with_deps=False, all_tags=False, all_commits=False, all_experiments=False, remote=None, force=False, jobs=None, recursive=False, used_run_cache=None, revs=None, ): """Get the stages related to the given target and collect the `info` of its outputs. This is useful to know what files from the cache are _in use_ (namely, a file described as an output on a stage). The scope is, by default, the working directory, but you can use `all_branches`/`all_tags`/`all_commits`/`all_experiments` to expand the scope. Returns: A dict mapping (remote) ODB instances to sets of objects that belong to each ODB. If the ODB instance is None, the objects are naive and do not belong to a specific remote ODB. """ used = defaultdict(set) def _add_suffix(objs: Set["HashFile"], suffix: str) -> None: from itertools import chain from dvc.objects import iterobjs for obj in chain.from_iterable(map(iterobjs, objs)): if obj.name is not None: obj.name += suffix for branch in self.brancher( revs=revs, all_branches=all_branches, all_tags=all_tags, all_commits=all_commits, all_experiments=all_experiments, ): for odb, objs in self.index.used_objs( targets, remote=remote, force=force, jobs=jobs, recursive=recursive, with_deps=with_deps, ).items(): if branch: _add_suffix(objs, f" ({branch})") used[odb].update(objs) if used_run_cache: for odb, objs in self.stage_cache.get_used_objs( used_run_cache, remote=remote, force=force, jobs=jobs ).items(): used[odb].update(objs) return used @property def stages(self): # obsolete, only for backward-compatibility return self.index.stages def find_outs_by_path(self, path, outs=None, recursive=False, strict=True): # using `outs_graph` to ensure graph checks are run outs = outs or self.index.outs_graph abs_path = os.path.abspath(path) path_info = PathInfo(abs_path) match = path_info.__eq__ if strict else path_info.isin_or_eq def func(out): if out.scheme == "local" and match(out.path_info): return True if recursive and out.path_info.isin(path_info): return True return False matched = list(filter(func, outs)) if not matched: raise OutputNotFoundError(path, self) return matched def is_dvc_internal(self, path): path_parts = os.path.normpath(path).split(os.path.sep) return self.DVC_DIR in path_parts @cached_property def dvcfs(self): from dvc.fs.dvc import DvcFileSystem return DvcFileSystem(repo=self) @cached_property def repo_fs(self): from dvc.fs.repo import RepoFileSystem return RepoFileSystem(self, subrepos=self.subrepos, **self._fs_conf) @contextmanager def open_by_relpath(self, path, remote=None, mode="r", encoding=None): """Opens a specified resource as a file descriptor""" from dvc.fs.repo import RepoFileSystem fs = RepoFileSystem(self, subrepos=True) path = PathInfo(self.root_dir) / path try: with fs.open( path, mode=mode, encoding=encoding, remote=remote ) as fobj: yield fobj except FileNotFoundError as exc: raise FileMissingError(path) from exc except IsADirectoryError as exc: raise DvcIsADirectoryError(f"'{path}' is a directory") from exc def close(self): self.scm.close() self.state.close() def _reset(self): self.state.close() self.scm._reset() # pylint: disable=protected-access self.__dict__.pop("index", None) self.__dict__.pop("dvcignore", None) def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): self._reset() self.scm.close()
class Repo: DVC_DIR = ".dvc" from dvc.repo.add import add # type: ignore[misc] from dvc.repo.checkout import checkout # type: ignore[misc] from dvc.repo.commit import commit # type: ignore[misc] from dvc.repo.destroy import destroy # type: ignore[misc] from dvc.repo.diff import diff # type: ignore[misc] from dvc.repo.fetch import fetch # type: ignore[misc] from dvc.repo.freeze import freeze, unfreeze # type: ignore[misc] from dvc.repo.gc import gc # type: ignore[misc] from dvc.repo.get import get as _get # type: ignore[misc] from dvc.repo.get_url import get_url as _get_url # type: ignore[misc] from dvc.repo.imp import imp # type: ignore[misc] from dvc.repo.imp_url import imp_url # type: ignore[misc] from dvc.repo.install import install # type: ignore[misc] from dvc.repo.ls import ls as _ls # type: ignore[misc] from dvc.repo.move import move # type: ignore[misc] from dvc.repo.pull import pull # type: ignore[misc] from dvc.repo.push import push # type: ignore[misc] from dvc.repo.remove import remove # type: ignore[misc] from dvc.repo.reproduce import reproduce # type: ignore[misc] from dvc.repo.run import run # type: ignore[misc] from dvc.repo.status import status # type: ignore[misc] from dvc.repo.update import update # type: ignore[misc] ls = staticmethod(_ls) get = staticmethod(_get) get_url = staticmethod(_get_url) def _get_repo_dirs( self, root_dir: str = None, fs: "FileSystem" = None, uninitialized: bool = False, scm: "Base" = None, ): from dvc.fs import localfs from dvc.scm import SCM, SCMError dvc_dir = None tmp_dir = None try: root_dir = self.find_root(root_dir, fs) fs = fs or localfs dvc_dir = fs.path.join(root_dir, self.DVC_DIR) tmp_dir = fs.path.join(dvc_dir, "tmp") except NotDvcRepoError: if not uninitialized: raise if not scm: try: scm = SCM(root_dir or os.curdir) except SCMError: scm = SCM(os.curdir, no_scm=True) if not fs or not root_dir: root_dir = scm.root_dir assert root_dir return root_dir, dvc_dir, tmp_dir def _get_database_dir(self, db_name): # NOTE: by default, store SQLite-based remote indexes and state's # `links` and `md5s` caches in the repository itself to avoid any # possible state corruption in 'shared cache dir' scenario, but allow # user to override this through config when, say, the repository is # located on a mounted volume — see # https://github.com/iterative/dvc/issues/4420 base_db_dir = self.config.get(db_name, {}).get("dir", None) if not base_db_dir: return self.tmp_dir import hashlib from dvc.utils.fs import makedirs root_dir_hash = hashlib.sha224( self.root_dir.encode("utf-8")).hexdigest() db_dir = os.path.join( base_db_dir, self.DVC_DIR, f"{os.path.basename(self.root_dir)}-{root_dir_hash[0:7]}", ) makedirs(db_dir, exist_ok=True) return db_dir def __init__( self, root_dir=None, fs=None, rev=None, subrepos=False, uninitialized=False, config=None, url=None, repo_factory=None, scm=None, ): from dvc.config import Config from dvc.data_cloud import DataCloud from dvc.fs import GitFileSystem, localfs from dvc.lock import LockNoop, make_lock from dvc.odbmgr import ODBManager from dvc.repo.metrics import Metrics from dvc.repo.params import Params from dvc.repo.plots import Plots from dvc.repo.stage import StageLoad from dvc.scm import SCM from dvc.stage.cache import StageCache from dvc_data.hashfile.state import State, StateNoop self.url = url self._fs_conf = {"repo_factory": repo_factory} self._fs = fs or localfs self._scm = scm if rev and not fs: self._scm = scm = SCM(root_dir or os.curdir) root_dir = "/" self._fs = GitFileSystem(scm=self._scm, rev=rev) self.root_dir, self.dvc_dir, self.tmp_dir = self._get_repo_dirs( root_dir=root_dir, fs=self.fs, uninitialized=uninitialized, scm=scm, ) self.config = Config(self.dvc_dir, fs=self.fs, config=config) self._uninitialized = uninitialized # used by DvcFileSystem to determine if it should traverse subrepos self.subrepos = subrepos self.cloud = DataCloud(self) self.stage = StageLoad(self) if isinstance(self.fs, GitFileSystem) or not self.dvc_dir: self.lock = LockNoop() self.state = StateNoop() self.odb = ODBManager(self) self.tmp_dir = None else: from dvc.utils.fs import makedirs makedirs(self.tmp_dir, exist_ok=True) self.lock = make_lock( os.path.join(self.tmp_dir, "lock"), tmp_dir=self.tmp_dir, hardlink_lock=self.config["core"].get("hardlink_lock", False), friendly=True, ) state_db_dir = self._get_database_dir("state") self.state = State(self.root_dir, state_db_dir, self.dvcignore) self.odb = ODBManager(self) self.stage_cache = StageCache(self) self._ignore() self.metrics = Metrics(self) self.plots = Plots(self) self.params = Params(self) self.stage_collection_error_handler: Optional[Callable[ [str, Exception], None]] = None self._lock_depth = 0 def __str__(self): return self.url or self.root_dir @cached_property def index(self): from dvc.repo.index import Index return Index(self) @staticmethod def open(url, *args, **kwargs): if url is None: url = os.getcwd() if os.path.exists(url): try: return Repo(url, *args, **kwargs) except NotDvcRepoError: pass # fallthrough to external_repo from dvc.external_repo import external_repo return external_repo(url, *args, **kwargs) @cached_property def scm(self): from dvc.scm import SCM, SCMError if self._scm: return self._scm no_scm = self.config["core"].get("no_scm", False) try: return SCM(self.root_dir, no_scm=no_scm) except SCMError: if self._uninitialized: # might not be a git/dvc repo at all # used in `params/metrics/plots/live` targets return SCM(self.root_dir, no_scm=True) raise @cached_property def scm_context(self) -> "SCMContext": from dvc.repo.scm_context import SCMContext return SCMContext(self.scm, self.config) @cached_property def dvcignore(self) -> DvcIgnoreFilter: return DvcIgnoreFilter(self.fs, self.root_dir) def get_rev(self): from dvc.fs import LocalFileSystem assert self.scm if isinstance(self.fs, LocalFileSystem): from dvc.scm import map_scm_exception with map_scm_exception(): return self.scm.get_rev() return self.fs.rev @cached_property def experiments(self): from dvc.repo.experiments import Experiments return Experiments(self) @cached_property def machine(self): from dvc.machine import MachineManager if self.tmp_dir and (self.config["feature"].get("machine", False) or env2bool("DVC_TEST")): return MachineManager(self) return None @property def fs(self) -> "FileSystem": return self._fs @fs.setter def fs(self, fs: "FileSystem"): self._fs = fs # Our graph cache is no longer valid, as it was based on the previous # fs. self._reset() def __repr__(self): return f"{self.__class__.__name__}: '{self.root_dir}'" @classmethod def find_root(cls, root=None, fs=None) -> str: from dvc.fs import LocalFileSystem, localfs fs = fs or localfs root = root or os.curdir root_dir = fs.path.realpath(root) if not fs.isdir(root_dir): raise NotDvcRepoError(f"directory '{root}' does not exist") while True: dvc_dir = fs.path.join(root_dir, cls.DVC_DIR) if fs.isdir(dvc_dir): return root_dir if isinstance(fs, LocalFileSystem) and os.path.ismount(root_dir): break parent = fs.path.parent(root_dir) if parent == root_dir: break root_dir = parent msg = "you are not inside of a DVC repository" if isinstance(fs, LocalFileSystem): msg = f"{msg} (checked up to mount point '{root_dir}')" raise NotDvcRepoError(msg) @classmethod def find_dvc_dir(cls, root=None): root_dir = cls.find_root(root) return os.path.join(root_dir, cls.DVC_DIR) @staticmethod def init(root_dir=os.curdir, no_scm=False, force=False, subdir=False): from dvc.repo.init import init return init(root_dir=root_dir, no_scm=no_scm, force=force, subdir=subdir) def unprotect(self, target): return self.odb.local.unprotect(target) def _ignore(self): flist = [self.config.files["local"], self.tmp_dir] if path_isin(self.odb.local.cache_dir, self.root_dir): flist += [self.odb.local.cache_dir] for file in flist: self.scm_context.ignore(file) def brancher(self, *args, **kwargs): from dvc.repo.brancher import brancher return brancher(self, *args, **kwargs) def used_objs( self, targets=None, all_branches=False, with_deps=False, all_tags=False, all_commits=False, all_experiments=False, remote=None, force=False, jobs=None, recursive=False, used_run_cache=None, revs=None, ): """Get the stages related to the given target and collect the `info` of its outputs. This is useful to know what files from the cache are _in use_ (namely, a file described as an output on a stage). The scope is, by default, the working directory, but you can use `all_branches`/`all_tags`/`all_commits`/`all_experiments` to expand the scope. Returns: A dict mapping (remote) ODB instances to sets of objects that belong to each ODB. If the ODB instance is None, the objects are naive and do not belong to a specific remote ODB. """ used = defaultdict(set) for _ in self.brancher( revs=revs, all_branches=all_branches, all_tags=all_tags, all_commits=all_commits, all_experiments=all_experiments, ): for odb, objs in self.index.used_objs( targets, remote=remote, force=force, jobs=jobs, recursive=recursive, with_deps=with_deps, ).items(): used[odb].update(objs) if used_run_cache: for odb, objs in self.stage_cache.get_used_objs(used_run_cache, remote=remote, force=force, jobs=jobs).items(): used[odb].update(objs) return used @property def stages(self): # obsolete, only for backward-compatibility return self.index.stages def find_outs_by_path(self, path, outs=None, recursive=False, strict=True): # using `outs_graph` to ensure graph checks are run outs = outs or self.index.outs_graph abs_path = self.fs.path.abspath(path) fs_path = abs_path def func(out): def eq(one, two): return one == two match = eq if strict else out.fs.path.isin_or_eq if out.protocol == "local" and match(fs_path, out.fs_path): return True if recursive and out.fs.path.isin(out.fs_path, fs_path): return True return False matched = list(filter(func, outs)) if not matched: raise OutputNotFoundError(path, self) return matched def is_dvc_internal(self, path): path_parts = self.fs.path.normpath(path).split(self.fs.sep) return self.DVC_DIR in path_parts @cached_property def datafs(self): from dvc.fs.data import DataFileSystem return DataFileSystem(repo=self) @cached_property def dvcfs(self): from dvc.fs.dvc import DvcFileSystem return DvcFileSystem(repo=self, subrepos=self.subrepos, **self._fs_conf) @cached_property def index_db_dir(self): return self._get_database_dir("index") @contextmanager def open_by_relpath(self, path, remote=None, mode="r", encoding=None): """Opens a specified resource as a file descriptor""" from dvc.fs.data import DataFileSystem from dvc.fs.dvc import DvcFileSystem if os.path.isabs(path): fs = DataFileSystem(repo=self, workspace="local") fs_path = path else: fs = DvcFileSystem(repo=self, subrepos=True) fs_path = fs.from_os_path(path) try: with fs.open( fs_path, mode=mode, encoding=encoding, remote=remote, ) as fobj: yield fobj except FileNotFoundError as exc: raise FileMissingError(path) from exc except IsADirectoryError as exc: raise DvcIsADirectoryError(f"'{path}' is a directory") from exc def close(self): self.scm.close() self.state.close() def _reset(self): self.state.close() self.scm._reset() # pylint: disable=protected-access self.__dict__.pop("index", None) self.__dict__.pop("dvcignore", None) def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): self._reset() self.scm.close()