def daemon(args): """Launch a `dvc daemon` command in a detached process. Args: args (list): list of arguments to append to `dvc daemon` command. """ cmd = [sys.executable] if not is_binary(): cmd += ["-m", "dvc"] cmd += ["daemon", "-q"] + args env = fix_env() file_path = os.path.abspath(inspect.stack()[0][1]) env[cast_bytes_py2("PYTHONPATH")] = cast_bytes_py2( os.path.dirname(os.path.dirname(file_path)) ) logger.debug("Trying to spawn '{}' with env '{}'".format(cmd, env)) if os.name == "nt": _spawn_windows(cmd, env) elif os.name == "posix": _spawn_posix(cmd, env) else: raise NotImplementedError logger.debug("Spawned '{}'".format(cmd))
def _popen(cmd, **kwargs): prefix = [sys.executable] if not is_binary(): main_entrypoint = os.path.join( os.path.abspath(os.path.dirname(__file__)), "__main__.py") prefix += [main_entrypoint] return Popen(prefix + cmd, close_fds=True, shell=False, **kwargs)
def _get_darwin(self): if not is_binary(): if __file__.startswith("/usr/local/Cellar"): return "formula" else: return "pip" return None
def run(self): from dvc.repo import Repo dvc_version = __version__ python_version = platform.python_version() platform_type = platform.platform() binary = is_binary() package_manager = get_package_manager() info = ( "DVC version: {dvc_version}\n" "Python version: {python_version}\n" "Platform: {platform_type}\n" "Binary: {binary}\n" "Package manager: {package_manager}\n" ).format( dvc_version=dvc_version, python_version=python_version, platform_type=platform_type, binary=binary, package_manager=package_manager, ) try: repo = Repo() root_directory = repo.root_dir # cache_dir might not exist yet (e.g. after `dvc init`), and we # can't auto-create it, as it might cause issues if the user # later decides to enable shared cache mode with # `dvc config cache.shared group`. if os.path.exists(repo.cache.local.cache_dir): info += "Cache: {cache}\n".format( cache=self.get_linktype_support_info(repo) ) else: logger.warning( "Unable to detect supported link types, as cache " "directory '{}' doesn't exist. It is usually auto-created " "by commands such as `dvc add/fetch/pull/run/import`, " "but you could create it manually to enable this " "check.".format(relpath(repo.cache.local.cache_dir)) ) if psutil: info += ( "Filesystem type (cache directory): {fs_cache}\n" ).format(fs_cache=self.get_fs_type(repo.cache.local.cache_dir)) except NotDvcRepoError: root_directory = os.getcwd() if psutil: info += ("Filesystem type (workspace): {fs_root}").format( fs_root=self.get_fs_type(os.path.abspath(root_directory)) ) logger.info(info) return 0
def clone( url: str, to_path: str, rev: Optional[str] = None, shallow_branch: Optional[str] = None, ): import git ld_key = "LD_LIBRARY_PATH" env = fix_env(None) if is_binary() and ld_key not in env.keys(): # In fix_env, we delete LD_LIBRARY_PATH key if it was empty before # PyInstaller modified it. GitPython, in git.Repo.clone_from, uses # env to update its own internal state. When there is no key in # env, this value is not updated and GitPython re-uses # LD_LIBRARY_PATH that has been set by PyInstaller. # See [1] for more info. # [1] https://github.com/gitpython-developers/GitPython/issues/924 env[ld_key] = "" try: if shallow_branch is not None and os.path.exists(url): # git disables --depth for local clones unless file:// url # scheme is used url = f"file://{url}" with TqdmGit(desc="Cloning", unit="obj") as pbar: clone_from = partial( git.Repo.clone_from, url, to_path, env=env, # needed before we can fix it in __init__ no_single_branch=True, progress=pbar.update_git, ) if shallow_branch is None: tmp_repo = clone_from() else: tmp_repo = clone_from(branch=shallow_branch, depth=1) tmp_repo.close() except git.exc.GitCommandError as exc: # pylint: disable=no-member raise CloneError(url, to_path) from exc # NOTE: using our wrapper to make sure that env is fixed in __init__ repo = GitPythonBackend(to_path) if rev: try: repo.checkout(rev) except git.exc.GitCommandError as exc: # pylint: disable=no-member raise RevError( "failed to access revision '{}' for repo '{}'".format( rev, url ) ) from exc
def _get_darwin(self): if not is_binary(): if __file__.startswith("/usr/local/Cellar"): return "formula" else: return "pip" # NOTE: both pkg and cask put dvc binary into /usr/local/bin, # so in order to know which method of installation was used, # we need to actually call `brew cask` return None
def _runtime_info(): """ Gather information from the environment where DVC runs to fill a report. """ return { "dvc_version": __version__, "is_binary": is_binary(), "scm_class": _scm_in_use(), "system_info": _system_info(), "user_id": _find_or_create_user_id(), }
def run(self): from dvc.repo import Repo info = [ "DVC version: {}".format(__version__), "Python version: {}".format(platform.python_version()), "Platform: {}".format(platform.platform()), "Binary: {}".format(is_binary()), "Package: {}".format(PKG), "Supported remotes: {}".format(self.get_supported_remotes()), ] try: repo = Repo() root_directory = repo.root_dir # cache_dir might not exist yet (e.g. after `dvc init`), and we # can't auto-create it, as it might cause issues if the user # later decides to enable shared cache mode with # `dvc config cache.shared group`. if os.path.exists(repo.cache.local.cache_dir): info.append( "Cache: {}".format(self.get_linktype_support_info(repo)) ) if psutil: fs_type = self.get_fs_type(repo.cache.local.cache_dir) info.append( "Filesystem type (cache directory): {}".format(fs_type) ) else: logger.warning( "Unable to detect supported link types, as cache " "directory '{}' doesn't exist. It is usually auto-created " "by commands such as `dvc add/fetch/pull/run/import`, " "but you could create it manually to enable this " "check.".format(relpath(repo.cache.local.cache_dir)) ) except NotDvcRepoError: root_directory = os.getcwd() except SCMError: root_directory = os.getcwd() info.append("Repo: dvc, git (broken)") else: info.append("Repo: {}".format(_get_dvc_repo_info(repo))) if psutil: fs_root = self.get_fs_type(os.path.abspath(root_directory)) info.append("Filesystem type (workspace): {}".format(fs_root)) logger.info("\n".join(info)) return 0
def _get_darwin(self): if not is_binary(): if __file__.startswith('/usr/local/Cellar'): return 'formula' else: return 'pip' # NOTE: both pkg and cask put dvc binary into /usr/local/bin, # so in order to know which method of installation was used, # we need to actually call `brew cask` ret = os.system('brew cask ls dvc') if ret == 0: return 'cask' return None
def __call__(self, args): cmd = [sys.executable] if not is_binary(): cmd += ['-m', 'dvc'] cmd += ['daemon', '-q'] + args logger.debug("Trying to spawn '{}'".format(cmd)) if os.name == 'nt': self._spawn_windows(cmd) elif os.name == 'posix': self._spawn_posix(cmd) else: raise NotImplementedError logger.debug("Spawned '{}'".format(cmd))
def _get_linux(self): import distro if not is_binary(): return 'pip' package_managers = { 'rhel': 'yum', 'centos': 'yum', 'fedora': 'yum', 'amazon': 'yum', 'opensuse': 'yum', 'ubuntu': 'apt', 'debian': 'apt', } return package_managers.get(distro.id())
def _get_linux(self): import distro if not is_binary(): return "pip" package_managers = { "rhel": "yum", "centos": "yum", "fedora": "yum", "amazon": "yum", "opensuse": "yum", "ubuntu": "apt", "debian": "apt", } return package_managers.get(distro.id())
def collect(self): from dvc.scm import SCM from dvc.utils import is_binary from dvc.project import Project from dvc.exceptions import NotDvcProjectError self.info[self.PARAM_DVC_VERSION] = VERSION self.info[self.PARAM_IS_BINARY] = is_binary() self.info[self.PARAM_USER_ID] = self._get_user_id() self.info[self.PARAM_SYSTEM_INFO] = self._collect_system_info() try: scm = SCM(root_dir=Project._find_root()) self.info[self.PARAM_SCM_CLASS] = type(scm).__name__ except NotDvcProjectError: pass
def collect(self): """Collect analytics report.""" from dvc.scm import SCM from dvc.utils import is_binary from dvc.repo import Repo from dvc.exceptions import NotDvcRepoError self.info[self.PARAM_DVC_VERSION] = __version__ self.info[self.PARAM_IS_BINARY] = is_binary() self.info[self.PARAM_USER_ID] = self._get_user_id() self.info[self.PARAM_SYSTEM_INFO] = self._collect_system_info() try: scm = SCM(root_dir=Repo.find_root()) self.info[self.PARAM_SCM_CLASS] = type(scm).__name__ except NotDvcRepoError: pass
def daemon(args): """Launch a `dvc daemon` command in a detached process. Args: args (list): list of arguments to append to `dvc daemon` command. """ if os.environ.get(DVC_DAEMON): logger.debug("skipping launching a new daemon.") return cmd = ["daemon", "-q"] + args env = fix_env() if not is_binary(): file_path = os.path.abspath(inspect.stack()[0][1]) env["PYTHONPATH"] = os.path.dirname(os.path.dirname(file_path)) env[DVC_DAEMON] = "1" _spawn(cmd, env)
def clone(url, to_path, rev=None): import git ld_key = "LD_LIBRARY_PATH" env = fix_env(None) if is_binary() and ld_key not in env.keys(): # In fix_env, we delete LD_LIBRARY_PATH key if it was empty before # PyInstaller modified it. GitPython, in git.Repo.clone_from, uses # env to update its own internal state. When there is no key in # env, this value is not updated and GitPython re-uses # LD_LIBRARY_PATH that has been set by PyInstaller. # See [1] for more info. # [1] https://github.com/gitpython-developers/GitPython/issues/924 env[ld_key] = "" try: with TqdmGit(desc="Cloning", unit="obj") as pbar: tmp_repo = git.Repo.clone_from( url, to_path, env=env, # needed before we can fix it in __init__ no_single_branch=True, progress=pbar.update_git, ) tmp_repo.close() except git.exc.GitCommandError as exc: # pylint: disable=no-member raise CloneError(url, to_path) from exc # NOTE: using our wrapper to make sure that env is fixed in __init__ repo = Git(to_path) if rev: try: repo.checkout(rev) except git.exc.GitCommandError as exc: # pylint: disable=no-member raise RevError( "failed to access revision '{}' for repo '{}'".format( rev, url ) ) from exc return repo
def run(self): from dvc.repo import Repo dvc_version = __version__ python_version = platform.python_version() platform_type = platform.platform() binary = is_binary() info = ( "DVC version: {dvc_version}\n" "Python version: {python_version}\n" "Platform: {platform_type}\n" "Binary: {binary}\n" ).format( dvc_version=dvc_version, python_version=python_version, platform_type=platform_type, binary=binary, ) try: repo = Repo() root_directory = repo.root_dir info += "Cache: {cache}\n".format( cache=self.get_linktype_support_info(repo) ) if psutil: info += ( "Filesystem type (cache directory): {fs_cache}\n" ).format(fs_cache=self.get_fs_type(repo.cache.local.cache_dir)) except NotDvcRepoError: root_directory = os.getcwd() if psutil: info += ("Filesystem type (workspace): {fs_root}").format( fs_root=self.get_fs_type(os.path.abspath(root_directory)) ) logger.info(info) return 0
def _spawn_windows(cmd, env): from subprocess import STARTUPINFO, STARTF_USESHOWWINDOW prefix = [sys.executable] if not is_binary(): prefix += [sys.argv[0]] creationflags = CREATE_NEW_PROCESS_GROUP | DETACHED_PROCESS startupinfo = STARTUPINFO() startupinfo.dwFlags |= STARTF_USESHOWWINDOW Popen( prefix + cmd, env=env, close_fds=True, shell=False, creationflags=creationflags, startupinfo=startupinfo, ).communicate()
def daemon(args): """Launch a `dvc daemon` command in a detached process. Args: args (list): list of arguments to append to `dvc daemon` command. """ cmd = [sys.executable] if not is_binary(): cmd += ['-m', 'dvc'] cmd += ['daemon', '-q'] + args logger.debug("Trying to spawn '{}'".format(cmd)) if os.name == 'nt': _spawn_windows(cmd) elif os.name == 'posix': _spawn_posix(cmd) else: raise NotImplementedError logger.debug("Spawned '{}'".format(cmd))
def daemon(args): """Launch a `dvc daemon` command in a detached process. Args: args (list): list of arguments to append to `dvc daemon` command. """ if os.environ.get(DVC_DAEMON): logger.debug("skipping launching a new daemon.") return cmd = [sys.executable] if not is_binary(): cmd += [sys.argv[0]] cmd += ["daemon", "-q"] + args env = fix_env() file_path = os.path.abspath(inspect.stack()[0][1]) env[cast_bytes_py2("PYTHONPATH")] = cast_bytes_py2( os.path.dirname(os.path.dirname(file_path))) env[cast_bytes_py2(DVC_DAEMON)] = cast_bytes_py2("1") _spawn(cmd, env)
def clone(url, to_path, rev=None): import git ld_key = "LD_LIBRARY_PATH" env = fix_env(None) if is_binary() and ld_key not in env.keys(): # In fix_env, we delete LD_LIBRARY_PATH key if it was empty before # PyInstaller modified it. GitPython, in git.Repo.clone_from, uses # env to update its own internal state. When there is no key in # env, this value is not updated and GitPython re-uses # LD_LIBRARY_PATH that has been set by PyInstaller. # See [1] for more info. # [1] https://github.com/gitpython-developers/GitPython/issues/924 env[cast_bytes_py2(ld_key)] = "" try: tmp_repo = git.Repo.clone_from( url, to_path, env=env, # needed before we can fix it in __init__ no_single_branch=True, ) tmp_repo.close() except git.exc.GitCommandError as exc: raise CloneError(url, to_path, exc) # NOTE: using our wrapper to make sure that env is fixed in __init__ repo = Git(to_path) if rev: try: repo.checkout(rev) except git.exc.GitCommandError as exc: raise RevError(url, rev, exc) return repo
def _get_windows(self): return None if is_binary() else 'pip'
def get_windows(): return None if is_binary() else "pip"
def _popen(cmd, **kwargs): prefix = [sys.executable] if not is_binary(): prefix += [sys.argv[0]] return Popen(prefix + cmd, close_fds=True, shell=False, **kwargs)