def daemon(args):
"""Launch a `dvc daemon` command in a detached process.
Args:
args (list): list of arguments to append to `dvc daemon` command.
"""
cmd = [sys.executable]
if not is_binary():
cmd += ["-m", "dvc"]
cmd += ["daemon", "-q"] + args
env = fix_env()
file_path = os.path.abspath(inspect.stack()[0][1])
env[cast_bytes_py2("PYTHONPATH")] = cast_bytes_py2(
os.path.dirname(os.path.dirname(file_path))
)
logger.debug("Trying to spawn '{}' with env '{}'".format(cmd, env))
if os.name == "nt":
_spawn_windows(cmd, env)
elif os.name == "posix":
_spawn_posix(cmd, env)
else:
raise NotImplementedError
logger.debug("Spawned '{}'".format(cmd))
def _popen(cmd, **kwargs):
prefix = [sys.executable]
if not is_binary():
main_entrypoint = os.path.join(
os.path.abspath(os.path.dirname(__file__)), "__main__.py")
prefix += [main_entrypoint]
return Popen(prefix + cmd, close_fds=True, shell=False, **kwargs)
def _get_darwin(self):
if not is_binary():
if __file__.startswith("/usr/local/Cellar"):
return "formula"
else:
return "pip"
return None
def run(self):
from dvc.repo import Repo
dvc_version = __version__
python_version = platform.python_version()
platform_type = platform.platform()
binary = is_binary()
package_manager = get_package_manager()
info = (
"DVC version: {dvc_version}\n"
"Python version: {python_version}\n"
"Platform: {platform_type}\n"
"Binary: {binary}\n"
"Package manager: {package_manager}\n"
).format(
dvc_version=dvc_version,
python_version=python_version,
platform_type=platform_type,
binary=binary,
package_manager=package_manager,
)
try:
repo = Repo()
root_directory = repo.root_dir
# cache_dir might not exist yet (e.g. after `dvc init`), and we
# can't auto-create it, as it might cause issues if the user
# later decides to enable shared cache mode with
# `dvc config cache.shared group`.
if os.path.exists(repo.cache.local.cache_dir):
info += "Cache: {cache}\n".format(
cache=self.get_linktype_support_info(repo)
)
else:
logger.warning(
"Unable to detect supported link types, as cache "
"directory '{}' doesn't exist. It is usually auto-created "
"by commands such as `dvc add/fetch/pull/run/import`, "
"but you could create it manually to enable this "
"check.".format(relpath(repo.cache.local.cache_dir))
)
if psutil:
info += (
"Filesystem type (cache directory): {fs_cache}\n"
).format(fs_cache=self.get_fs_type(repo.cache.local.cache_dir))
except NotDvcRepoError:
root_directory = os.getcwd()
if psutil:
info += ("Filesystem type (workspace): {fs_root}").format(
fs_root=self.get_fs_type(os.path.abspath(root_directory))
)
logger.info(info)
return 0
def clone(
url: str,
to_path: str,
rev: Optional[str] = None,
shallow_branch: Optional[str] = None,
):
import git
ld_key = "LD_LIBRARY_PATH"
env = fix_env(None)
if is_binary() and ld_key not in env.keys():
# In fix_env, we delete LD_LIBRARY_PATH key if it was empty before
# PyInstaller modified it. GitPython, in git.Repo.clone_from, uses
# env to update its own internal state. When there is no key in
# env, this value is not updated and GitPython re-uses
# LD_LIBRARY_PATH that has been set by PyInstaller.
# See [1] for more info.
# [1] https://github.com/gitpython-developers/GitPython/issues/924
env[ld_key] = ""
try:
if shallow_branch is not None and os.path.exists(url):
# git disables --depth for local clones unless file:// url
# scheme is used
url = f"file://{url}"
with TqdmGit(desc="Cloning", unit="obj") as pbar:
clone_from = partial(
git.Repo.clone_from,
url,
to_path,
env=env, # needed before we can fix it in __init__
no_single_branch=True,
progress=pbar.update_git,
)
if shallow_branch is None:
tmp_repo = clone_from()
else:
tmp_repo = clone_from(branch=shallow_branch, depth=1)
tmp_repo.close()
except git.exc.GitCommandError as exc: # pylint: disable=no-member
raise CloneError(url, to_path) from exc
# NOTE: using our wrapper to make sure that env is fixed in __init__
repo = GitPythonBackend(to_path)
if rev:
try:
repo.checkout(rev)
except git.exc.GitCommandError as exc: # pylint: disable=no-member
raise RevError(
"failed to access revision '{}' for repo '{}'".format(
rev, url
)
) from exc
def _get_darwin(self):
if not is_binary():
if __file__.startswith("/usr/local/Cellar"):
return "formula"
else:
return "pip"
# NOTE: both pkg and cask put dvc binary into /usr/local/bin,
# so in order to know which method of installation was used,
# we need to actually call `brew cask`
return None
def _runtime_info():
"""
Gather information from the environment where DVC runs to fill a report.
"""
return {
"dvc_version": __version__,
"is_binary": is_binary(),
"scm_class": _scm_in_use(),
"system_info": _system_info(),
"user_id": _find_or_create_user_id(),
}
def run(self):
from dvc.repo import Repo
info = [
"DVC version: {}".format(__version__),
"Python version: {}".format(platform.python_version()),
"Platform: {}".format(platform.platform()),
"Binary: {}".format(is_binary()),
"Package: {}".format(PKG),
"Supported remotes: {}".format(self.get_supported_remotes()),
]
try:
repo = Repo()
root_directory = repo.root_dir
# cache_dir might not exist yet (e.g. after `dvc init`), and we
# can't auto-create it, as it might cause issues if the user
# later decides to enable shared cache mode with
# `dvc config cache.shared group`.
if os.path.exists(repo.cache.local.cache_dir):
info.append(
"Cache: {}".format(self.get_linktype_support_info(repo))
)
if psutil:
fs_type = self.get_fs_type(repo.cache.local.cache_dir)
info.append(
"Filesystem type (cache directory): {}".format(fs_type)
)
else:
logger.warning(
"Unable to detect supported link types, as cache "
"directory '{}' doesn't exist. It is usually auto-created "
"by commands such as `dvc add/fetch/pull/run/import`, "
"but you could create it manually to enable this "
"check.".format(relpath(repo.cache.local.cache_dir))
)
except NotDvcRepoError:
root_directory = os.getcwd()
except SCMError:
root_directory = os.getcwd()
info.append("Repo: dvc, git (broken)")
else:
info.append("Repo: {}".format(_get_dvc_repo_info(repo)))
if psutil:
fs_root = self.get_fs_type(os.path.abspath(root_directory))
info.append("Filesystem type (workspace): {}".format(fs_root))
logger.info("\n".join(info))
return 0
def _get_darwin(self):
if not is_binary():
if __file__.startswith('/usr/local/Cellar'):
return 'formula'
else:
return 'pip'
# NOTE: both pkg and cask put dvc binary into /usr/local/bin,
# so in order to know which method of installation was used,
# we need to actually call `brew cask`
ret = os.system('brew cask ls dvc')
if ret == 0:
return 'cask'
return None
def __call__(self, args):
cmd = [sys.executable]
if not is_binary():
cmd += ['-m', 'dvc']
cmd += ['daemon', '-q'] + args
logger.debug("Trying to spawn '{}'".format(cmd))
if os.name == 'nt':
self._spawn_windows(cmd)
elif os.name == 'posix':
self._spawn_posix(cmd)
else:
raise NotImplementedError
logger.debug("Spawned '{}'".format(cmd))
def _get_linux(self):
import distro
if not is_binary():
return 'pip'
package_managers = {
'rhel': 'yum',
'centos': 'yum',
'fedora': 'yum',
'amazon': 'yum',
'opensuse': 'yum',
'ubuntu': 'apt',
'debian': 'apt',
}
return package_managers.get(distro.id())
def _get_linux(self):
import distro
if not is_binary():
return "pip"
package_managers = {
"rhel": "yum",
"centos": "yum",
"fedora": "yum",
"amazon": "yum",
"opensuse": "yum",
"ubuntu": "apt",
"debian": "apt",
}
return package_managers.get(distro.id())
def collect(self):
from dvc.scm import SCM
from dvc.utils import is_binary
from dvc.project import Project
from dvc.exceptions import NotDvcProjectError
self.info[self.PARAM_DVC_VERSION] = VERSION
self.info[self.PARAM_IS_BINARY] = is_binary()
self.info[self.PARAM_USER_ID] = self._get_user_id()
self.info[self.PARAM_SYSTEM_INFO] = self._collect_system_info()
try:
scm = SCM(root_dir=Project._find_root())
self.info[self.PARAM_SCM_CLASS] = type(scm).__name__
except NotDvcProjectError:
pass
def collect(self):
"""Collect analytics report."""
from dvc.scm import SCM
from dvc.utils import is_binary
from dvc.repo import Repo
from dvc.exceptions import NotDvcRepoError
self.info[self.PARAM_DVC_VERSION] = __version__
self.info[self.PARAM_IS_BINARY] = is_binary()
self.info[self.PARAM_USER_ID] = self._get_user_id()
self.info[self.PARAM_SYSTEM_INFO] = self._collect_system_info()
try:
scm = SCM(root_dir=Repo.find_root())
self.info[self.PARAM_SCM_CLASS] = type(scm).__name__
except NotDvcRepoError:
pass
def daemon(args):
"""Launch a `dvc daemon` command in a detached process.
Args:
args (list): list of arguments to append to `dvc daemon` command.
"""
if os.environ.get(DVC_DAEMON):
logger.debug("skipping launching a new daemon.")
return
cmd = ["daemon", "-q"] + args
env = fix_env()
if not is_binary():
file_path = os.path.abspath(inspect.stack()[0][1])
env["PYTHONPATH"] = os.path.dirname(os.path.dirname(file_path))
env[DVC_DAEMON] = "1"
_spawn(cmd, env)
def clone(url, to_path, rev=None):
import git
ld_key = "LD_LIBRARY_PATH"
env = fix_env(None)
if is_binary() and ld_key not in env.keys():
# In fix_env, we delete LD_LIBRARY_PATH key if it was empty before
# PyInstaller modified it. GitPython, in git.Repo.clone_from, uses
# env to update its own internal state. When there is no key in
# env, this value is not updated and GitPython re-uses
# LD_LIBRARY_PATH that has been set by PyInstaller.
# See [1] for more info.
# [1] https://github.com/gitpython-developers/GitPython/issues/924
env[ld_key] = ""
try:
with TqdmGit(desc="Cloning", unit="obj") as pbar:
tmp_repo = git.Repo.clone_from(
url,
to_path,
env=env, # needed before we can fix it in __init__
no_single_branch=True,
progress=pbar.update_git,
)
tmp_repo.close()
except git.exc.GitCommandError as exc: # pylint: disable=no-member
raise CloneError(url, to_path) from exc
# NOTE: using our wrapper to make sure that env is fixed in __init__
repo = Git(to_path)
if rev:
try:
repo.checkout(rev)
except git.exc.GitCommandError as exc: # pylint: disable=no-member
raise RevError(
"failed to access revision '{}' for repo '{}'".format(
rev, url
)
) from exc
return repo
def run(self):
from dvc.repo import Repo
dvc_version = __version__
python_version = platform.python_version()
platform_type = platform.platform()
binary = is_binary()
info = (
"DVC version: {dvc_version}\n"
"Python version: {python_version}\n"
"Platform: {platform_type}\n"
"Binary: {binary}\n"
).format(
dvc_version=dvc_version,
python_version=python_version,
platform_type=platform_type,
binary=binary,
)
try:
repo = Repo()
root_directory = repo.root_dir
info += "Cache: {cache}\n".format(
cache=self.get_linktype_support_info(repo)
)
if psutil:
info += (
"Filesystem type (cache directory): {fs_cache}\n"
).format(fs_cache=self.get_fs_type(repo.cache.local.cache_dir))
except NotDvcRepoError:
root_directory = os.getcwd()
if psutil:
info += ("Filesystem type (workspace): {fs_root}").format(
fs_root=self.get_fs_type(os.path.abspath(root_directory))
)
logger.info(info)
return 0
def _spawn_windows(cmd, env):
from subprocess import STARTUPINFO, STARTF_USESHOWWINDOW
prefix = [sys.executable]
if not is_binary():
prefix += [sys.argv[0]]
creationflags = CREATE_NEW_PROCESS_GROUP | DETACHED_PROCESS
startupinfo = STARTUPINFO()
startupinfo.dwFlags |= STARTF_USESHOWWINDOW
Popen(
prefix + cmd,
env=env,
close_fds=True,
shell=False,
creationflags=creationflags,
startupinfo=startupinfo,
).communicate()
def daemon(args):
"""Launch a `dvc daemon` command in a detached process.
Args:
args (list): list of arguments to append to `dvc daemon` command.
"""
cmd = [sys.executable]
if not is_binary():
cmd += ['-m', 'dvc']
cmd += ['daemon', '-q'] + args
logger.debug("Trying to spawn '{}'".format(cmd))
if os.name == 'nt':
_spawn_windows(cmd)
elif os.name == 'posix':
_spawn_posix(cmd)
else:
raise NotImplementedError
logger.debug("Spawned '{}'".format(cmd))
def daemon(args):
"""Launch a `dvc daemon` command in a detached process.
Args:
args (list): list of arguments to append to `dvc daemon` command.
"""
if os.environ.get(DVC_DAEMON):
logger.debug("skipping launching a new daemon.")
return
cmd = [sys.executable]
if not is_binary():
cmd += [sys.argv[0]]
cmd += ["daemon", "-q"] + args
env = fix_env()
file_path = os.path.abspath(inspect.stack()[0][1])
env[cast_bytes_py2("PYTHONPATH")] = cast_bytes_py2(
os.path.dirname(os.path.dirname(file_path)))
env[cast_bytes_py2(DVC_DAEMON)] = cast_bytes_py2("1")
_spawn(cmd, env)
def clone(url, to_path, rev=None):
import git
ld_key = "LD_LIBRARY_PATH"
env = fix_env(None)
if is_binary() and ld_key not in env.keys():
# In fix_env, we delete LD_LIBRARY_PATH key if it was empty before
# PyInstaller modified it. GitPython, in git.Repo.clone_from, uses
# env to update its own internal state. When there is no key in
# env, this value is not updated and GitPython re-uses
# LD_LIBRARY_PATH that has been set by PyInstaller.
# See [1] for more info.
# [1] https://github.com/gitpython-developers/GitPython/issues/924
env[cast_bytes_py2(ld_key)] = ""
try:
tmp_repo = git.Repo.clone_from(
url,
to_path,
env=env, # needed before we can fix it in __init__
no_single_branch=True,
)
tmp_repo.close()
except git.exc.GitCommandError as exc:
raise CloneError(url, to_path, exc)
# NOTE: using our wrapper to make sure that env is fixed in __init__
repo = Git(to_path)
if rev:
try:
repo.checkout(rev)
except git.exc.GitCommandError as exc:
raise RevError(url, rev, exc)
return repo
def _get_windows(self):
return None if is_binary() else 'pip'
def get_windows():
return None if is_binary() else "pip"
def _popen(cmd, **kwargs):
prefix = [sys.executable]
if not is_binary():
prefix += [sys.argv[0]]
return Popen(prefix + cmd, close_fds=True, shell=False, **kwargs)