def daemon(args):
"""Launch a `dvc daemon` command in a detached process.
Args:
args (list): list of arguments to append to `dvc daemon` command.
"""
cmd = [sys.executable]
if not is_binary():
cmd += ["-m", "dvc"]
cmd += ["daemon", "-q"] + args
env = fix_env()
file_path = os.path.abspath(inspect.stack()[0][1])
env[cast_bytes_py2("PYTHONPATH")] = cast_bytes_py2(
os.path.dirname(os.path.dirname(file_path))
)
logger.debug("Trying to spawn '{}' with env '{}'".format(cmd, env))
if os.name == "nt":
_spawn_windows(cmd, env)
elif os.name == "posix":
_spawn_posix(cmd, env)
else:
raise NotImplementedError
logger.debug("Spawned '{}'".format(cmd))
def fix_env(env=None):
if env is None:
env = os.environ.copy()
else:
env = env.copy()
if is_binary():
lp_key = "LD_LIBRARY_PATH"
lp_orig = env.get(lp_key + "_ORIG", None)
if lp_orig is not None:
# NOTE: py2 doesn't like unicode strings in environ
env[cast_bytes_py2(lp_key)] = cast_bytes_py2(lp_orig)
else:
env.pop(lp_key, None)
return env
def fix_env(env=None):
"""Fix env variables modified by PyInstaller [1] and pyenv [2].
[1] http://pyinstaller.readthedocs.io/en/stable/runtime-information.html
[2] https://github.com/pyenv/pyenv/issues/985
"""
if env is None:
env = os.environ.copy()
else:
env = env.copy()
if is_binary():
lp_key = "LD_LIBRARY_PATH"
lp_orig = env.get(lp_key + "_ORIG", None)
if lp_orig is not None:
# NOTE: py2 doesn't like unicode strings in environ
env[cast_bytes_py2(lp_key)] = cast_bytes_py2(lp_orig)
else:
env.pop(lp_key, None)
# Unlike PyInstaller, pyenv doesn't leave backups of original env vars
# when it modifies them. If we look into the shim, pyenv and pyenv-exec,
# we can figure out that the PATH is modified like this:
#
# PATH=$PYENV_BIN_PATH:${bin_path}:${plugin_bin}:$PATH
#
# where
#
# PYENV_BIN_PATH - starts with $PYENV_ROOT, see pyenv-exec source code.
# bin_path - might not start with $PYENV_ROOT as it runs realpath on
# it, see pyenv source code.
# plugin_bin - might contain more than 1 entry, which start with
# $PYENV_ROOT, see pyenv source code.
#
# So having this, we can make a rightful assumption about what parts of the
# PATH we need to remove in order to get the original PATH.
path = env.get("PATH")
pyenv_root = env.get("PYENV_ROOT")
if path and pyenv_root and path.startswith(pyenv_root):
# removing PYENV_BIN_PATH and bin_path
parts = path.split(":")[2:]
# removing plugin_bin from the left
while pyenv_root in parts[0]:
del parts[0]
env["PATH"] = ":".join(parts)
return env
def daemon(args):
"""Launch a `dvc daemon` command in a detached process.
Args:
args (list): list of arguments to append to `dvc daemon` command.
"""
if os.environ.get(DVC_DAEMON):
logger.debug("skipping launching a new daemon.")
return
cmd = [sys.executable]
if not is_binary():
cmd += [sys.argv[0]]
cmd += ["daemon", "-q"] + args
env = fix_env()
file_path = os.path.abspath(inspect.stack()[0][1])
env[cast_bytes_py2("PYTHONPATH")] = cast_bytes_py2(
os.path.dirname(os.path.dirname(file_path)))
env[cast_bytes_py2(DVC_DAEMON)] = cast_bytes_py2("1")
_spawn(cmd, env)
def clone(url, to_path, rev=None):
import git
ld_key = "LD_LIBRARY_PATH"
env = fix_env(None)
if is_binary() and ld_key not in env.keys():
# In fix_env, we delete LD_LIBRARY_PATH key if it was empty before
# PyInstaller modified it. GitPython, in git.Repo.clone_from, uses
# env to update its own internal state. When there is no key in
# env, this value is not updated and GitPython re-uses
# LD_LIBRARY_PATH that has been set by PyInstaller.
# See [1] for more info.
# [1] https://github.com/gitpython-developers/GitPython/issues/924
env[cast_bytes_py2(ld_key)] = ""
try:
tmp_repo = git.Repo.clone_from(
url,
to_path,
env=env, # needed before we can fix it in __init__
no_single_branch=True,
)
tmp_repo.close()
except git.exc.GitCommandError as exc:
raise CloneError(url, to_path, exc)
# NOTE: using our wrapper to make sure that env is fixed in __init__
repo = Git(to_path)
if rev:
try:
repo.checkout(rev)
except git.exc.GitCommandError as exc:
raise RevError(url, rev, exc)
return repo
from __future__ import unicode_literals
import mockssh
import pytest
import os
from git import Repo
from git.exc import GitCommandNotFound
from dvc.remote.config import RemoteConfig
from dvc.remote.gdrive import RemoteGDrive
from dvc.utils.compat import cast_bytes_py2
from dvc.remote.ssh.connection import SSHConnection
from dvc.repo import Repo as DvcRepo
from .basic_env import TestDirFixture, TestDvcGitFixture, TestGitFixture
# Prevent updater and analytics from running their processes
os.environ[cast_bytes_py2("DVC_TEST")] = cast_bytes_py2("true")
# Make DVC tests use separate OAuth token to access Google Drive
def skip_pydrive_init(_):
pass
RemoteGDrive.init_gdrive = skip_pydrive_init
@pytest.fixture(autouse=True)
def reset_loglevel(request, caplog):
"""
Use it to ensure log level at the start of each test
regardless of dvc.logger.setup(), Repo configs or whatever.
def fix_env(env=None):
"""Fix env variables modified by PyInstaller [1] and pyenv [2].
[1] http://pyinstaller.readthedocs.io/en/stable/runtime-information.html
[2] https://github.com/pyenv/pyenv/issues/985
"""
if env is None:
env = os.environ.copy()
else:
env = env.copy()
if is_binary():
lp_key = "LD_LIBRARY_PATH"
lp_orig = env.get(lp_key + "_ORIG", None)
if lp_orig is not None:
# NOTE: py2 doesn't like unicode strings in environ
env[cast_bytes_py2(lp_key)] = cast_bytes_py2(lp_orig)
else:
env.pop(lp_key, None)
# Unlike PyInstaller, pyenv doesn't leave backups of original env vars
# when it modifies them. If we look into the shim, pyenv and pyenv-exec,
# we can figure out that the PATH is modified like this:
#
# PATH=$PYENV_BIN_PATH:${bin_path}:${plugin_bin}:$PATH
#
# where
#
# PYENV_BIN_PATH - might not start with $PYENV_ROOT if we are running
# `system` version of the command, see pyenv-exec source code.
# bin_path - might not start with $PYENV_ROOT as it runs realpath on
# it, but always has `libexec` part in it, see pyenv source code.
# plugin_bin - might contain more than 1 entry, which start with
# $PYENV_ROOT, see pyenv source code.
#
# Also, we know that whenever pyenv is running, it exports these env vars:
#
# PYENV_DIR
# PYENV_HOOK_PATH
# PYENV_VERSION
# PYENV_ROOT
#
# So having this, we can make a rightful assumption about what parts of the
# PATH we need to remove in order to get the original PATH.
path = env.get("PATH", "")
parts = path.split(":")
bin_path = parts[1] if len(parts) > 2 else ""
pyenv_dir = env.get("PYENV_DIR")
pyenv_hook_path = env.get("PYENV_HOOK_PATH")
pyenv_version = env.get("PYENV_VERSION")
pyenv_root = env.get("PYENV_ROOT")
env_matches = all([pyenv_dir, pyenv_hook_path, pyenv_version, pyenv_root])
bin_path_matches = os.path.basename(bin_path) == "libexec"
# NOTE: we don't support pyenv-win
if os.name != "nt" and env_matches and bin_path_matches:
# removing PYENV_BIN_PATH and bin_path
parts = parts[2:]
if parts:
# removing plugin_bin from the left
plugin_bin = os.path.join(pyenv_root, "plugins")
while parts[0].startswith(plugin_bin):
del parts[0]
env["PATH"] = ":".join(parts)
return env
import os
import mockssh
import pytest
from git import Repo
from git.exc import GitCommandNotFound
from dvc.remote.config import RemoteConfig
from dvc.remote.ssh.connection import SSHConnection
from dvc.repo import Repo as DvcRepo
from dvc.utils.compat import cast_bytes_py2
from .basic_env import TestDirFixture, TestDvcGitFixture, TestGitFixture
from .dir_helpers import * # noqa
# Prevent updater and analytics from running their processes
os.environ[cast_bytes_py2("DVC_TEST")] = cast_bytes_py2("true")
# Ensure progress output even when not outputting to raw sys.stderr console
os.environ[cast_bytes_py2("DVC_IGNORE_ISATTY")] = cast_bytes_py2("true")
@pytest.fixture(autouse=True)
def reset_loglevel(request, caplog):
"""
Use it to ensure log level at the start of each test
regardless of dvc.logger.setup(), Repo configs or whatever.
"""
level = request.config.getoption("--log-level")
if level:
with caplog.at_level(level.upper(), logger="dvc"):
yield
else:
