def daemon(args): """Launch a `dvc daemon` command in a detached process. Args: args (list): list of arguments to append to `dvc daemon` command. """ cmd = [sys.executable] if not is_binary(): cmd += ["-m", "dvc"] cmd += ["daemon", "-q"] + args env = fix_env() file_path = os.path.abspath(inspect.stack()[0][1]) env[cast_bytes_py2("PYTHONPATH")] = cast_bytes_py2( os.path.dirname(os.path.dirname(file_path)) ) logger.debug("Trying to spawn '{}' with env '{}'".format(cmd, env)) if os.name == "nt": _spawn_windows(cmd, env) elif os.name == "posix": _spawn_posix(cmd, env) else: raise NotImplementedError logger.debug("Spawned '{}'".format(cmd))
def fix_env(env=None): if env is None: env = os.environ.copy() else: env = env.copy() if is_binary(): lp_key = "LD_LIBRARY_PATH" lp_orig = env.get(lp_key + "_ORIG", None) if lp_orig is not None: # NOTE: py2 doesn't like unicode strings in environ env[cast_bytes_py2(lp_key)] = cast_bytes_py2(lp_orig) else: env.pop(lp_key, None) return env
def fix_env(env=None): """Fix env variables modified by PyInstaller [1] and pyenv [2]. [1] http://pyinstaller.readthedocs.io/en/stable/runtime-information.html [2] https://github.com/pyenv/pyenv/issues/985 """ if env is None: env = os.environ.copy() else: env = env.copy() if is_binary(): lp_key = "LD_LIBRARY_PATH" lp_orig = env.get(lp_key + "_ORIG", None) if lp_orig is not None: # NOTE: py2 doesn't like unicode strings in environ env[cast_bytes_py2(lp_key)] = cast_bytes_py2(lp_orig) else: env.pop(lp_key, None) # Unlike PyInstaller, pyenv doesn't leave backups of original env vars # when it modifies them. If we look into the shim, pyenv and pyenv-exec, # we can figure out that the PATH is modified like this: # # PATH=$PYENV_BIN_PATH:${bin_path}:${plugin_bin}:$PATH # # where # # PYENV_BIN_PATH - starts with $PYENV_ROOT, see pyenv-exec source code. # bin_path - might not start with $PYENV_ROOT as it runs realpath on # it, see pyenv source code. # plugin_bin - might contain more than 1 entry, which start with # $PYENV_ROOT, see pyenv source code. # # So having this, we can make a rightful assumption about what parts of the # PATH we need to remove in order to get the original PATH. path = env.get("PATH") pyenv_root = env.get("PYENV_ROOT") if path and pyenv_root and path.startswith(pyenv_root): # removing PYENV_BIN_PATH and bin_path parts = path.split(":")[2:] # removing plugin_bin from the left while pyenv_root in parts[0]: del parts[0] env["PATH"] = ":".join(parts) return env
def daemon(args): """Launch a `dvc daemon` command in a detached process. Args: args (list): list of arguments to append to `dvc daemon` command. """ if os.environ.get(DVC_DAEMON): logger.debug("skipping launching a new daemon.") return cmd = [sys.executable] if not is_binary(): cmd += [sys.argv[0]] cmd += ["daemon", "-q"] + args env = fix_env() file_path = os.path.abspath(inspect.stack()[0][1]) env[cast_bytes_py2("PYTHONPATH")] = cast_bytes_py2( os.path.dirname(os.path.dirname(file_path))) env[cast_bytes_py2(DVC_DAEMON)] = cast_bytes_py2("1") _spawn(cmd, env)
def clone(url, to_path, rev=None): import git ld_key = "LD_LIBRARY_PATH" env = fix_env(None) if is_binary() and ld_key not in env.keys(): # In fix_env, we delete LD_LIBRARY_PATH key if it was empty before # PyInstaller modified it. GitPython, in git.Repo.clone_from, uses # env to update its own internal state. When there is no key in # env, this value is not updated and GitPython re-uses # LD_LIBRARY_PATH that has been set by PyInstaller. # See [1] for more info. # [1] https://github.com/gitpython-developers/GitPython/issues/924 env[cast_bytes_py2(ld_key)] = "" try: tmp_repo = git.Repo.clone_from( url, to_path, env=env, # needed before we can fix it in __init__ no_single_branch=True, ) tmp_repo.close() except git.exc.GitCommandError as exc: raise CloneError(url, to_path, exc) # NOTE: using our wrapper to make sure that env is fixed in __init__ repo = Git(to_path) if rev: try: repo.checkout(rev) except git.exc.GitCommandError as exc: raise RevError(url, rev, exc) return repo
from __future__ import unicode_literals import mockssh import pytest import os from git import Repo from git.exc import GitCommandNotFound from dvc.remote.config import RemoteConfig from dvc.remote.gdrive import RemoteGDrive from dvc.utils.compat import cast_bytes_py2 from dvc.remote.ssh.connection import SSHConnection from dvc.repo import Repo as DvcRepo from .basic_env import TestDirFixture, TestDvcGitFixture, TestGitFixture # Prevent updater and analytics from running their processes os.environ[cast_bytes_py2("DVC_TEST")] = cast_bytes_py2("true") # Make DVC tests use separate OAuth token to access Google Drive def skip_pydrive_init(_): pass RemoteGDrive.init_gdrive = skip_pydrive_init @pytest.fixture(autouse=True) def reset_loglevel(request, caplog): """ Use it to ensure log level at the start of each test regardless of dvc.logger.setup(), Repo configs or whatever.
def fix_env(env=None): """Fix env variables modified by PyInstaller [1] and pyenv [2]. [1] http://pyinstaller.readthedocs.io/en/stable/runtime-information.html [2] https://github.com/pyenv/pyenv/issues/985 """ if env is None: env = os.environ.copy() else: env = env.copy() if is_binary(): lp_key = "LD_LIBRARY_PATH" lp_orig = env.get(lp_key + "_ORIG", None) if lp_orig is not None: # NOTE: py2 doesn't like unicode strings in environ env[cast_bytes_py2(lp_key)] = cast_bytes_py2(lp_orig) else: env.pop(lp_key, None) # Unlike PyInstaller, pyenv doesn't leave backups of original env vars # when it modifies them. If we look into the shim, pyenv and pyenv-exec, # we can figure out that the PATH is modified like this: # # PATH=$PYENV_BIN_PATH:${bin_path}:${plugin_bin}:$PATH # # where # # PYENV_BIN_PATH - might not start with $PYENV_ROOT if we are running # `system` version of the command, see pyenv-exec source code. # bin_path - might not start with $PYENV_ROOT as it runs realpath on # it, but always has `libexec` part in it, see pyenv source code. # plugin_bin - might contain more than 1 entry, which start with # $PYENV_ROOT, see pyenv source code. # # Also, we know that whenever pyenv is running, it exports these env vars: # # PYENV_DIR # PYENV_HOOK_PATH # PYENV_VERSION # PYENV_ROOT # # So having this, we can make a rightful assumption about what parts of the # PATH we need to remove in order to get the original PATH. path = env.get("PATH", "") parts = path.split(":") bin_path = parts[1] if len(parts) > 2 else "" pyenv_dir = env.get("PYENV_DIR") pyenv_hook_path = env.get("PYENV_HOOK_PATH") pyenv_version = env.get("PYENV_VERSION") pyenv_root = env.get("PYENV_ROOT") env_matches = all([pyenv_dir, pyenv_hook_path, pyenv_version, pyenv_root]) bin_path_matches = os.path.basename(bin_path) == "libexec" # NOTE: we don't support pyenv-win if os.name != "nt" and env_matches and bin_path_matches: # removing PYENV_BIN_PATH and bin_path parts = parts[2:] if parts: # removing plugin_bin from the left plugin_bin = os.path.join(pyenv_root, "plugins") while parts[0].startswith(plugin_bin): del parts[0] env["PATH"] = ":".join(parts) return env
import os import mockssh import pytest from git import Repo from git.exc import GitCommandNotFound from dvc.remote.config import RemoteConfig from dvc.remote.ssh.connection import SSHConnection from dvc.repo import Repo as DvcRepo from dvc.utils.compat import cast_bytes_py2 from .basic_env import TestDirFixture, TestDvcGitFixture, TestGitFixture from .dir_helpers import * # noqa # Prevent updater and analytics from running their processes os.environ[cast_bytes_py2("DVC_TEST")] = cast_bytes_py2("true") # Ensure progress output even when not outputting to raw sys.stderr console os.environ[cast_bytes_py2("DVC_IGNORE_ISATTY")] = cast_bytes_py2("true") @pytest.fixture(autouse=True) def reset_loglevel(request, caplog): """ Use it to ensure log level at the start of each test regardless of dvc.logger.setup(), Repo configs or whatever. """ level = request.config.getoption("--log-level") if level: with caplog.at_level(level.upper(), logger="dvc"): yield else: