def test_conda_env_yaml_r_install(r_setup): """Test the environment.yml file in detail.""" name = r_setup["name"] env_dir = r_setup["env_dir"] channels = r_setup["channels"] packages = get_dependencies(name=name) conda_packages = packages["conda"] expected_start = [f"name: {name}", "channels:"] for channel in channels + ["nodefaults"]: expected_start.append(f" - {channel}") expected_start.append("dependencies:") expected_conda_packages = [ " - r-base=" + conda_packages["r-base"].version, " - r-devtools=" + conda_packages["r-devtools"].version, ] expected = "\n".join(expected_start + expected_conda_packages) + "\n" actual = (env_dir / "environment.yml").read_text() print(actual) assert actual == expected install_r = "\n".join( [ 'library("devtools"); install_version("jsonlite", version="1.2")', 'install.packages("praise")', ] ) actual_install_r = (env_dir / "install.R").read_text() print(actual_install_r) assert actual_install_r == install_r
def history_diff(env_name: str, env, env_reader) -> ListLike: """return the difference between history and local environment""" version_diff_pkges: ListLike = [] new_pkges: ListLike = [] missing_pkges: ListLike = [] history_conda_pkges = env_reader.get_environment()["dependencies"] history_conda_pkges_dict = {} for spec in history_conda_pkges: name, package = Package.from_spec(spec) history_conda_pkges_dict[name] = package local_conda_pkges = get_dependencies(name=env_name)["conda"] for name, package in local_conda_pkges.items(): if name in history_conda_pkges_dict: if package.version != history_conda_pkges_dict[name].version: version_diff_pkges.append("-" + name + "=" + history_conda_pkges_dict[name]) version_diff_pkges.append("+" + name + "=" + package.version) else: new_pkges.append("+" + name + "=" + package.version) for package in env.history.packages["conda"]: if package not in local_conda_pkges.keys(): missing_pkges.append("-" + package) return version_diff_pkges, new_pkges, missing_pkges
def get_pip_version(name: str) -> Optional[str]: """Check for the version of pip (if installed).""" if is_current_conda_env(name): import pip return pip.__version__ dependencies = get_dependencies(name=name) return dependencies["conda"].get("pip", Package(name="pip")).version
def infer(cls, name: str, packages: Packages, channels: ListLike = None): """create conda_env_tracker environment by inferring to existing conda environment""" if name == "base": raise CondaEnvTrackerCondaError( "Environment can not be created using default name base" ) if name not in get_all_existing_environment(): raise CondaEnvTrackerCondaError( f"Environment {name} can not be inferred, does not exist" ) existing_packages = get_dependencies(name=name) if "r-base" in existing_packages["conda"]: existing_r_packages = get_r_dependencies(name=name) else: existing_r_packages = {} user_packages = {"conda": Packages(), "pip": Packages(), "r": Packages()} for package in packages: if package.name in existing_packages.get("conda", Packages()): user_packages["conda"].append(package) elif package.name in existing_packages.get("pip", Packages()): user_packages["pip"].append(package) elif package.name in existing_r_packages: raise RError( "Cannot infer R packages, must run follow-up R install command.\n" f"Found '{package.name}' in installed R packages {existing_r_packages}." ) else: raise CondaEnvTrackerCondaError( f"Environment {name} does not have {package.spec} installed" ) conda_create_cmd = get_conda_create_command( name, user_packages["conda"], channels ) specs = Actions.get_package_specs( packages=user_packages["conda"], dependencies=existing_packages["conda"] ) history = History( name=name, channels=Channels(channels), packages=HistoryPackages.create(user_packages["conda"]), logs=Logs.create(conda_create_cmd), actions=Actions.create(name=name, specs=specs, channels=Channels(channels)), debug=Debug.create(name=name), ) env = cls(name=name, history=history) if user_packages["pip"]: handler = PipHandler(env=env) handler.update_history_install(packages=user_packages["pip"]) env = handler.env env.export() return env
def __init__(self, name: str, history: Optional[History] = None): self.name = name self.history = history self.local_io = EnvIO(env_directory=USER_ENVS_DIR / name) self.dependencies = {} if self.history: self.dependencies = get_dependencies(name=self.name) if self.history.packages.get("r"): self.dependencies["r"] = get_r_dependencies(name=self.name)
def create( cls, name: str, packages: Packages, channels: ListLike = None, yes: bool = False, strict_channel_priority: bool = True, ): """Creating a conda environment from a list of packages.""" if name == "base": raise CondaEnvTrackerCondaError( "Environment can not be created using default name base" ) if name in get_all_existing_environment(): raise CondaEnvTrackerCondaError(f"Environment {name} already exist") logger.debug(f"creating conda env {name}") conda_create( name=name, packages=packages, channels=channels, yes=yes, strict_channel_priority=strict_channel_priority, ) create_cmd = get_conda_create_command( name=name, packages=packages, channels=channels, strict_channel_priority=strict_channel_priority, ) specs = Actions.get_package_specs( packages=packages, dependencies=get_dependencies(name=name)["conda"] ) if not channels: channels = get_conda_channels() history = History( name=name, channels=Channels(channels), packages=HistoryPackages.create(packages), logs=Logs.create(create_cmd), actions=Actions.create( name=name, specs=specs, channels=Channels(channels), strict_channel_priority=strict_channel_priority, ), debug=Debug.create(name=name), ) env = cls(name=name, history=history) env.export() return env
def test_get_packages_empty(mocker, pkg_list): run_mock = mocker.patch("conda_env_tracker.gateways.conda.subprocess.run") attrs = { "return_value.stdout": pkg_list, "return_value.stderr": None, "return_value.returncode": 0, } run_mock.configure_mock(**attrs) package_dict = get_dependencies("test-create") assert not package_dict["conda"] assert "pip" not in package_dict
def test_r_remove_package(r_setup): name = r_setup["name"] env_dir = r_setup["env_dir"] channels = r_setup["channels"] history_file = env_dir / "history.yaml" r_remove(name=name, specs=["praise"], yes=True) actual_history_content = history_file.read_text() print(actual_history_content) expected_packages = { "conda": { "r-base": "*", "r-devtools": "*" }, "r": { "jsonlite": "1.2" }, } actual = yaml.load(actual_history_content, Loader=yaml.FullLoader) remove_command = 'remove.packages(c("praise"))' expected_log = f"R --quiet --vanilla -e '{remove_command}'" assert actual["packages"] == expected_packages assert actual["logs"][-1] == expected_log assert actual["actions"][-1] == expected_log packages = get_dependencies(name=name) conda_packages = packages["conda"] expected_start = [f"name: {name}", "channels:"] for channel in channels + ["nodefaults"]: expected_start.append(f" - {channel}") expected_start.append("dependencies:") expected_conda_packages = [ " - r-base=" + conda_packages["r-base"].version, " - r-devtools=" + conda_packages["r-devtools"].version, ] expected = "\n".join(expected_start + expected_conda_packages) + "\n" actual = (env_dir / "conda-env.yaml").read_text() print(actual) assert actual == expected install_r = "\n".join( ['library("devtools"); install_version("jsonlite", version="1.2")']) actual_install_r = (env_dir / "install.R").read_text() print(actual_install_r) assert actual_install_r == install_r
def test_pip_remove(pip_setup): name = pip_setup["name"] env_dir = pip_setup["env_dir"] channels = pip_setup["channels"] history_file = env_dir / "history.yaml" pip_remove(name=name, specs=["pytest-cov"], yes=True) actual_history_content = history_file.read_text() print(actual_history_content) expected_packages = { "conda": {"colorama": "*", "python": "3.6"}, "pip": {"pytest": "4.0.0"}, } actual = yaml.load(actual_history_content, Loader=yaml.FullLoader) expected_log = "pip uninstall pytest-cov" assert actual["packages"] == expected_packages assert actual["logs"][-1] == expected_log assert actual["actions"][-1] == expected_log packages = get_dependencies(name=name) conda_packages = packages["conda"] pip_packages = packages["pip"] expected_start = [f"name: {name}", "channels:"] for channel in channels + ["nodefaults"]: expected_start.append(f" - {channel}") expected_start.append("dependencies:") expected_conda_packages = [ " - python=" + conda_packages["python"].version, " - colorama=" + conda_packages["colorama"].version, ] expected_pip_packages = [" - pytest==" + pip_packages["pytest"].version] expected = ( "\n".join( expected_start + expected_conda_packages + [" - pip:"] + expected_pip_packages ) + "\n" ) actual = (env_dir / "conda-env.yaml").read_text() print(actual) assert actual == expected
def __init__( self, name: str, history: Optional[History] = None, dependencies: Optional[dict] = None, ): self.name = name self.history = history self.local_io = EnvIO(env_directory=USER_ENVS_DIR / name) if dependencies: self.dependencies = dependencies else: self.dependencies = {} if self.history: self.dependencies = get_dependencies(name=self.name) if self.history.packages.get("r"): self.dependencies["r"] = get_r_dependencies(name=self.name) if history: self.history.packages.update_versions(dependencies=self.dependencies)
def test_get_packages(mocker, packages): run_mock = mocker.patch("conda_env_tracker.gateways.conda.subprocess.run") attrs = { "return_value.stdout": packages, "return_value.stderr": None, "return_value.returncode": 0, } run_mock.configure_mock(**attrs) actual = get_dependencies("test-create") expected = { "conda": { "pylint": Package("pylint", "pylint", "2.0.4", "py37_0") } } if packages.endswith("pypi") or packages.endswith("pip"): expected["pip"] = {"pytest": Package("pytest", "pytest", "1.0.7")} else: expected["conda"]["pytest"] = Package("pytest", "pytest", "1.0.7", "py37_0") assert actual == expected
def test_conda_env_yaml_pip_install(pip_setup): """Test the environment.yml file in detail.""" name = pip_setup["name"] env_dir = pip_setup["env_dir"] channels = pip_setup["channels"] packages = get_dependencies(name=name) conda_packages = packages["conda"] pip_packages = packages["pip"] expected_start = [f"name: {name}", "channels:"] for channel in channels + ["nodefaults"]: expected_start.append(f" - {channel}") expected_start.append("dependencies:") expected_conda_packages = [ " - python=" + conda_packages["python"].version, " - colorama=" + conda_packages["colorama"].version, ] expected_pip_packages = [ " - pytest==" + pip_packages["pytest"].version, " - pytest-cov==" + pip_packages["pytest-cov"].version, ] expected = ( "\n".join( expected_start + expected_conda_packages + [" - pip:"] + expected_pip_packages ) + "\n" ) actual = (env_dir / "environment.yml").read_text() print(actual) assert actual == expected
def test_conda_env_yaml_after_create(end_to_end_setup): """Test the conda-env.yaml file in detail.""" name = end_to_end_setup["name"] env_dir = end_to_end_setup["env_dir"] channels = end_to_end_setup["channels"] conda_packages = get_dependencies(name=name)["conda"] expected_start = [f"name: {name}", "channels:"] for channel in channels + ["nodefaults"]: expected_start.append(f" - {channel}") expected_start.append("dependencies:") expected_packages = [ " - python=" + conda_packages["python"].version, " - colorama=" + conda_packages["colorama"].version, ] expected = "\n".join(expected_start + expected_packages) + "\n" actual = (env_dir / "conda-env.yaml").read_text() print(actual) assert actual == expected
def test_r_remove_package(r_setup): # pylint: disable=too-many-locals name = r_setup["name"] env_dir = r_setup["env_dir"] channels = r_setup["channels"] r_remove(name=name, specs=["praise"], yes=True) history_file = env_dir / "history.yaml" actual_history_content = history_file.read_text() print(actual_history_content) expected_packages = { "conda": {"r-base": "*", "r-devtools": "*"}, "r": { "jsonlite": 'library("devtools"); install_version("jsonlite", version="1.2")' }, } actual = yaml.load(actual_history_content, Loader=yaml.FullLoader) remove_command = r"remove.packages(c(\"praise\"))" expected_log = f'R --quiet --vanilla -e "{remove_command}"' assert actual["packages"] == expected_packages assert actual["revisions"][-1]["log"] == expected_log assert actual["revisions"][-1]["action"] == expected_log packages = get_dependencies(name=name) conda_packages = packages["conda"] expected_start = [f"name: {name}", "channels:"] for channel in channels + ["nodefaults"]: expected_start.append(f" - {channel}") expected_start.append("dependencies:") expected_conda_packages = [ " - r-base=" + conda_packages["r-base"].version, " - r-devtools=" + conda_packages["r-devtools"].version, ] expected = "\n".join(expected_start + expected_conda_packages) + "\n" actual = (env_dir / "environment.yml").read_text() print(actual) assert actual == expected install_r = "\n".join( ['library("devtools"); install_version("jsonlite", version="1.2")'] ) actual_install_r = (env_dir / "install.R").read_text() print(actual_install_r) assert actual_install_r == install_r expected_packages_section = "\n".join( [ "packages:", " conda:", " r-base: '*'", " r-devtools: '*'", " r:", ' jsonlite: library("devtools"); install_version("jsonlite",version="1.2")', "revisions:", ] ) assert expected_packages_section in actual_history_content expected_third_revision = "\n".join( [ " - packages:", " conda:", " r-base: '*'", " r-devtools: '*'", " r:", ' jsonlite: library("devtools"); install_version("jsonlite",version="1.2")', " diff:", " r:", " remove:", " - praise", rf' log: R --quiet --vanilla -e "remove.packages(c(\"praise\"))"', rf' action: R --quiet --vanilla -e "remove.packages(c(\"praise\"))"', ] ) index_first_revision = actual_history_content.find(" - packages:") index_second_revision = actual_history_content.find( " - packages:", index_first_revision + 1 ) index_third_revision = actual_history_content.find( " - packages:", index_second_revision + 1 ) third_action = f" action: {expected_log}" index_third_action = actual_history_content.find( third_action, index_third_revision ) + len(third_action) actual_third_revision = actual_history_content[ index_third_revision:index_third_action ] assert actual_third_revision == expected_third_revision
def test_remote_end_to_end(end_to_end_setup, mocker): """Test setup, create, install, pull and push feature of conda_env_tracker""" env = end_to_end_setup["env"] remote_dir = end_to_end_setup["remote_dir"] channels = end_to_end_setup["channels"] channel_command = end_to_end_setup["channel_command"] setup_remote(name=env.name, remote_dir=remote_dir) local_io = EnvIO(env_directory=USER_ENVS_DIR / env.name) assert str(local_io.get_remote_dir()) == str(remote_dir) assert not list(remote_dir.glob("*")) push(name=env.name) remote_io = EnvIO(env_directory=remote_dir) assert remote_io.get_history() == local_io.get_history() assert remote_io.get_environment() == local_io.get_environment() env = conda_install(name=env.name, specs=["pytest"], yes=True) assert env.local_io.get_history() != remote_io.get_history() env = push(name=env.name) assert env.local_io.get_history() == remote_io.get_history() log_mock = mocker.patch("conda_env_tracker.pull.logging.Logger.info") env = pull(name=env.name) log_mock.assert_called_once_with("Nothing to pull.") remove_package_from_history(env, "pytest") conda_dependencies = env.dependencies["conda"] assert env.local_io.get_history() != remote_io.get_history() assert env.history.packages == { "conda": { "python": Package( "python", "python=3.6", version=conda_dependencies["python"].version, build=conda_dependencies["python"].build, ), "colorama": Package( "colorama", "colorama", version=conda_dependencies["colorama"].version, build=conda_dependencies["colorama"].build, ), } } env = pull(name=env.name) assert env.local_io.get_history() == remote_io.get_history() assert remote_io.get_environment() == local_io.get_environment() remove_package_from_history(env, "pytest") assert env.local_io.get_history() != remote_io.get_history() env = conda_install(name=env.name, specs=["pytest-cov"], channels=("main", ), yes=True) env = pull(name=env.name, yes=True) conda_dependencies = env.dependencies["conda"] assert env.history.packages == { "conda": { "python": Package( "python", "python=3.6", version=conda_dependencies["python"].version, build=conda_dependencies["python"].build, ), "colorama": Package( "colorama", "colorama", version=conda_dependencies["colorama"].version, build=conda_dependencies["colorama"].build, ), "pytest": Package( "pytest", "pytest", version=conda_dependencies["pytest"].version, build=conda_dependencies["pytest"].build, ), "pytest-cov": Package( "pytest-cov", "pytest-cov", version=conda_dependencies["pytest-cov"].version, build=conda_dependencies["pytest-cov"].build, ), } } log_list = [ f"conda create --name {env.name} python=3.6 colorama {channel_command}", f"conda install --name {env.name} pytest", f"conda install --name {env.name} pytest-cov --channel main", ] assert env.history.logs == log_list actual_env = (env.local_io.env_dir / "environment.yml").read_text() conda_dependencies = get_dependencies(name=env.name)["conda"] expected_env = [f"name: {env.name}", "channels:"] for channel in channels + ["nodefaults"]: expected_env.append(f" - {channel}") expected_env = ("\n".join(expected_env + [ "dependencies:", " - python=" + conda_dependencies["python"].version, " - colorama=" + conda_dependencies["colorama"].version, " - pytest=" + conda_dependencies["pytest"].version, " - pytest-cov=" + conda_dependencies["pytest-cov"].version, ]) + "\n") assert actual_env == expected_env expected_debug = 3 * [{ "platform": get_platform_name(), "conda_version": CONDA_VERSION, "pip_version": get_pip_version(name=env.name), "timestamp": str(date.today()), }] for i in range(len(env.history.debug)): for key, val in expected_debug[i].items(): if key == "timestamp": assert env.history.debug[i][key].startswith(val) else: assert env.history.debug[i][key] == val
def update_dependencies(self, update_r_dependencies=False): """Update the list of all conda, pip, and R dependencies installed.""" self.dependencies = get_dependencies(name=self.name) if update_r_dependencies: self.dependencies["r"] = get_r_dependencies(name=self.name)
def test_pip_remove(pip_setup): name = pip_setup["name"] env_dir = pip_setup["env_dir"] channels = pip_setup["channels"] pip_remove(name=name, specs=["pytest-cov"], yes=True) history_file = env_dir / "history.yaml" actual_history_content = history_file.read_text() print(actual_history_content) expected_packages = { "conda": {"colorama": "*", "python": "python=3.6"}, "pip": {"pytest": "pytest==4.0.0"}, } actual = yaml.load(actual_history_content, Loader=yaml.FullLoader) expected_log = "pip uninstall pytest-cov" assert actual["packages"] == expected_packages assert actual["revisions"][-1]["log"] == expected_log assert actual["revisions"][-1]["action"] == expected_log packages = get_dependencies(name=name) conda_packages = packages["conda"] pip_packages = packages["pip"] expected_start = [f"name: {name}", "channels:"] for channel in channels + ["nodefaults"]: expected_start.append(f" - {channel}") expected_start.append("dependencies:") expected_conda_packages = [ " - python=" + conda_packages["python"].version, " - colorama=" + conda_packages["colorama"].version, ] expected_pip_packages = [" - pytest==" + pip_packages["pytest"].version] expected = ( "\n".join( expected_start + expected_conda_packages + [" - pip:"] + expected_pip_packages ) + "\n" ) actual = (env_dir / "environment.yml").read_text() print(actual) assert actual == expected expected_packages_section = "\n".join( [ "packages:", " conda:", " python: python=3.6", " colorama: '*'", " pip:", " pytest: pytest==4.0.0", "revisions:", ] ) assert expected_packages_section in actual_history_content expected_third_revision = "\n".join( [ " - packages:", " conda:", " python: python=3.6", " colorama: '*'", " pip:", " pytest: pytest==4.0.0", " diff:", " pip:", " remove:", " - pytest-cov", f" log: pip uninstall pytest-cov", f" action: pip uninstall pytest-cov", ] ) index_first_revision = actual_history_content.find(" - packages:") index_second_revision = actual_history_content.find( " - packages:", index_first_revision + 1 ) index_third_revision = actual_history_content.find( " - packages:", index_second_revision + 1 ) third_action = f" action: {expected_log}" index_third_action = actual_history_content.find( third_action, index_third_revision ) + len(third_action) actual_third_revision = actual_history_content[ index_third_revision:index_third_action ] assert actual_third_revision == expected_third_revision
def create( cls, name: str, packages: Packages, channels: ListLike = None, yes: bool = False, strict_channel_priority: bool = True, ): """Creating a conda environment from a list of packages.""" if name == "base": raise CondaEnvTrackerCondaError( "Environment can not be created using default name base" ) if name in get_all_existing_environment(): message = ( f"This environment {name} already exists. Would you like to replace it" ) if prompt_yes_no(prompt_msg=message, default=False): delete_conda_environment(name=name) local_io = EnvIO(env_directory=USER_ENVS_DIR / name) if local_io.env_dir.exists(): local_io.delete_all() else: raise CondaEnvTrackerCondaError(f"Environment {name} already exists") logger.debug(f"creating conda env {name}") conda_create( name=name, packages=packages, channels=channels, yes=yes, strict_channel_priority=strict_channel_priority, ) create_cmd = get_conda_create_command( name=name, packages=packages, channels=channels, strict_channel_priority=strict_channel_priority, ) specs = Actions.get_package_specs( packages=packages, dependencies=get_dependencies(name=name)["conda"] ) if not channels: channels = get_conda_channels() dependencies = get_dependencies(name=name) history = History.create( name=name, channels=Channels(channels), packages=PackageRevision.create(packages, dependencies=dependencies), logs=Logs(create_cmd), actions=Actions.create( name=name, specs=specs, channels=Channels(channels), strict_channel_priority=strict_channel_priority, ), diff=Diff.create(packages=packages, dependencies=dependencies), debug=Debug.create(name=name), ) env = cls(name=name, history=history, dependencies=dependencies) env.export() return env