def test_conda_env_yaml_r_install(r_setup):
"""Test the environment.yml file in detail."""
name = r_setup["name"]
env_dir = r_setup["env_dir"]
channels = r_setup["channels"]
packages = get_dependencies(name=name)
conda_packages = packages["conda"]
expected_start = [f"name: {name}", "channels:"]
for channel in channels + ["nodefaults"]:
expected_start.append(f" - {channel}")
expected_start.append("dependencies:")
expected_conda_packages = [
" - r-base=" + conda_packages["r-base"].version,
" - r-devtools=" + conda_packages["r-devtools"].version,
]
expected = "\n".join(expected_start + expected_conda_packages) + "\n"
actual = (env_dir / "environment.yml").read_text()
print(actual)
assert actual == expected
install_r = "\n".join(
[
'library("devtools"); install_version("jsonlite", version="1.2")',
'install.packages("praise")',
]
)
actual_install_r = (env_dir / "install.R").read_text()
print(actual_install_r)
assert actual_install_r == install_r
def history_diff(env_name: str, env, env_reader) -> ListLike:
"""return the difference between history and local environment"""
version_diff_pkges: ListLike = []
new_pkges: ListLike = []
missing_pkges: ListLike = []
history_conda_pkges = env_reader.get_environment()["dependencies"]
history_conda_pkges_dict = {}
for spec in history_conda_pkges:
name, package = Package.from_spec(spec)
history_conda_pkges_dict[name] = package
local_conda_pkges = get_dependencies(name=env_name)["conda"]
for name, package in local_conda_pkges.items():
if name in history_conda_pkges_dict:
if package.version != history_conda_pkges_dict[name].version:
version_diff_pkges.append("-" + name + "=" +
history_conda_pkges_dict[name])
version_diff_pkges.append("+" + name + "=" +
package.version)
else:
new_pkges.append("+" + name + "=" + package.version)
for package in env.history.packages["conda"]:
if package not in local_conda_pkges.keys():
missing_pkges.append("-" + package)
return version_diff_pkges, new_pkges, missing_pkges
def get_pip_version(name: str) -> Optional[str]:
"""Check for the version of pip (if installed)."""
if is_current_conda_env(name):
import pip
return pip.__version__
dependencies = get_dependencies(name=name)
return dependencies["conda"].get("pip", Package(name="pip")).version
def infer(cls, name: str, packages: Packages, channels: ListLike = None):
"""create conda_env_tracker environment by inferring to existing conda environment"""
if name == "base":
raise CondaEnvTrackerCondaError(
"Environment can not be created using default name base"
)
if name not in get_all_existing_environment():
raise CondaEnvTrackerCondaError(
f"Environment {name} can not be inferred, does not exist"
)
existing_packages = get_dependencies(name=name)
if "r-base" in existing_packages["conda"]:
existing_r_packages = get_r_dependencies(name=name)
else:
existing_r_packages = {}
user_packages = {"conda": Packages(), "pip": Packages(), "r": Packages()}
for package in packages:
if package.name in existing_packages.get("conda", Packages()):
user_packages["conda"].append(package)
elif package.name in existing_packages.get("pip", Packages()):
user_packages["pip"].append(package)
elif package.name in existing_r_packages:
raise RError(
"Cannot infer R packages, must run follow-up R install command.\n"
f"Found '{package.name}' in installed R packages {existing_r_packages}."
)
else:
raise CondaEnvTrackerCondaError(
f"Environment {name} does not have {package.spec} installed"
)
conda_create_cmd = get_conda_create_command(
name, user_packages["conda"], channels
)
specs = Actions.get_package_specs(
packages=user_packages["conda"], dependencies=existing_packages["conda"]
)
history = History(
name=name,
channels=Channels(channels),
packages=HistoryPackages.create(user_packages["conda"]),
logs=Logs.create(conda_create_cmd),
actions=Actions.create(name=name, specs=specs, channels=Channels(channels)),
debug=Debug.create(name=name),
)
env = cls(name=name, history=history)
if user_packages["pip"]:
handler = PipHandler(env=env)
handler.update_history_install(packages=user_packages["pip"])
env = handler.env
env.export()
return env
def __init__(self, name: str, history: Optional[History] = None):
self.name = name
self.history = history
self.local_io = EnvIO(env_directory=USER_ENVS_DIR / name)
self.dependencies = {}
if self.history:
self.dependencies = get_dependencies(name=self.name)
if self.history.packages.get("r"):
self.dependencies["r"] = get_r_dependencies(name=self.name)
def create(
cls,
name: str,
packages: Packages,
channels: ListLike = None,
yes: bool = False,
strict_channel_priority: bool = True,
):
"""Creating a conda environment from a list of packages."""
if name == "base":
raise CondaEnvTrackerCondaError(
"Environment can not be created using default name base"
)
if name in get_all_existing_environment():
raise CondaEnvTrackerCondaError(f"Environment {name} already exist")
logger.debug(f"creating conda env {name}")
conda_create(
name=name,
packages=packages,
channels=channels,
yes=yes,
strict_channel_priority=strict_channel_priority,
)
create_cmd = get_conda_create_command(
name=name,
packages=packages,
channels=channels,
strict_channel_priority=strict_channel_priority,
)
specs = Actions.get_package_specs(
packages=packages, dependencies=get_dependencies(name=name)["conda"]
)
if not channels:
channels = get_conda_channels()
history = History(
name=name,
channels=Channels(channels),
packages=HistoryPackages.create(packages),
logs=Logs.create(create_cmd),
actions=Actions.create(
name=name,
specs=specs,
channels=Channels(channels),
strict_channel_priority=strict_channel_priority,
),
debug=Debug.create(name=name),
)
env = cls(name=name, history=history)
env.export()
return env
def test_get_packages_empty(mocker, pkg_list):
run_mock = mocker.patch("conda_env_tracker.gateways.conda.subprocess.run")
attrs = {
"return_value.stdout": pkg_list,
"return_value.stderr": None,
"return_value.returncode": 0,
}
run_mock.configure_mock(**attrs)
package_dict = get_dependencies("test-create")
assert not package_dict["conda"]
assert "pip" not in package_dict
def test_r_remove_package(r_setup):
name = r_setup["name"]
env_dir = r_setup["env_dir"]
channels = r_setup["channels"]
history_file = env_dir / "history.yaml"
r_remove(name=name, specs=["praise"], yes=True)
actual_history_content = history_file.read_text()
print(actual_history_content)
expected_packages = {
"conda": {
"r-base": "*",
"r-devtools": "*"
},
"r": {
"jsonlite": "1.2"
},
}
actual = yaml.load(actual_history_content, Loader=yaml.FullLoader)
remove_command = 'remove.packages(c("praise"))'
expected_log = f"R --quiet --vanilla -e '{remove_command}'"
assert actual["packages"] == expected_packages
assert actual["logs"][-1] == expected_log
assert actual["actions"][-1] == expected_log
packages = get_dependencies(name=name)
conda_packages = packages["conda"]
expected_start = [f"name: {name}", "channels:"]
for channel in channels + ["nodefaults"]:
expected_start.append(f" - {channel}")
expected_start.append("dependencies:")
expected_conda_packages = [
" - r-base=" + conda_packages["r-base"].version,
" - r-devtools=" + conda_packages["r-devtools"].version,
]
expected = "\n".join(expected_start + expected_conda_packages) + "\n"
actual = (env_dir / "conda-env.yaml").read_text()
print(actual)
assert actual == expected
install_r = "\n".join(
['library("devtools"); install_version("jsonlite", version="1.2")'])
actual_install_r = (env_dir / "install.R").read_text()
print(actual_install_r)
assert actual_install_r == install_r
def test_pip_remove(pip_setup):
name = pip_setup["name"]
env_dir = pip_setup["env_dir"]
channels = pip_setup["channels"]
history_file = env_dir / "history.yaml"
pip_remove(name=name, specs=["pytest-cov"], yes=True)
actual_history_content = history_file.read_text()
print(actual_history_content)
expected_packages = {
"conda": {"colorama": "*", "python": "3.6"},
"pip": {"pytest": "4.0.0"},
}
actual = yaml.load(actual_history_content, Loader=yaml.FullLoader)
expected_log = "pip uninstall pytest-cov"
assert actual["packages"] == expected_packages
assert actual["logs"][-1] == expected_log
assert actual["actions"][-1] == expected_log
packages = get_dependencies(name=name)
conda_packages = packages["conda"]
pip_packages = packages["pip"]
expected_start = [f"name: {name}", "channels:"]
for channel in channels + ["nodefaults"]:
expected_start.append(f" - {channel}")
expected_start.append("dependencies:")
expected_conda_packages = [
" - python=" + conda_packages["python"].version,
" - colorama=" + conda_packages["colorama"].version,
]
expected_pip_packages = [" - pytest==" + pip_packages["pytest"].version]
expected = (
"\n".join(
expected_start
+ expected_conda_packages
+ [" - pip:"]
+ expected_pip_packages
)
+ "\n"
)
actual = (env_dir / "conda-env.yaml").read_text()
print(actual)
assert actual == expected
def __init__(
self,
name: str,
history: Optional[History] = None,
dependencies: Optional[dict] = None,
):
self.name = name
self.history = history
self.local_io = EnvIO(env_directory=USER_ENVS_DIR / name)
if dependencies:
self.dependencies = dependencies
else:
self.dependencies = {}
if self.history:
self.dependencies = get_dependencies(name=self.name)
if self.history.packages.get("r"):
self.dependencies["r"] = get_r_dependencies(name=self.name)
if history:
self.history.packages.update_versions(dependencies=self.dependencies)
def test_get_packages(mocker, packages):
run_mock = mocker.patch("conda_env_tracker.gateways.conda.subprocess.run")
attrs = {
"return_value.stdout": packages,
"return_value.stderr": None,
"return_value.returncode": 0,
}
run_mock.configure_mock(**attrs)
actual = get_dependencies("test-create")
expected = {
"conda": {
"pylint": Package("pylint", "pylint", "2.0.4", "py37_0")
}
}
if packages.endswith("pypi") or packages.endswith("pip"):
expected["pip"] = {"pytest": Package("pytest", "pytest", "1.0.7")}
else:
expected["conda"]["pytest"] = Package("pytest", "pytest", "1.0.7",
"py37_0")
assert actual == expected
def test_conda_env_yaml_pip_install(pip_setup):
"""Test the environment.yml file in detail."""
name = pip_setup["name"]
env_dir = pip_setup["env_dir"]
channels = pip_setup["channels"]
packages = get_dependencies(name=name)
conda_packages = packages["conda"]
pip_packages = packages["pip"]
expected_start = [f"name: {name}", "channels:"]
for channel in channels + ["nodefaults"]:
expected_start.append(f" - {channel}")
expected_start.append("dependencies:")
expected_conda_packages = [
" - python=" + conda_packages["python"].version,
" - colorama=" + conda_packages["colorama"].version,
]
expected_pip_packages = [
" - pytest==" + pip_packages["pytest"].version,
" - pytest-cov==" + pip_packages["pytest-cov"].version,
]
expected = (
"\n".join(
expected_start
+ expected_conda_packages
+ [" - pip:"]
+ expected_pip_packages
)
+ "\n"
)
actual = (env_dir / "environment.yml").read_text()
print(actual)
assert actual == expected
def test_conda_env_yaml_after_create(end_to_end_setup):
"""Test the conda-env.yaml file in detail."""
name = end_to_end_setup["name"]
env_dir = end_to_end_setup["env_dir"]
channels = end_to_end_setup["channels"]
conda_packages = get_dependencies(name=name)["conda"]
expected_start = [f"name: {name}", "channels:"]
for channel in channels + ["nodefaults"]:
expected_start.append(f" - {channel}")
expected_start.append("dependencies:")
expected_packages = [
" - python=" + conda_packages["python"].version,
" - colorama=" + conda_packages["colorama"].version,
]
expected = "\n".join(expected_start + expected_packages) + "\n"
actual = (env_dir / "conda-env.yaml").read_text()
print(actual)
assert actual == expected
def test_r_remove_package(r_setup):
# pylint: disable=too-many-locals
name = r_setup["name"]
env_dir = r_setup["env_dir"]
channels = r_setup["channels"]
r_remove(name=name, specs=["praise"], yes=True)
history_file = env_dir / "history.yaml"
actual_history_content = history_file.read_text()
print(actual_history_content)
expected_packages = {
"conda": {"r-base": "*", "r-devtools": "*"},
"r": {
"jsonlite": 'library("devtools"); install_version("jsonlite", version="1.2")'
},
}
actual = yaml.load(actual_history_content, Loader=yaml.FullLoader)
remove_command = r"remove.packages(c(\"praise\"))"
expected_log = f'R --quiet --vanilla -e "{remove_command}"'
assert actual["packages"] == expected_packages
assert actual["revisions"][-1]["log"] == expected_log
assert actual["revisions"][-1]["action"] == expected_log
packages = get_dependencies(name=name)
conda_packages = packages["conda"]
expected_start = [f"name: {name}", "channels:"]
for channel in channels + ["nodefaults"]:
expected_start.append(f" - {channel}")
expected_start.append("dependencies:")
expected_conda_packages = [
" - r-base=" + conda_packages["r-base"].version,
" - r-devtools=" + conda_packages["r-devtools"].version,
]
expected = "\n".join(expected_start + expected_conda_packages) + "\n"
actual = (env_dir / "environment.yml").read_text()
print(actual)
assert actual == expected
install_r = "\n".join(
['library("devtools"); install_version("jsonlite", version="1.2")']
)
actual_install_r = (env_dir / "install.R").read_text()
print(actual_install_r)
assert actual_install_r == install_r
expected_packages_section = "\n".join(
[
"packages:",
" conda:",
" r-base: '*'",
" r-devtools: '*'",
" r:",
' jsonlite: library("devtools"); install_version("jsonlite",version="1.2")',
"revisions:",
]
)
assert expected_packages_section in actual_history_content
expected_third_revision = "\n".join(
[
" - packages:",
" conda:",
" r-base: '*'",
" r-devtools: '*'",
" r:",
' jsonlite: library("devtools"); install_version("jsonlite",version="1.2")',
" diff:",
" r:",
" remove:",
" - praise",
rf' log: R --quiet --vanilla -e "remove.packages(c(\"praise\"))"',
rf' action: R --quiet --vanilla -e "remove.packages(c(\"praise\"))"',
]
)
index_first_revision = actual_history_content.find(" - packages:")
index_second_revision = actual_history_content.find(
" - packages:", index_first_revision + 1
)
index_third_revision = actual_history_content.find(
" - packages:", index_second_revision + 1
)
third_action = f" action: {expected_log}"
index_third_action = actual_history_content.find(
third_action, index_third_revision
) + len(third_action)
actual_third_revision = actual_history_content[
index_third_revision:index_third_action
]
assert actual_third_revision == expected_third_revision
def test_remote_end_to_end(end_to_end_setup, mocker):
"""Test setup, create, install, pull and push feature of conda_env_tracker"""
env = end_to_end_setup["env"]
remote_dir = end_to_end_setup["remote_dir"]
channels = end_to_end_setup["channels"]
channel_command = end_to_end_setup["channel_command"]
setup_remote(name=env.name, remote_dir=remote_dir)
local_io = EnvIO(env_directory=USER_ENVS_DIR / env.name)
assert str(local_io.get_remote_dir()) == str(remote_dir)
assert not list(remote_dir.glob("*"))
push(name=env.name)
remote_io = EnvIO(env_directory=remote_dir)
assert remote_io.get_history() == local_io.get_history()
assert remote_io.get_environment() == local_io.get_environment()
env = conda_install(name=env.name, specs=["pytest"], yes=True)
assert env.local_io.get_history() != remote_io.get_history()
env = push(name=env.name)
assert env.local_io.get_history() == remote_io.get_history()
log_mock = mocker.patch("conda_env_tracker.pull.logging.Logger.info")
env = pull(name=env.name)
log_mock.assert_called_once_with("Nothing to pull.")
remove_package_from_history(env, "pytest")
conda_dependencies = env.dependencies["conda"]
assert env.local_io.get_history() != remote_io.get_history()
assert env.history.packages == {
"conda": {
"python":
Package(
"python",
"python=3.6",
version=conda_dependencies["python"].version,
build=conda_dependencies["python"].build,
),
"colorama":
Package(
"colorama",
"colorama",
version=conda_dependencies["colorama"].version,
build=conda_dependencies["colorama"].build,
),
}
}
env = pull(name=env.name)
assert env.local_io.get_history() == remote_io.get_history()
assert remote_io.get_environment() == local_io.get_environment()
remove_package_from_history(env, "pytest")
assert env.local_io.get_history() != remote_io.get_history()
env = conda_install(name=env.name,
specs=["pytest-cov"],
channels=("main", ),
yes=True)
env = pull(name=env.name, yes=True)
conda_dependencies = env.dependencies["conda"]
assert env.history.packages == {
"conda": {
"python":
Package(
"python",
"python=3.6",
version=conda_dependencies["python"].version,
build=conda_dependencies["python"].build,
),
"colorama":
Package(
"colorama",
"colorama",
version=conda_dependencies["colorama"].version,
build=conda_dependencies["colorama"].build,
),
"pytest":
Package(
"pytest",
"pytest",
version=conda_dependencies["pytest"].version,
build=conda_dependencies["pytest"].build,
),
"pytest-cov":
Package(
"pytest-cov",
"pytest-cov",
version=conda_dependencies["pytest-cov"].version,
build=conda_dependencies["pytest-cov"].build,
),
}
}
log_list = [
f"conda create --name {env.name} python=3.6 colorama {channel_command}",
f"conda install --name {env.name} pytest",
f"conda install --name {env.name} pytest-cov --channel main",
]
assert env.history.logs == log_list
actual_env = (env.local_io.env_dir / "environment.yml").read_text()
conda_dependencies = get_dependencies(name=env.name)["conda"]
expected_env = [f"name: {env.name}", "channels:"]
for channel in channels + ["nodefaults"]:
expected_env.append(f" - {channel}")
expected_env = ("\n".join(expected_env + [
"dependencies:",
" - python=" + conda_dependencies["python"].version,
" - colorama=" + conda_dependencies["colorama"].version,
" - pytest=" + conda_dependencies["pytest"].version,
" - pytest-cov=" + conda_dependencies["pytest-cov"].version,
]) + "\n")
assert actual_env == expected_env
expected_debug = 3 * [{
"platform": get_platform_name(),
"conda_version": CONDA_VERSION,
"pip_version": get_pip_version(name=env.name),
"timestamp": str(date.today()),
}]
for i in range(len(env.history.debug)):
for key, val in expected_debug[i].items():
if key == "timestamp":
assert env.history.debug[i][key].startswith(val)
else:
assert env.history.debug[i][key] == val
def update_dependencies(self, update_r_dependencies=False):
"""Update the list of all conda, pip, and R dependencies installed."""
self.dependencies = get_dependencies(name=self.name)
if update_r_dependencies:
self.dependencies["r"] = get_r_dependencies(name=self.name)
def test_pip_remove(pip_setup):
name = pip_setup["name"]
env_dir = pip_setup["env_dir"]
channels = pip_setup["channels"]
pip_remove(name=name, specs=["pytest-cov"], yes=True)
history_file = env_dir / "history.yaml"
actual_history_content = history_file.read_text()
print(actual_history_content)
expected_packages = {
"conda": {"colorama": "*", "python": "python=3.6"},
"pip": {"pytest": "pytest==4.0.0"},
}
actual = yaml.load(actual_history_content, Loader=yaml.FullLoader)
expected_log = "pip uninstall pytest-cov"
assert actual["packages"] == expected_packages
assert actual["revisions"][-1]["log"] == expected_log
assert actual["revisions"][-1]["action"] == expected_log
packages = get_dependencies(name=name)
conda_packages = packages["conda"]
pip_packages = packages["pip"]
expected_start = [f"name: {name}", "channels:"]
for channel in channels + ["nodefaults"]:
expected_start.append(f" - {channel}")
expected_start.append("dependencies:")
expected_conda_packages = [
" - python=" + conda_packages["python"].version,
" - colorama=" + conda_packages["colorama"].version,
]
expected_pip_packages = [" - pytest==" + pip_packages["pytest"].version]
expected = (
"\n".join(
expected_start
+ expected_conda_packages
+ [" - pip:"]
+ expected_pip_packages
)
+ "\n"
)
actual = (env_dir / "environment.yml").read_text()
print(actual)
assert actual == expected
expected_packages_section = "\n".join(
[
"packages:",
" conda:",
" python: python=3.6",
" colorama: '*'",
" pip:",
" pytest: pytest==4.0.0",
"revisions:",
]
)
assert expected_packages_section in actual_history_content
expected_third_revision = "\n".join(
[
" - packages:",
" conda:",
" python: python=3.6",
" colorama: '*'",
" pip:",
" pytest: pytest==4.0.0",
" diff:",
" pip:",
" remove:",
" - pytest-cov",
f" log: pip uninstall pytest-cov",
f" action: pip uninstall pytest-cov",
]
)
index_first_revision = actual_history_content.find(" - packages:")
index_second_revision = actual_history_content.find(
" - packages:", index_first_revision + 1
)
index_third_revision = actual_history_content.find(
" - packages:", index_second_revision + 1
)
third_action = f" action: {expected_log}"
index_third_action = actual_history_content.find(
third_action, index_third_revision
) + len(third_action)
actual_third_revision = actual_history_content[
index_third_revision:index_third_action
]
assert actual_third_revision == expected_third_revision
def create(
cls,
name: str,
packages: Packages,
channels: ListLike = None,
yes: bool = False,
strict_channel_priority: bool = True,
):
"""Creating a conda environment from a list of packages."""
if name == "base":
raise CondaEnvTrackerCondaError(
"Environment can not be created using default name base"
)
if name in get_all_existing_environment():
message = (
f"This environment {name} already exists. Would you like to replace it"
)
if prompt_yes_no(prompt_msg=message, default=False):
delete_conda_environment(name=name)
local_io = EnvIO(env_directory=USER_ENVS_DIR / name)
if local_io.env_dir.exists():
local_io.delete_all()
else:
raise CondaEnvTrackerCondaError(f"Environment {name} already exists")
logger.debug(f"creating conda env {name}")
conda_create(
name=name,
packages=packages,
channels=channels,
yes=yes,
strict_channel_priority=strict_channel_priority,
)
create_cmd = get_conda_create_command(
name=name,
packages=packages,
channels=channels,
strict_channel_priority=strict_channel_priority,
)
specs = Actions.get_package_specs(
packages=packages, dependencies=get_dependencies(name=name)["conda"]
)
if not channels:
channels = get_conda_channels()
dependencies = get_dependencies(name=name)
history = History.create(
name=name,
channels=Channels(channels),
packages=PackageRevision.create(packages, dependencies=dependencies),
logs=Logs(create_cmd),
actions=Actions.create(
name=name,
specs=specs,
channels=Channels(channels),
strict_channel_priority=strict_channel_priority,
),
diff=Diff.create(packages=packages, dependencies=dependencies),
debug=Debug.create(name=name),
)
env = cls(name=name, history=history, dependencies=dependencies)
env.export()
return env
