def test_pull_new_action_in_both(setup_tests, local_packages, local_logs,
local_actions):
overwrite_mock = setup_tests["overwrite_mock"]
remote_history = setup_tests["remote_history"]
packages_mock = setup_tests["get_dependencies"]
packages = HistoryPackages.create(local_packages["conda"])
if local_packages.get("pip"):
packages.update_packages(local_packages["pip"], source="pip")
local_history = History(
name=ENV_NAME,
channels=CHANNELS,
packages=packages,
logs=Logs([log for log in local_logs]),
actions=Actions(local_actions),
debug=Debug(),
)
env = Environment(name=ENV_NAME, history=local_history)
if local_packages.get("pip") is not None:
packages_mock.configure_mock(
return_value={
"conda": {
"pandas": Package("pandas", "pandas", "0.23", "py36"),
"pytest": Package("pytest", "pytest", "0.1", "py36_3"),
},
"pip": {
"pylint": Package("pylint", "pylint", "1.11")
},
})
else:
packages_mock.configure_mock(
return_value={
"conda": {
"pandas": Package("pandas", "pandas", "0.23", "py36"),
"pytest": Package("pytest", "pytest", "0.1", "py36_3"),
"pylint": Package("pylint", "pylint", "1.11", "py36"),
},
"pip": {},
})
pull(env=env)
expected_packages = remote_history.packages
if local_packages.get("pip"):
expected_packages["pip"]["pylint"] = local_history.packages["pip"][
"pylint"]
elif "pylint" in local_history.packages["conda"]:
expected_packages["conda"]["pylint"] = local_history.packages["conda"][
"pylint"]
expected_logs = remote_history.logs
expected_actions = remote_history.actions
assert env.history.packages == expected_packages
assert env.history.logs == expected_logs
assert env.history.actions == expected_actions
assert env.history.logs[-1] == local_history.logs[-1]
assert env.history.actions[-1] == local_history.actions[-1]
overwrite_mock.called_once()
def conda_update(
name: str,
specs: ListLike = (),
channels: ListLike = None,
all=False,
yes: bool = False,
strict_channel_priority: bool = True,
) -> Environment:
"""Install conda packages into the environment."""
env = Environment.read(name=name)
cleaned = clean_specs(specs)
if all:
CondaHandler(env=env).update_all(
packages=cleaned,
channels=channels,
yes=yes,
strict_channel_priority=strict_channel_priority,
)
else:
CondaHandler(env=env).install(
packages=cleaned,
channels=channels,
yes=yes,
strict_channel_priority=strict_channel_priority,
)
_ask_user_to_sync(name=name, yes=yes)
return env
def test_get_packages_with_version_spec(mocker):
mocker.patch(
"conda_env_tracker.env.get_dependencies",
return_value={
"conda": {
"pandas": Package("pandas", "pandas", "0.23", "py_36"),
"numpy": Package("numpy", "numpy", "0.13", "py_36"),
},
"pip": {
"pytest": Package("pytest", "pytest", "4.0")
},
},
)
local_packages = HistoryPackages.create(
(Package.from_spec("pandas=0.23"), ))
local_packages.update_packages(
packages=(Package.from_spec("pytest==4.0"), ), source="pip")
local_history = History(
name=ENV_NAME,
packages=local_packages,
channels=Channels("defaults"),
logs=Logs([]),
actions=Actions(),
debug=Debug(),
)
env = Environment(name=ENV_NAME, history=local_history)
expected = {
"conda": [Package("pandas", "pandas=0.23", "0.23", "py_36")],
"pip": [Package("pytest", "pytest==4.0", "4.0")],
}
actual = get_packages(env)
assert actual == expected
def test_pull_no_new_action_in_local(setup_tests):
overwrite_mock = setup_tests["overwrite_mock"]
remote_history = setup_tests["remote_history"]
local_packages = (Package.from_spec("pandas"), )
local_logs = ["conda create --name pull_testing_environment pandas"]
local_actions = [
"conda create --name pull_testing_environment pandas=0.23=py36"
]
local_history = History(
name=ENV_NAME,
packages=HistoryPackages.create(local_packages),
channels=Channels(),
logs=Logs([log for log in local_logs]),
actions=Actions(local_actions),
debug=Debug(),
)
env = Environment(name=ENV_NAME, history=local_history)
pull(env=env)
assert env.history.packages == remote_history.packages
assert env.history.logs == remote_history.logs
assert env.history.actions == remote_history.actions
overwrite_mock.called_once()
def test_infer_environment_does_not_exist(mocker):
env_name = "infer-test"
mocker.patch("conda_env_tracker.history.get_pip_version",
mocker.Mock(return_value="18.1"))
mocker.patch(
"conda_env_tracker.env.get_all_existing_environment",
return_value=["env-not-exist"],
)
with pytest.raises(Exception) as err:
Environment.infer(
name=env_name,
packages=Packages.from_specs("pandas"),
channels=["conda-forge", "main"],
)
assert (str(err.value) ==
f"Environment {env_name} can not be inferred, does not exist")
def test_missing_r_base():
with pytest.raises(RError) as err:
env = Environment.read(name="test_env_name_that_does_not_exist")
check_r_base_package(env=env)
assert (
str(err.value)
== f'"r-base" not installed.\nFound conda packages:\n[]\nMust have "r-base" conda installed to install R packages.'
)
def pip_remove(name: str, specs: ListLike, yes: bool = False) -> Environment:
"""Remove pip packages including custom packages"""
env = Environment.read(name=name)
check_pip(env=env)
cleaned = process_specs(specs)
PipHandler(env=env).remove(packages=cleaned, yes=yes)
_ask_user_to_sync(name=name, yes=yes)
return env
def r_remove(name: str, specs=ListLike, yes: bool = False) -> Environment:
"""R remove spec"""
env = Environment.read(name=name)
check_r_base_package(env=env)
packages = Packages.from_specs(specs)
RHandler(env=env).remove(packages=packages)
_ask_user_to_sync(name=name, yes=yes)
return env
def diff(name: str) -> ListLike:
"""Get the difference between history yaml and local conda environment"""
env = Environment.read(name=name)
env_reader = EnvIO(env_directory=USER_ENVS_DIR / name)
version_diff_pkges, new_pkges, missing_pkges = History.history_diff(
env_name=name, env=env, env_reader=env_reader
)
return missing_pkges + version_diff_pkges + new_pkges
def conda_remove(
name: str, specs: ListLike, channels: ListLike = None, yes: bool = False
) -> Environment:
"""Remove conda packages into the environment."""
env = Environment.read(name=name)
cleaned = clean_specs(specs)
CondaHandler(env=env).remove(packages=cleaned, channels=channels, yes=yes)
_ask_user_to_sync(name=name, yes=yes)
return env
def test_create_history_success(mocker):
env_name = "conda_env_tracker-test-create-history"
create_cmd = f"conda create --name {env_name} pandas"
channels = ["conda-forge", "main"]
packages = Packages.from_specs("pandas")
action = "pandas=0.23=py_36"
mocker.patch("conda_env_tracker.env.get_all_existing_environment")
mocker.patch("conda_env_tracker.env.conda_create",
mocker.Mock(return_value=create_cmd))
mocker.patch(
"conda_env_tracker.env.get_dependencies",
mocker.Mock(return_value={
"conda": {
"pandas": Package("pandas", "*", "0.23=py_36")
},
"pip": {},
}),
)
mocker.patch(
"conda_env_tracker.history.debug.get_pip_version",
mocker.Mock(return_value="18.1"),
)
mocker.patch("conda_env_tracker.env.get_conda_channels",
mocker.Mock(return_value=channels))
Environment.create(name=env_name, packages=packages)
writer = EnvIO(env_directory=USER_ENVS_DIR / env_name)
history = writer.get_history()
channel_string = "--override-channels --strict-channel-priority " + " ".join(
"--channel " + channel for channel in channels)
assert history.actions == [
f"conda create --name {env_name} {action} {channel_string}"
]
assert history.packages == {
"conda": {
"pandas": Package.from_spec("pandas")
}
}
assert history.channels == channels
assert history.logs == [create_cmd]
env_dir = Path(USER_ENVS_DIR) / env_name
shutil.rmtree(env_dir)
def r_install(
name: str, package_names: ListLike, commands: ListLike, yes: bool = False
) -> Environment:
"""Install R packages with corresponding R command."""
env = Environment.read(name=name)
check_r_base_package(env=env)
packages = clean_r_specs(package_names=package_names, commands=commands)
RHandler(env=env).install(packages=packages)
_ask_user_to_sync(name=name, yes=yes)
return env
def pip_custom_install(
name: str, package: str, url_path: str, yes: bool = False
) -> Environment:
"""Install custom pip package"""
env = Environment.read(name=name)
check_pip(env=env)
cleaned = Package(name=package.lower(), spec=url_path)
PipHandler(env=env).custom_install(package=cleaned)
_ask_user_to_sync(name=name, yes=yes)
return env
def setup(mocker, request):
"""Set up for diff function"""
local_packages = request.param["local_packages"]
local_logs = request.param["local_logs"]
local_actions = request.param["local_actions"]
env_name = request.param["env_name"]
remote_packages = request.param["remote_packages"]
remote_logs = request.param["remote_logs"]
remote_actions = request.param["remote_actions"]
dependencies = {
"conda": {
"pandas": Package("pandas", "pandas", version="0.23.0"),
"pytest": Package("pytest", "pytest", version="4.0.0"),
}
}
mocker.patch(
"conda_env_tracker.env.get_dependencies", mocker.Mock(return_value=dependencies)
)
history = History.create(
name=env_name,
packages=PackageRevision.create(local_packages, dependencies=dependencies),
channels=Channels(["conda-forge"]),
logs=Logs([log for log in local_logs]),
actions=local_actions,
diff=Diff(),
debug=Debug(),
)
env = Environment(name=env_name, history=history)
mocker.patch("conda_env_tracker.main.Environment.read", return_value=env)
mocker.patch(
"conda_env_tracker.gateways.io.EnvIO.get_remote_dir",
return_value="~/path/to/remote",
)
mocker.patch("pathlib.Path.is_dir", return_value=True)
history = History.create(
name=env_name,
channels=Channels(["conda-forge"]),
packages=PackageRevision.create(remote_packages, dependencies=dependencies),
logs=Logs([log for log in remote_logs]),
actions=remote_actions,
diff=Diff(),
debug=Debug(),
)
mocker.patch(
"conda_env_tracker.gateways.io.EnvIO.get_history", return_value=history
)
mocker.patch("pathlib.Path.write_text")
yield env, request.param["push_should_fail"]
if (USER_ENVS_DIR / env_name).exists():
shutil.rmtree(USER_ENVS_DIR / env_name)
def test_pull_pip(setup_tests):
new = Package.from_spec("pytest-pylint==0.13.0")
remote_history = setup_tests["remote_history"]
remote_history.packages.update_packages(packages=[new], source="pip")
remote_history.logs.append(f"pip install {new}")
remote_history.actions.append(f"pip install {new}")
env = Environment(name="test", history=None)
final_env = pull(env=env)
assert final_env.history == remote_history
def update_packages(name: str, specs: ListLike, remove: ListLike) -> Environment:
"""Update the history with local packages installed without cet cli."""
# pylint: disable=redefined-outer-name
env = Environment.read(name=name)
handler = CondaHandler(env=env)
if remove:
cleaned_remove = clean_specs(remove)
handler.update_history_remove(packages=cleaned_remove)
if specs:
cleaned = clean_specs(specs)
handler.update_history_install(packages=cleaned)
env.export()
return env
def pip_install(
name: str,
specs: ListLike,
index_url: Union[str, ListLike] = PIP_DEFAULT_INDEX_URL,
yes: bool = False,
) -> Environment:
"""Install pip packages into the environment."""
env = Environment.read(name=name)
check_pip(env=env)
cleaned = process_specs(specs, check_custom=True)
PipHandler(env=env).install(packages=cleaned, index_url=index_url)
_ask_user_to_sync(name=name, yes=yes)
return env
def test_pull_different_actions_with_pip(setup_tests):
new = Package.from_spec("pytest-pylint==0.13.0")
remote_history = setup_tests["remote_history"]
local_history = copy.deepcopy(remote_history)
env = Environment(name=local_history.name, history=local_history)
remote_history.packages.update_packages(packages=[new], source="pip")
remote_history.logs.append(f"pip install {new}")
remote_history.actions.append(f"pip install {new}")
final_env = pull(env=env)
assert final_env.history == remote_history
def test_infer_environment_package_does_not_exist(mocker):
env_name = "infer-test"
dependencies = {
"conda": {
"pandas": Package("pandas", "pandas", "0.23", "py_36")
}
}
mocker.patch("conda_env_tracker.history.get_pip_version",
mocker.Mock(return_value="18.1"))
mocker.patch("conda_env_tracker.env.get_all_existing_environment",
return_value=[env_name])
mocker.patch("conda_env_tracker.env.get_dependencies",
mocker.Mock(return_value=dependencies))
mocker.patch("conda_env_tracker.env.get_r_dependencies")
with pytest.raises(Exception) as err:
Environment.infer(
name=env_name,
packages=Packages.from_specs("pytest"),
channels=["conda-forge", "main"],
)
assert str(
err.value) == "Environment infer-test does not have pytest installed"
def test_infer_environment_with_pip_package_success(mocker):
env_name = "infer-test"
dependencies = {
"conda": {
"pandas": Package("pandas", "pandas", "0.23", "py_36")
},
"pip": {
"pytest": Package("pytest", "pytest", "0.11")
},
}
mocker.patch(
"conda_env_tracker.history.debug.get_pip_version",
mocker.Mock(return_value="18.1"),
)
mocker.patch("conda_env_tracker.env.get_all_existing_environment",
return_value=[env_name])
mocker.patch("conda_env_tracker.env.get_dependencies",
mocker.Mock(return_value=dependencies))
mocker.patch("conda_env_tracker.env.get_r_dependencies")
env = Environment.infer(
name=env_name,
packages=Packages.from_specs(["pandas", "pytest"]),
channels=["conda-forge", "main"],
)
assert env.history.packages == {
"conda": {
"pandas": Package("pandas", "pandas", "0.23", "py_36")
},
"pip": {
"pytest": Package("pytest", "pytest", "0.11")
},
}
assert env.history.channels == ["conda-forge", "main"]
assert env.history.logs == [
f"conda create --name {env_name} pandas --override-channels --strict-channel-priority "
"--channel conda-forge "
"--channel main",
"pip install pytest --index-url https://pypi.org/simple",
]
assert env.history.actions == [
f"conda create --name {env_name} pandas=0.23=py_36 "
"--override-channels --strict-channel-priority "
"--channel conda-forge "
"--channel main",
"pip install pytest==0.11 --index-url"
" https://pypi.org/simple",
]
def test_pull_r(setup_r_tests):
remote_history = setup_r_tests["remote_history"]
remote_history.packages.update_packages(
packages=Packages(Package("jsonlite", "install.packages('jsonlite')")),
source="r",
)
remote_history.logs.append(
f"{R_COMMAND} -e 'install.packages('jsonlite')'")
remote_history.actions.append(
f"{R_COMMAND} -e 'install.packages('jsonlite')'")
env = Environment(name="test", history=None)
final_env = pull(env=env)
assert final_env.history == remote_history
def test_pull_different_actions_in_both_r(setup_r_tests):
remote_history = setup_r_tests["remote_history"]
get_r_dependencies = setup_r_tests["get_r_dependencies"]
get_r_dependencies.configure_mock(
**{
"return_value": {
"praise": Package("praise", "praise", "1.0.0"),
"jsonlite": Package("jsonlite", "jsonlite", "1.6"),
}
})
local_history = copy.deepcopy(remote_history)
local_history.packages.update_packages(packages=Packages(
Package("praise", 'install.packages("praise")')),
source="r")
local_history.logs.append(f"{R_COMMAND} -e 'install.packages(\"praise\")'")
local_history.actions.append(
f"{R_COMMAND} -e 'install.packages(\"praise\")'")
env = Environment(name=local_history.name, history=local_history)
remote_history.packages.update_packages(
packages=Packages(Package("jsonlite", 'install.packages("jsonlite")')),
source="r",
)
remote_history.logs.append(
f"{R_COMMAND} -e 'install.packages(\"jsonlite\")'")
remote_history.actions.append(
f"{R_COMMAND} -e 'install.packages(\"jsonlite\")'")
expected_history = copy.deepcopy(remote_history)
expected_history.packages.update_packages(packages=Packages(
Package("praise", 'install.packages("praise")')),
source="r")
expected_history.logs.append(
f"{R_COMMAND} -e 'install.packages(\"praise\")'")
expected_history.actions.append(
f"{R_COMMAND} -e 'install.packages(\"praise\")'")
final_env = pull(env=env)
assert final_env.history.packages == expected_history.packages
assert final_env.history.logs == expected_history.logs
assert final_env.history.actions == expected_history.actions
def test_pull_remote_local_different_create_commands(setup_tests,
local_packages,
local_logs,
local_actions):
# pylint: disable=unused-argument
local_history = History(
name=ENV_NAME,
channels=CHANNELS,
packages=HistoryPackages.create(local_packages),
logs=Logs([log for log in local_logs]),
actions=Actions(local_actions),
debug=Debug(),
)
env = Environment(name=ENV_NAME, history=local_history)
with pytest.raises(CondaEnvTrackerCreationError):
pull(env=env)
def test_get_packages_r(mocker):
mocker.patch(
"conda_env_tracker.env.get_dependencies",
return_value={
"conda": {
"r-base":
Package("r-base", "r-base", "3.5.1", "h539"),
"r-devtools":
Package("r-devtools", "r-devtools", "0.1.0", "r351_0"),
}
},
)
mocker.patch(
"conda_env_tracker.env.get_r_dependencies",
return_value={
"trelliscopejs": Package("trelliscopejs", "trelliscopejs"
"0.1.0")
},
)
local_packages = HistoryPackages.create(
Packages(
(Package.from_spec("r-base"), Package.from_spec("r-devtools"))))
local_packages.update_packages(
packages=(Package.from_spec("trelliscopejs"), ), source="r")
local_history = History(
name=ENV_NAME,
packages=local_packages,
channels=Channels("defaults"),
logs=Logs([]),
actions=Actions(),
debug=Debug(),
)
env = Environment(name=ENV_NAME, history=local_history)
expected = {
"conda": [
Package("r-base", "r-base", "3.5.1", "h539"),
Package("r-devtools", "r-devtools", "0.1.0", "r351_0"),
],
"r": [Package("trelliscopejs", "trelliscopejs", "0.1.0")],
}
actual = get_packages(env)
assert actual == expected
def test_pull_actions_in_different_order(setup_tests):
overwrite_mock = setup_tests["overwrite_mock"]
remote_history = setup_tests["remote_history"]
new_log = "conda install --name pull_testing_environment pylint"
new_action = "conda install --name pull_testing_environment pylint=1.11=py36"
remote_history.packages["pylint"] = "*"
remote_history.logs.append(new_log)
remote_history.actions.append(new_action)
local_packages = (
Package.from_spec("pandas"),
Package.from_spec("pylint"),
Package.from_spec("pytest"),
)
local_logs = [
"conda create --name pull_testing_environment pandas",
new_log,
"conda install --name pull_testing_environment pytest",
]
local_actions = [
"conda create --name pull_testing_environment pandas=0.23=py36",
new_action,
"conda install --name pull_testing_environment pytest=0.1=py36_3",
]
local_history = History(
name=ENV_NAME,
channels=CHANNELS,
packages=HistoryPackages.create(local_packages),
logs=Logs([log for log in local_logs]),
actions=Actions(local_actions),
debug=Debug(),
)
env = Environment(name=ENV_NAME, history=local_history)
pull(env=env)
assert env.history.packages == remote_history.packages
assert env.history.logs == remote_history.logs
assert env.history.actions == remote_history.actions
overwrite_mock.called_once()
def create(
name: str,
specs: ListLike,
channels: ListLike = None,
yes: bool = False,
strict_channel_priority: bool = True,
) -> Environment:
"""create conda environment given environment name and packages to install"""
cleaned = process_specs(specs)
env = Environment.create(
name=name,
packages=cleaned,
channels=channels,
yes=yes,
strict_channel_priority=strict_channel_priority,
)
env.export()
if not yes:
jupyter_kernel_install_query(name=name, packages=cleaned)
return env
def conda_install(
name: str,
specs: ListLike,
channels: ListLike = None,
yes: bool = False,
strict_channel_priority: bool = True,
) -> Environment:
"""Install conda packages into the environment."""
env = Environment.read(name=name)
cleaned = process_specs(specs)
CondaHandler(env=env).install(
packages=cleaned,
channels=channels,
yes=yes,
strict_channel_priority=strict_channel_priority,
)
if not yes:
jupyter_kernel_install_query(name=name, packages=cleaned)
_ask_user_to_sync(name=name, yes=yes)
return env
def test_pull_no_new_action_in_remote(setup_tests, local_packages, local_logs,
local_actions):
overwrite_mock = setup_tests["overwrite_mock"]
local_history = History(
name=ENV_NAME,
packages=HistoryPackages.create(local_packages),
channels=Channels(),
logs=Logs([log for log in local_logs]),
actions=Actions(local_actions),
debug=Debug(),
)
env = Environment(name=ENV_NAME, history=local_history)
pull(env=env)
assert env.history.packages == local_history.packages
assert env.history.logs == local_history.logs
assert env.history.actions == local_history.actions
overwrite_mock.assert_not_called()
def expected_diff(mocker, request):
"""Set up for diff function"""
packages = request.param["packages"]
conda_dependencies = request.param["conda_dependencies"]
local = request.param["local"]
name = "test-diff"
mocker.patch("conda_env_tracker.gateways.io.Path.mkdir")
mocker.patch("conda_env_tracker.gateways.io.Path.write_text")
dependencies = {"conda": conda_dependencies, "pip": {}}
mocker.patch("conda_env_tracker.env.get_dependencies",
mocker.Mock(return_value=dependencies))
history = History.create(
name=name,
packages=PackageRevision.create(packages=packages,
dependencies=dependencies),
channels=None,
logs=None,
actions=None,
debug=None,
diff=None,
)
env = Environment(name=name, history=history)
mocker.patch("conda_env_tracker.main.Environment.read", return_value=env)
mocker.patch(
"conda_env_tracker.gateways.io.EnvIO.get_environment",
return_value={
"name": "test-diff",
"channels": ["conda-forge"],
"dependencies": dependencies,
},
)
mocker.patch(
"conda_env_tracker.history.history.get_dependencies",
return_value={"conda": local},
)
yield request.param["expected"]
if (USER_ENVS_DIR / name).exists():
shutil.rmtree(USER_ENVS_DIR / name)
def test_infer_environment_with_spec_success(mocker):
env_name = "infer-test"
dependencies = {
"conda": {
"pandas": Package("pandas", "pandas", "0.23", "py_36")
}
}
mocker.patch("conda_env_tracker.history.get_pip_version",
mocker.Mock(return_value="18.1"))
mocker.patch("conda_env_tracker.env.get_all_existing_environment",
return_value=[env_name])
mocker.patch("conda_env_tracker.env.get_dependencies",
mocker.Mock(return_value=dependencies))
mocker.patch("conda_env_tracker.env.get_r_dependencies")
env = Environment.infer(
name=env_name,
packages=Packages.from_specs("pandas=0.23"),
channels=["conda-forge", "main"],
)
assert env.history.packages == {
"conda": {
"pandas": Package("pandas", "pandas=0.23", "0.23", "py_36")
}
}
assert env.history.channels == ["conda-forge", "main"]
assert env.history.logs == [
f"conda create --name {env_name} pandas=0.23 --override-channels --strict-channel-priority "
"--channel conda-forge "
"--channel main"
]
assert env.history.actions == [
f"conda create --name {env_name} pandas=0.23=py_36 "
"--override-channels --strict-channel-priority "
"--channel conda-forge "
"--channel main"
]
