def diff(name: str) -> ListLike:
"""Get the difference between history yaml and local conda environment"""
env = Environment.read(name=name)
env_reader = EnvIO(env_directory=USER_ENVS_DIR / name)
version_diff_pkges, new_pkges, missing_pkges = History.history_diff(
env_name=name, env=env, env_reader=env_reader)
return missing_pkges + version_diff_pkges + new_pkges
def merge_conflicting_changes(env: Environment):
"""Reconciles packages between local and remote"""
remote_dir = env.local_io.get_remote_dir()
remote_io = EnvIO(env_directory=remote_dir)
remote_history = remote_io.get_history()
local_history = env.history
update_conda_environment(env_dir=remote_dir)
if _r_env_needs_updating(local_history=local_history,
remote_history=remote_history):
update_r_environment(name=env.name, env_dir=remote_dir)
EnvIO.overwrite_local(local_io=env.local_io, remote_io=remote_io)
new_env = Environment(name=env.name, history=remote_history)
new_env.validate()
extra_logs = []
for log in local_history.logs:
if log not in set(remote_history.logs):
extra_logs.append(log)
for log in extra_logs:
new_env = _update_from_extra_log(env=new_env,
history=local_history,
log=log)
new_env.validate()
new_env.export()
env.history = new_env.history
logger.info("Successfully updated the environment.")
return new_env
def __init__(self, name: str, history: Optional[History] = None):
self.name = name
self.history = history
self.local_io = EnvIO(env_directory=USER_ENVS_DIR / name)
self.dependencies = {}
if self.history:
self.dependencies = get_dependencies(name=self.name)
if self.history.packages.get("r"):
self.dependencies["r"] = get_r_dependencies(name=self.name)
def setup_remote(
name: str,
remote_dir: Optional[PathLike] = None,
if_missing: bool = False,
yes: bool = False,
) -> None:
"""setup remote directory for push and pull"""
if not remote_dir:
remote_dir = infer_remote_dir(check_history_exists=False)
env_io = EnvIO(env_directory=USER_ENVS_DIR / name)
env_io.set_remote_dir(remote_dir=remote_dir, if_missing=if_missing, yes=yes)
def test_replace_existing_env_success(mocker):
mocker.patch("conda_env_tracker.env.delete_conda_environment")
mocker.patch("conda_env_tracker.env.prompt_yes_no").return_value = True
env_name = "conda_env_tracker-test-create-history"
create_cmd = f"conda create --name {env_name} pandas"
channels = ["conda-forge", "main"]
packages = Packages.from_specs("pandas")
action = "pandas=0.23=py_36"
mocker.patch(
"conda_env_tracker.env.get_all_existing_environment").return_value = [
env_name
]
mocker.patch("conda_env_tracker.env.conda_create",
mocker.Mock(return_value=create_cmd))
mocker.patch(
"conda_env_tracker.env.get_dependencies",
mocker.Mock(return_value={
"conda": {
"pandas": Package("pandas", "*", "0.23=py_36")
},
"pip": {},
}),
)
mocker.patch(
"conda_env_tracker.history.debug.get_pip_version",
mocker.Mock(return_value="18.1"),
)
mocker.patch("conda_env_tracker.env.get_conda_channels",
mocker.Mock(return_value=channels))
Environment.create(name=env_name, packages=packages)
writer = EnvIO(env_directory=USER_ENVS_DIR / env_name)
history = writer.get_history()
channel_string = "--override-channels --strict-channel-priority " + " ".join(
"--channel " + channel for channel in channels)
assert history.actions == [
f"conda create --name {env_name} {action} {channel_string}"
]
assert history.packages == {
"conda": {
"pandas": Package.from_spec("pandas")
}
}
assert history.channels == channels
assert history.logs == [create_cmd]
env_dir = Path(USER_ENVS_DIR) / env_name
shutil.rmtree(env_dir)
def get_env_name(infer: bool = False) -> str:
"""Get the name of the environment either from current conda environment, from remote directory,
or inferred from git repo.
"""
# pylint: disable=redefined-outer-name
if not infer:
return get_active_conda_env_name()
try:
remote_dir = infer_remote_dir()
history = EnvIO(env_directory=remote_dir).get_history()
return history.name
except CondaEnvTrackerHistoryNotFoundError as err:
raise CondaEnvTrackerHistoryNotFoundError(
f"Cannot infer name from history, often resolved by passing the name argument. Full error: {str(err)}"
)
def remove(self, yes=False) -> None:
"""Remove the environment and history."""
delete_conda_environment(name=self.name)
try:
remote_io = EnvIO(self.local_io.get_remote_dir())
except CondaEnvTrackerRemoteError:
remote_io = None
self.local_io.delete_all()
if remote_io and (
yes
or prompt_yes_no(
prompt_msg=f"Do you want to remove remote files in dir: {remote_io.env_dir}?"
)
):
remote_io.delete_all()
def setup_env(mocker, request):
"""Setup and teardown for tests with parameterized inputs. Even yes is passed in commands like create and install,
the option itself don't show up in actions and logs as it provide no additional value for reproducibility."""
env_dir = USER_ENVS_DIR / ENV_NAME
mocker.patch(
"conda_env_tracker.history.debug.get_pip_version",
mocker.Mock(return_value="18.1"),
)
mocker.patch("conda_env_tracker.env.get_all_existing_environment")
mocker.patch("conda_env_tracker.main._ask_user_to_sync")
mocker.patch("conda_env_tracker.gateways.conda.run_command")
initial_conda_packages = {
"pandas": Package("pandas", "pandas", "0.23", "py_36")
}
get_package_mock = mocker.patch(
"conda_env_tracker.env.get_dependencies",
mocker.Mock(return_value={
"conda": initial_conda_packages,
"pip": {}
}),
)
condarc_channels = ["conda-forge", "main"]
mocker.patch(
"conda_env_tracker.env.get_conda_channels",
mocker.Mock(return_value=condarc_channels),
)
if "channels" in request.param["input"]:
channels = request.param["input"]["channels"]
else:
channels = condarc_channels
id_mock = mocker.patch("conda_env_tracker.history.history.uuid4")
id_mock.return_value = "my_unique_id"
env = main.create(name=ENV_NAME, **request.param["input"])
env_io = EnvIO(env_directory=env_dir)
yield {
"channels": channels,
"env": env,
"env_io": env_io,
"expected": request.param["expected"],
"get_package_mock": get_package_mock,
"initial_conda_packages": initial_conda_packages,
"id": id_mock,
}
if env_dir.exists():
shutil.rmtree(env_dir)
def __init__(
self,
name: str,
history: Optional[History] = None,
dependencies: Optional[dict] = None,
):
self.name = name
self.history = history
self.local_io = EnvIO(env_directory=USER_ENVS_DIR / name)
if dependencies:
self.dependencies = dependencies
else:
self.dependencies = {}
if self.history:
self.dependencies = get_dependencies(name=self.name)
if self.history.packages.get("r"):
self.dependencies["r"] = get_r_dependencies(name=self.name)
if history:
self.history.packages.update_versions(dependencies=self.dependencies)
def pull(env: Environment, yes: bool = False) -> Environment:
"""Pull history from remote to local"""
try:
remote_dir = env.local_io.get_remote_dir()
except CondaEnvTrackerRemoteError as remote_err:
raise CondaEnvTrackerPullError(str(remote_err))
remote_io = EnvIO(env_directory=remote_dir)
remote_history = remote_io.get_history()
local_history = env.history
if _create_env_with_new_id(env=env, remote_history=remote_history,
yes=yes):
return replace_local_with_remote(
env=env,
remote_dir=remote_dir,
remote_io=remote_io,
remote_history=remote_history,
)
if _nothing_to_pull(local_history=local_history,
remote_history=remote_history):
logger.info("Nothing to pull.")
return env
if _local_needs_update(local_history=local_history,
remote_history=remote_history,
yes=yes):
return replace_local_with_remote(
env=env,
remote_dir=remote_dir,
remote_io=remote_io,
remote_history=remote_history,
)
if not yes and not prompt_yes_no(prompt_msg=(
"Remote and local have different packages, do you want to overwrite "
"with remote and append local")):
logger.info("Exiting without updating local environment.")
raise CondaEnvTrackerPullError(
"Conflicting Packages in remote and local; user elected not to update"
)
return merge_conflicting_changes(env=env)
def push(env: Environment) -> Environment:
"""Handle push to remote"""
remote_dir = env.local_io.get_remote_dir()
remote_io = EnvIO(env_directory=remote_dir)
remote_history = remote_io.get_history()
if remote_history == env.history:
logger.info("Nothing to push.")
return env
if remote_history and (not is_ordered_subset(
set=env.history.actions, subset=remote_history.actions) or
not is_ordered_subset(set=env.history.logs,
subset=remote_history.logs)):
raise CondaEnvTrackerPushError(
PUSH_ERROR_STR.format(remote_dir=remote_dir,
local_dir=USER_ENVS_DIR / env.name))
env.local_io.copy_environment(remote_dir)
logger.info(f"Successfully push {env.name} to {remote_dir}")
return env
def _remote_dir_is_set(name: str) -> bool:
"""Check if the remote directory has been setup for the cet environment."""
env_io = EnvIO(env_directory=USER_ENVS_DIR / name)
return env_io.is_remote_dir_set()
def pull(env: Environment, yes: bool = False) -> Environment:
"""Pull history from remote to local"""
remote_dir = env.local_io.get_remote_dir()
remote_io = EnvIO(env_directory=remote_dir)
remote_history = remote_io.get_history()
local_history = env.history
_check_for_errors(local_history=local_history, remote_history=remote_history)
if _nothing_to_pull(local_history=local_history, remote_history=remote_history):
logger.info("Nothing to pull.")
return env
if _local_needs_update(local_history=local_history, remote_history=remote_history):
update(
env=env,
remote_dir=remote_dir,
remote_io=remote_io,
remote_history=remote_history,
)
return env
if _actions_in_different_order(
local_history=local_history, remote_history=remote_history
):
if not yes and not prompt_yes_no(
prompt_msg="Remote environment has same packages but in different order, "
"Should we overwrite local with remote environment"
):
logger.info("Exiting without updating local environment.")
return env
update(
env=env,
remote_dir=remote_dir,
remote_io=remote_io,
remote_history=remote_history,
)
return env
if not yes and not prompt_yes_no(
prompt_msg=(
"Remote and local have different packages, do you want to overwrite "
"with remote and append local"
)
):
logger.info("Exiting without updating local environment.")
return env
update_conda_environment(env_dir=remote_dir)
if _r_env_needs_updating(
local_history=local_history, remote_history=remote_history
):
update_r_environment(name=env.name, env_dir=remote_dir)
EnvIO.overwrite_local(local_io=env.local_io, remote_io=remote_io)
new_env = Environment(name=env.name, history=remote_history)
new_env.validate()
extra_logs = []
for log in local_history.logs:
if log not in set(remote_history.logs):
extra_logs.append(log)
for log in extra_logs:
new_env = _update_from_extra_log(env=new_env, history=local_history, log=log)
new_env = _update_r_packages(
env=new_env, local_history=local_history, remote_history=remote_history
)
new_env.validate()
new_env.export()
env.history = new_env.history
logger.info("Successfully updated the environment.")
return new_env
def create(
cls,
name: str,
packages: Packages,
channels: ListLike = None,
yes: bool = False,
strict_channel_priority: bool = True,
):
"""Creating a conda environment from a list of packages."""
if name == "base":
raise CondaEnvTrackerCondaError(
"Environment can not be created using default name base"
)
if name in get_all_existing_environment():
message = (
f"This environment {name} already exists. Would you like to replace it"
)
if prompt_yes_no(prompt_msg=message, default=False):
delete_conda_environment(name=name)
local_io = EnvIO(env_directory=USER_ENVS_DIR / name)
if local_io.env_dir.exists():
local_io.delete_all()
else:
raise CondaEnvTrackerCondaError(f"Environment {name} already exists")
logger.debug(f"creating conda env {name}")
conda_create(
name=name,
packages=packages,
channels=channels,
yes=yes,
strict_channel_priority=strict_channel_priority,
)
create_cmd = get_conda_create_command(
name=name,
packages=packages,
channels=channels,
strict_channel_priority=strict_channel_priority,
)
specs = Actions.get_package_specs(
packages=packages, dependencies=get_dependencies(name=name)["conda"]
)
if not channels:
channels = get_conda_channels()
dependencies = get_dependencies(name=name)
history = History.create(
name=name,
channels=Channels(channels),
packages=PackageRevision.create(packages, dependencies=dependencies),
logs=Logs(create_cmd),
actions=Actions.create(
name=name,
specs=specs,
channels=Channels(channels),
strict_channel_priority=strict_channel_priority,
),
diff=Diff.create(packages=packages, dependencies=dependencies),
debug=Debug.create(name=name),
)
env = cls(name=name, history=history, dependencies=dependencies)
env.export()
return env
def read(cls, name: str):
"""read the environment from history file"""
reader = EnvIO(env_directory=USER_ENVS_DIR / name)
history = reader.get_history()
return cls(name=name, history=history)
def test_remote_end_to_end(end_to_end_setup, mocker):
"""Test setup, create, install, pull and push feature of conda_env_tracker"""
env = end_to_end_setup["env"]
remote_dir = end_to_end_setup["remote_dir"]
channels = end_to_end_setup["channels"]
channel_command = end_to_end_setup["channel_command"]
setup_remote(name=env.name, remote_dir=remote_dir)
local_io = EnvIO(env_directory=USER_ENVS_DIR / env.name)
assert str(local_io.get_remote_dir()) == str(remote_dir)
assert not list(remote_dir.glob("*"))
push(name=env.name)
remote_io = EnvIO(env_directory=remote_dir)
assert remote_io.get_history() == local_io.get_history()
assert remote_io.get_environment() == local_io.get_environment()
env = conda_install(name=env.name, specs=["pytest"], yes=True)
assert env.local_io.get_history() != remote_io.get_history()
env = push(name=env.name)
assert env.local_io.get_history() == remote_io.get_history()
log_mock = mocker.patch("conda_env_tracker.pull.logging.Logger.info")
env = pull(name=env.name)
log_mock.assert_called_once_with("Nothing to pull.")
remove_package_from_history(env, "pytest")
conda_dependencies = env.dependencies["conda"]
assert env.local_io.get_history() != remote_io.get_history()
assert env.history.packages == {
"conda": {
"python":
Package(
"python",
"python=3.6",
version=conda_dependencies["python"].version,
build=conda_dependencies["python"].build,
),
"colorama":
Package(
"colorama",
"colorama",
version=conda_dependencies["colorama"].version,
build=conda_dependencies["colorama"].build,
),
}
}
env = pull(name=env.name)
assert env.local_io.get_history() == remote_io.get_history()
assert remote_io.get_environment() == local_io.get_environment()
remove_package_from_history(env, "pytest")
assert env.local_io.get_history() != remote_io.get_history()
env = conda_install(name=env.name,
specs=["pytest-cov"],
channels=("main", ),
yes=True)
env = pull(name=env.name, yes=True)
conda_dependencies = env.dependencies["conda"]
assert env.history.packages == {
"conda": {
"python":
Package(
"python",
"python=3.6",
version=conda_dependencies["python"].version,
build=conda_dependencies["python"].build,
),
"colorama":
Package(
"colorama",
"colorama",
version=conda_dependencies["colorama"].version,
build=conda_dependencies["colorama"].build,
),
"pytest":
Package(
"pytest",
"pytest",
version=conda_dependencies["pytest"].version,
build=conda_dependencies["pytest"].build,
),
"pytest-cov":
Package(
"pytest-cov",
"pytest-cov",
version=conda_dependencies["pytest-cov"].version,
build=conda_dependencies["pytest-cov"].build,
),
}
}
log_list = [
f"conda create --name {env.name} python=3.6 colorama {channel_command}",
f"conda install --name {env.name} pytest",
f"conda install --name {env.name} pytest-cov --channel main",
]
assert env.history.logs == log_list
actual_env = (env.local_io.env_dir / "environment.yml").read_text()
conda_dependencies = get_dependencies(name=env.name)["conda"]
expected_env = [f"name: {env.name}", "channels:"]
for channel in channels + ["nodefaults"]:
expected_env.append(f" - {channel}")
expected_env = ("\n".join(expected_env + [
"dependencies:",
" - python=" + conda_dependencies["python"].version,
" - colorama=" + conda_dependencies["colorama"].version,
" - pytest=" + conda_dependencies["pytest"].version,
" - pytest-cov=" + conda_dependencies["pytest-cov"].version,
]) + "\n")
assert actual_env == expected_env
expected_debug = 3 * [{
"platform": get_platform_name(),
"conda_version": CONDA_VERSION,
"pip_version": get_pip_version(name=env.name),
"timestamp": str(date.today()),
}]
for i in range(len(env.history.debug)):
for key, val in expected_debug[i].items():
if key == "timestamp":
assert env.history.debug[i][key].startswith(val)
else:
assert env.history.debug[i][key] == val