def update_history_custom_urls(self, package: Package):
"""Update history for pip install with custom urls"""
self.env.update_dependencies()
self.env.history.packages.update_packages(Packages(package),
source="pip")
self.env.validate_installed_packages(Packages(package))
log = get_pip_custom_install_command(spec=package.spec)
self.env.history.append(log=log, action=log)
def infer(cls, name: str, packages: Packages, channels: ListLike = None):
"""create conda_env_tracker environment by inferring to existing conda environment"""
if name == "base":
raise CondaEnvTrackerCondaError(
"Environment can not be created using default name base"
)
if name not in get_all_existing_environment():
raise CondaEnvTrackerCondaError(
f"Environment {name} can not be inferred, does not exist"
)
existing_packages = get_dependencies(name=name)
if "r-base" in existing_packages["conda"]:
existing_r_packages = get_r_dependencies(name=name)
else:
existing_r_packages = {}
user_packages = {"conda": Packages(), "pip": Packages(), "r": Packages()}
for package in packages:
if package.name in existing_packages.get("conda", Packages()):
user_packages["conda"].append(package)
elif package.name in existing_packages.get("pip", Packages()):
user_packages["pip"].append(package)
elif package.name in existing_r_packages:
raise RError(
"Cannot infer R packages, must run follow-up R install command.\n"
f"Found '{package.name}' in installed R packages {existing_r_packages}."
)
else:
raise CondaEnvTrackerCondaError(
f"Environment {name} does not have {package.spec} installed"
)
conda_create_cmd = get_conda_create_command(
name, user_packages["conda"], channels
)
specs = Actions.get_package_specs(
packages=user_packages["conda"], dependencies=existing_packages["conda"]
)
history = History(
name=name,
channels=Channels(channels),
packages=HistoryPackages.create(user_packages["conda"]),
logs=Logs.create(conda_create_cmd),
actions=Actions.create(name=name, specs=specs, channels=Channels(channels)),
debug=Debug.create(name=name),
)
env = cls(name=name, history=history)
if user_packages["pip"]:
handler = PipHandler(env=env)
handler.update_history_install(packages=user_packages["pip"])
env = handler.env
env.export()
return env
def test_custom_r_install(setup_env, mocker):
command = 'install.packages("h2o")'
env = setup_env["env"]
env_io = setup_env["env_io"]
expected = copy.deepcopy(setup_env["expected"])
run_mock = mocker.patch("conda_env_tracker.gateways.r.run_command")
run_mock.configure_mock(**{
"return_value.failed": False,
"return_value.stderr": ""
})
mocker.patch(
"conda_env_tracker.env.get_r_dependencies",
mocker.Mock(return_value={"h2o": "3.24.0.3"}),
)
h2o = Package("h2o", command)
packages = Packages([h2o])
RHandler(env=env).install(packages=packages)
expected["logs"].append(f"R --quiet --vanilla -e '{command}'")
expected["packages"]["r"] = {"h2o": h2o}
actual = env_io.get_history()
assert actual.logs == expected["logs"]
assert actual.packages == expected["packages"]
assert actual.actions[-1] == expected["logs"][-1]
actual_install_r = (env_io.env_dir / "install.R").read_text()
assert actual_install_r == command
def test_r_install_double_quotes(setup_env, quote_mark, mocker):
double_quote_command = "install.packages('h2o')"
expected_command = double_quote_command.replace("'", r"\"")
command = double_quote_command.replace("'", quote_mark)
env = setup_env["env"]
env_io = setup_env["env_io"]
expected = copy.deepcopy(setup_env["expected"])
run_mock = mocker.patch("conda_env_tracker.gateways.r.run_command")
run_mock.configure_mock(**{
"return_value.failed": False,
"return_value.stderr": ""
})
mocker.patch(
"conda_env_tracker.env.get_r_dependencies",
mocker.Mock(return_value={"h2o": Package("h2o", "h2o", "3.24.0.3")}),
)
h2o = Package("h2o", command)
packages = Packages([h2o])
RHandler(env=env).install(packages=packages)
expected["logs"].append(f'R --quiet --vanilla -e "{expected_command}"')
expected["packages"]["r"] = {"h2o": h2o}
actual = env_io.get_history()
assert actual.logs == expected["logs"]
assert actual.packages == expected["packages"]
assert actual.actions[-1] == expected["logs"][-1]
actual_install_r = (env_io.env_dir / "install.R").read_text()
assert actual_install_r == command
def test_two_same_package_r_install(setup_env, mocker):
"""Test that the second command replaces the first command (and updates the version of the package)"""
command_h2o_1 = 'install.packages("h2o")'
escaped_command_1 = command_h2o_1.replace('"', r"\"")
command_h2o_2 = 'install.packages("h2o", type="source", repos=(c("http://h2o-release.s3.amazonaws.com/h2o/latest_stable_R")))'
escaped_command_2 = command_h2o_2.replace('"', r"\"")
env = setup_env["env"]
env_io = setup_env["env_io"]
expected = copy.deepcopy(setup_env["expected"])
run_mock = mocker.patch("conda_env_tracker.gateways.r.run_command")
run_mock.configure_mock(**{
"return_value.failed": False,
"return_value.stderr": ""
})
mocker.patch(
"conda_env_tracker.env.get_r_dependencies",
side_effect=[
{
"h2o": Package("h2o", "h2o", "3.4.0.1")
},
{
"h2o": Package("h2o", "h2o", "3.24.0.5")
},
],
)
h2o_1 = Package("h2o", command_h2o_1)
h2o_2 = Package("h2o", command_h2o_2)
packages_1 = Packages([h2o_1])
packages_2 = Packages([h2o_2])
RHandler(env=env).install(packages=packages_1)
RHandler(env=env).install(packages=packages_2)
expected["logs"].append(f'R --quiet --vanilla -e "{escaped_command_1}"')
expected["logs"].append(f'R --quiet --vanilla -e "{escaped_command_2}"')
expected["packages"]["r"] = {"h2o": h2o_2}
actual = env_io.get_history()
assert actual.logs == expected["logs"]
assert actual.packages == expected["packages"]
assert actual.actions[-1] == expected["logs"][-1]
actual_install_r = (env_io.env_dir / "install.R").read_text()
assert actual_install_r == command_h2o_2
def test_pull_different_actions_in_both_r(setup_r_tests):
remote_history = setup_r_tests["remote_history"]
get_r_dependencies = setup_r_tests["get_r_dependencies"]
get_r_dependencies.configure_mock(
**{
"return_value": {
"praise": Package("praise", "praise", "1.0.0"),
"jsonlite": Package("jsonlite", "jsonlite", "1.6"),
}
})
local_history = copy.deepcopy(remote_history)
local_history.packages.update_packages(packages=Packages(
Package("praise", 'install.packages("praise")')),
source="r")
local_history.logs.append(f"{R_COMMAND} -e 'install.packages(\"praise\")'")
local_history.actions.append(
f"{R_COMMAND} -e 'install.packages(\"praise\")'")
env = Environment(name=local_history.name, history=local_history)
remote_history.packages.update_packages(
packages=Packages(Package("jsonlite", 'install.packages("jsonlite")')),
source="r",
)
remote_history.logs.append(
f"{R_COMMAND} -e 'install.packages(\"jsonlite\")'")
remote_history.actions.append(
f"{R_COMMAND} -e 'install.packages(\"jsonlite\")'")
expected_history = copy.deepcopy(remote_history)
expected_history.packages.update_packages(packages=Packages(
Package("praise", 'install.packages("praise")')),
source="r")
expected_history.logs.append(
f"{R_COMMAND} -e 'install.packages(\"praise\")'")
expected_history.actions.append(
f"{R_COMMAND} -e 'install.packages(\"praise\")'")
final_env = pull(env=env)
assert final_env.history.packages == expected_history.packages
assert final_env.history.logs == expected_history.logs
assert final_env.history.actions == expected_history.actions
def test_export_install_r_single_package():
packages = Packages(
Package(
"jsonlite",
'library("devtools"); install_version("jsonlite",version="1.6")'))
actual = r.export_install_r(packages)
expected = 'library("devtools"); install_version("jsonlite",version="1.6")'
assert actual == expected
def process_r_specs(package_names: ListLike, commands: ListLike) -> Packages:
"""Generate the install commands for R packages and return as the package spec."""
if len(package_names) != len(commands):
raise RError(
f"Must have same number of package names ({len(package_names)}) and install commands ({len(commands)})."
)
packages = Packages()
for package_name, command in zip(package_names, commands):
packages.append(Package(package_name, spec=command))
return packages
def test_two_custom_r_install(setup_env, mocker):
"""Test that the second custom command replaces the first command (and updates the version of the package)"""
custom_command_h2o = 'install.packages("h2o")'
custom_command_trelliscopejs = (
'library("devtools"); install_github("hafen/trelliscopejs")')
env = setup_env["env"]
env_io = setup_env["env_io"]
expected = copy.deepcopy(setup_env["expected"])
run_mock = mocker.patch("conda_env_tracker.gateways.r.run_command")
run_mock.configure_mock(**{
"return_value.failed": False,
"return_value.stderr": ""
})
mocker.patch(
"conda_env_tracker.env.get_r_dependencies",
return_value={
"h2o": Package("h2o", "h2o", "3.4.0.1"),
"trelliscopejs": Package("trelliscopejs", "trelliscopejs",
"0.1.7"),
},
)
h2o = Package("h2o", custom_command_h2o)
trelliscopejs = Package("trelliscopejs", custom_command_trelliscopejs)
RHandler(env=env).install(packages=Packages([h2o]))
RHandler(env=env).install(packages=Packages([trelliscopejs]))
expected["logs"].append(f"R --quiet --vanilla -e '{custom_command_h2o}'")
expected["logs"].append(
f"R --quiet --vanilla -e '{custom_command_trelliscopejs}'")
expected["packages"]["r"] = {"h2o": h2o, "trelliscopejs": trelliscopejs}
actual = env_io.get_history()
assert actual.logs == expected["logs"]
assert actual.packages == expected["packages"]
assert actual.actions[-1] == expected["logs"][-1]
actual_install_r = (env_io.env_dir / "install.R").read_text()
expected_install_r = "\n".join(
[custom_command_h2o, custom_command_trelliscopejs])
assert actual_install_r == expected_install_r
def _install_missing_r_packages(
env: Environment, local_history: History, remote_history: History
) -> Environment:
local_r_packages = local_history.packages.get("r", {})
remote_r_packages = remote_history.packages.get("r", {})
handler = RHandler(env=env)
packages_to_install = Packages()
packages_to_add_to_history = Packages()
for package_name, package in local_r_packages.items():
if package_name not in remote_r_packages:
if package_name in env.dependencies.get("r", {}):
packages_to_add_to_history.append(package)
else:
packages_to_install.append(package)
if packages_to_install:
handler.install(packages_to_install)
if packages_to_add_to_history:
handler.update_history_install(packages_to_add_to_history)
return env
def _extract_packages(self, index: int, packages: Packages) -> Packages:
"""Extracting conda and pip packages"""
log = self[index]
extracted_packages = Packages()
for package in packages:
package_expression = re.compile(
f"(({package.name})(=[a-z0-9_=.]+)?)")
spec = package_expression.search(log).group(0)
extracted_packages.append_spec(spec)
return extracted_packages
def _export_install_r_if_necessary(self) -> None:
"""Export an install.R file that can be used to install the same R packages and versions."""
if self.history.packages.get("r"):
install_r = export_install_r(
packages=Packages(
[package for package in self.history.packages["r"].values()]
)
)
self.local_io.export_install_r(install_r)
else:
self.local_io.delete_install_r()
def validate(self) -> None:
"""Check that all packages are installed correctly."""
if self.history.packages.get("r"):
self.update_dependencies(update_r_dependencies=True)
else:
self.update_dependencies()
packages = Packages()
for source in self.sources:
packages += [
package for package in self.history.packages.get(source, {}).values()
]
self.validate_packages(packages)
def test_r_install_error(mocker):
run_mock = mocker.patch("conda_env_tracker.gateways.r.run_command")
run_mock.configure_mock(**{
"return_value.stderr": "stderr",
"return_value.failed": True
})
expected_command = "install.packages('jsonlite')"
with pytest.raises(errors.RError) as err:
r.r_install(name="env_name",
packages=Packages(Package("jsonlite", expected_command)))
assert str(err.value) == (
"Error installing R packages:\nstderr\n"
f"environment='env_name' and command='{expected_command}'.")
def parse(cls, history_section: dict):
"""Parse the history file and create the packages."""
packages_instance = cls()
for source, packages in history_section.items():
source_packages = Packages()
for name, spec in packages.items():
if spec == "*":
source_packages.append_spec(name)
else:
source_packages.append(Package(name, spec))
packages_instance.update_packages(packages=source_packages,
source=source)
return packages_instance
def clean_r_specs(package_names: ListLike, commands: ListLike) -> Packages:
"""Cleaning all R package specs and converting to a package class."""
packages = Packages()
for package_name, command in zip_longest(package_names, commands):
if not package_name or not command:
raise RError(
(
"Must have same number of R package names and install commands.\n"
f"Package names: {package_names}\n"
f"and install commands: {commands}"
)
)
packages.append(Package(package_name, command))
return packages
def test_remove_r_package(setup_env, mocker):
env = setup_env["env"]
env_io = setup_env["env_io"]
expected = copy.deepcopy(setup_env["expected"])
r_command = 'install.packages("h2o")'
escaped_command = r_command.replace('"', r"\"")
run_mock = mocker.patch("conda_env_tracker.gateways.r.run_command")
run_mock.configure_mock(**{
"return_value.failed": False,
"return_value.stderr": ""
})
mocker.patch(
"conda_env_tracker.env.get_r_dependencies",
mocker.Mock(return_value={"h2o": Package("h2o", "h2o", "3.24.0.3")}),
)
handler = RHandler(env=env)
h2o = Package("h2o", r_command)
packages = Packages([h2o])
handler.install(packages=packages)
install_log = f'R --quiet --vanilla -e "{escaped_command}"'
expected["logs"].append(install_log)
expected["packages"]["r"] = {"h2o": h2o}
actual = env_io.get_history()
assert actual.logs == expected["logs"]
assert actual.packages == expected["packages"]
assert actual.actions[-1] == install_log
expected_install_r = r_command
actual_install_r = (env_io.env_dir / "install.R").read_text()
assert actual_install_r == expected_install_r
mocker.patch("conda_env_tracker.r.r_remove")
handler.remove(packages=packages)
actual = env_io.get_history()
command = r"remove.packages(c(\"h2o\"))"
expected["packages"].pop("r")
expected["logs"].append(f'R --quiet --vanilla -e "{command}"')
assert actual.logs == expected["logs"]
assert actual.packages == expected["packages"]
assert actual.actions[-1] == expected["logs"][-1]
assert not (env_io.env_dir / "install.R").exists()
def clean_specs(specs: ListLike, check_custom: bool = False) -> Packages:
"""Cleaning all package specs and converting to a package class.
All package names for both pip and conda can be used with lowercase only. Conda automatically converts to lowercase.
This allows our internal dictionaries that use package names as keys to be consistent with `conda list`.
"""
cleaned = Packages()
for spec in specs:
if check_custom and "/" in spec:
raise CondaEnvTrackerPackageNameError(
f"Found illegal character in package name or spec: '{spec}'.\n"
"Maybe you want to use --custom which requires package name and custom url, e.g.\n"
f"'cet pip install package_name --custom package_url'"
)
cleaned.append_spec(spec.lower())
return cleaned
def test_pull_r(setup_r_tests):
remote_history = setup_r_tests["remote_history"]
remote_history.packages.update_packages(
packages=Packages(Package("jsonlite", "install.packages('jsonlite')")),
source="r",
)
remote_history.logs.append(
f"{R_COMMAND} -e 'install.packages('jsonlite')'")
remote_history.actions.append(
f"{R_COMMAND} -e 'install.packages('jsonlite')'")
env = Environment(name="test", history=None)
final_env = pull(env=env)
assert final_env.history == remote_history
def test_export_install_r_multiple_installs():
packages = Packages([
Package(
"jsonlite",
'library("devtools"); install_version("jsonlite",version="1.6")',
),
Package(
"praise",
'library("devtools"); install_version("praise",version="1.0.0")',
),
])
actual = r.export_install_r(packages)
expected = "\n".join([
'library("devtools"); install_version("jsonlite",version="1.6")',
'library("devtools"); install_version("praise",version="1.0.0")',
])
assert actual == expected
def _handle_r_extra_log(env: Environment, history: History,
index: int) -> Environment:
log = history.logs[index]
if "remove.packages(" in log:
packages = history.logs.extra_removed_packages(index=index)
RHandler(env=env).remove(packages=packages)
else:
package_names = [
package_name
for package_name in history.revisions.diffs[index]["r"]["upsert"]
]
packages = Packages()
for package_name in package_names:
packages.append(
history.revisions.packages[index]["r"][package_name])
RHandler(env=env).install(packages=packages)
return env
def update_histories(history_1, history_2):
add_log_1 = f"{R_COMMAND} -e 'install.packages(\"praise\")'"
remove_log_1 = f"{R_COMMAND} -e 'remove.packages(c(\"praise\"))'"
history_1.logs.append(add_log_1)
history_1.logs.append(remove_log_1)
history_1.actions.append(add_log_1)
history_1.actions.append(remove_log_1)
packages_2 = Packages(
Package("jsonlite", 'install.packages("jsonlite")'))
log_2 = f"{R_COMMAND} -e 'install.packages(\"jsonlite\")'"
action_2 = f"{R_COMMAND} -e 'install.packages(\"jsonlite\")'"
history_2.packages.update_packages(packages=packages_2, source="r")
history_2.logs.append(log_2)
history_2.actions.append(action_2)
return history_1, history_2
def parse(cls, history_section: dict):
"""Parse the history file and create the packages."""
packages_instance = cls()
for source, packages in history_section.items():
source_packages = Packages()
for name, spec in packages.items():
if spec == "*":
source_packages.append_spec(name)
elif spec[0].isdigit() and all(
letter.isalnum() or letter in [".", "=", "_"]
for letter in spec):
source_packages.append_spec(name + cls.separators[source] +
spec)
else:
source_packages.append(Package(name, spec))
packages_instance.update_packages(packages=source_packages,
source=source)
return packages_instance
def extract_r_packages(self, index: int) -> Optional[Packages]:
"""Extract R packages with versions (if the version was specified in the log)."""
if self[index].startswith(R_COMMAND):
r_packages = Packages()
install_commands = self[index].split(";")[1:]
for command in install_commands:
start = command.index('"') + 1
end = command.index('"', start)
name = command[start:end]
if "version" in command:
start = command.index('"', end + 1) + 1
end = command.index('"', start)
version = command[start:end]
r_packages.append_spec(f"{name}={version}")
else:
r_packages.append_spec(name)
return r_packages
return None
def test_get_packages_r(mocker):
mocker.patch(
"conda_env_tracker.env.get_dependencies",
return_value={
"conda": {
"r-base":
Package("r-base", "r-base", "3.5.1", "h539"),
"r-devtools":
Package("r-devtools", "r-devtools", "0.1.0", "r351_0"),
}
},
)
mocker.patch(
"conda_env_tracker.env.get_r_dependencies",
return_value={
"trelliscopejs": Package("trelliscopejs", "trelliscopejs"
"0.1.0")
},
)
local_packages = HistoryPackages.create(
Packages(
(Package.from_spec("r-base"), Package.from_spec("r-devtools"))))
local_packages.update_packages(
packages=(Package.from_spec("trelliscopejs"), ), source="r")
local_history = History(
name=ENV_NAME,
packages=local_packages,
channels=Channels("defaults"),
logs=Logs([]),
actions=Actions(),
debug=Debug(),
)
env = Environment(name=ENV_NAME, history=local_history)
expected = {
"conda": [
Package("r-base", "r-base", "3.5.1", "h539"),
Package("r-devtools", "r-devtools", "0.1.0", "r351_0"),
],
"r": [Package("trelliscopejs", "trelliscopejs", "0.1.0")],
}
actual = get_packages(env)
assert actual == expected
def extract_r_packages(self, index: int) -> (Packages, list):
"""Get the package and date from an R action."""
action = self[index]
packages = Packages()
dates = []
install_commands = action.split(";")[1:]
for command in install_commands:
start = command.index('"') + 1
end = command.index('"', start)
name = command[start:end]
start = command.index('"', end + 1) + 1
end = command.index('"', start)
version = command[start:end]
packages.append_spec(f"{name}={version}")
start = command.index('"', end + 1) + 1
end = command.index('"', start)
date = command[start:end]
dates.append(date)
return packages, dates
def extract_r_packages(self, index: int) -> Optional[Packages]:
"""Extract R packages with versions (if the version was specified in the log)."""
if self[index].startswith(R_COMMAND):
r_packages = Packages()
quote = '"'
package_names = self._get_r_package_names(self[index])
start = self[index].index(quote) + len(quote)
r_commands = self[index]
for package_name in package_names:
i_name = r_commands.index(package_name)
try:
end = r_commands.index(";", i_name)
except ValueError:
end = r_commands.rindex(quote, i_name)
spec = r_commands[start:end].strip()
start = end + 1
r_packages.append(
Package(package_name, spec.replace(r"\"", '"')))
return r_packages
return None
def test_update_all(setup_env, package, mocker):
env = setup_env["env"]
env_io = setup_env["env_io"]
expected = setup_env["expected"]
get_package_mock = setup_env["get_package_mock"]
get_package_mock.configure_mock(return_value={
"conda": {
"pandas": Package("pandas", "pandas", "0.24", "py_36")
}
})
mocker.patch("conda_env_tracker.conda.conda_update_all")
packages = Packages(package)
CondaHandler(env=env).update_all(packages=packages)
history = env_io.get_history()
channel_string = "--override-channels --strict-channel-priority " + " ".join(
[f"--channel " + channel for channel in expected["channels"]])
expected_log = expected["logs"].copy()
expected_log.append(f"conda update --all --name {env.name} {package.spec}")
action_packages_cmd = " pandas=0.24=py_36"
expected_action = expected["actions"].copy()
expected_action.append(
f"conda update --all --name {env.name}{action_packages_cmd} {channel_string}"
)
expected_packages = {
"conda": {
"pandas": Package("pandas", package.spec, "0.24", "py_36")
}
}
assert history.packages == expected_packages
assert history.logs == expected_log
assert history.actions == expected_action
def test_export_install_r_no_r_packages():
actual = r.export_install_r(Packages())
expected = ""
assert actual == expected