def _setup(self):
log.info("Creating oggm conda environment for {0}".format(self.name))
env_file = tempfile.NamedTemporaryFile(mode="w", delete=False,
suffix=".yml")
try:
pyver = str(self._python).replace(".", "")[:2]
oggm_env = OGGM_CONDA_ENVS[pyver]
req = requests.get(OGGM_CONDA_ENV_URL.format(oggm_env))
req.raise_for_status()
for line in req.text.splitlines():
if line.startswith("prefix:"):
continue
elif line.startswith("name:"):
env_file.write("name: {0}\n".format(self.name))
else:
env_file.write(line + "\n")
env_file.close()
self._conda_channels = ["conda-forge", "defaults"]
self._conda_environment_file = env_file.name
return super()._setup()
except Exception as exc:
if os.path.isfile(env_file.name):
with open(env_file.name, "r") as f:
text = f.read()
log.info("oggm conda env create failed: in {} with:\n{}"
.format(self._path, text))
raise
finally:
os.unlink(env_file.name)
def _nox_prep_env(self, setup: bool = False) -> None:
message = f"Running Nox environment update for: {self.name}"
log.info(message)
build_root_path = Path(self._build_root)
env_path = Path(self._path)
def copy_asv_files(src_parent: Path, dst_parent: Path) -> None:
"""For copying between self._path and a temporary cache."""
asv_files = list(src_parent.glob("asv*"))
# build_root_path.name usually == "project" .
asv_files += [src_parent / build_root_path.name]
for src_path in asv_files:
dst_path = dst_parent / src_path.name
if not dst_path.exists():
# Only cache-ing in case Nox has rebuilt the env @
# self._path. If the dst_path already exists: rebuilding
# hasn't happened. Also a non-issue when copying in the
# reverse direction because the cache dir is temporary.
if src_path.is_dir():
func = copytree
else:
func = copy2
func(src_path, dst_path)
with TemporaryDirectory(prefix="nox_asv_cache_") as asv_cache:
asv_cache_path = Path(asv_cache)
if setup:
noxfile = self.setup_noxfile
else:
# Cache all of ASV's files as Nox may remove and re-build the environment.
copy_asv_files(env_path, asv_cache_path)
# Get location of noxfile in cache.
noxfile_original = (build_root_path / self._repo_subdir /
self.noxfile_rel_path)
noxfile_subpath = noxfile_original.relative_to(
build_root_path.parent)
noxfile = asv_cache_path / noxfile_subpath
nox_cmd = [
"nox",
f"--noxfile={noxfile}",
# Place the env in the ASV env directory, instead of the default.
f"--envdir={env_path.parent}",
f"--session={self.nox_session_name}",
f"--python={self._python}",
"--install-only",
"--no-error-on-external-run",
"--verbose",
]
_ = asv_util.check_output(nox_cmd)
if not env_path.is_dir():
message = f"Expected Nox environment not found: {env_path}"
log.error(message)
raise RuntimeError(message)
if not setup:
# Restore ASV's files from the cache (if necessary).
copy_asv_files(asv_cache_path, env_path)
def _uninstall_project(self):
installed_hash = self._get_installed_commit_hash()
if installed_hash:
log.info(f"Clearing conda environment for {self.name}")
self._cache._remove_cache_dir(installed_hash)
# we can only run the uninstall command if an environment has already
# been made before, otherwise there is no python to use to uninstall
super()._uninstall_project()
def _uninstall_project(self):
self._set_installed_commit_hash(None)
log.info('Uninstalling from {}'.format(self.name))
if not self._uninstall_command:
log.info('Nothing to uninstall')
return
for cmd in self._uninstall_command:
self._run(cmd,
shell=True,
cwd=self._env_dir,
check=True,
timeout=self._install_timeout)
def _install_project(self, repo, commit_hash, build_dir):
commit_name = repo.get_decorated_hash(commit_hash, 8)
log.info('Installing {} into {}'.format(commit_name, self.name))
if not self._install_command:
log.info('Nothing to install')
return
for cmd in self._install_command:
self._run(cmd,
shell=True,
cwd=build_dir,
check=True,
timeout=self._install_timeout)
def test_log_indent(capsys):
log.set_nitems(0)
log.info("First\nSecond")
out, err = capsys.readouterr()
lines = out.lstrip().splitlines()
assert lines[0].index('First') == lines[1].index('Second')
log.set_nitems(1)
log.info("First\nSecond")
out, err = capsys.readouterr()
lines = out.lstrip().splitlines()
assert lines[0].index('First') == lines[1].index('Second')
def test_log_indent(capsys):
log.set_nitems(0)
log.info("First\nSecond")
out, err = capsys.readouterr()
lines = out.lstrip().splitlines()
assert lines[0].index('First') == lines[1].index('Second')
log.set_nitems(1)
log.info("First\nSecond")
out, err = capsys.readouterr()
lines = out.lstrip().splitlines()
assert lines[0].index('First') == lines[1].index('Second')
def _setup(self):
log.info("Creating oggm conda environment for {0}".format(self.name))
try:
conda = _find_conda()
except IOError as e:
raise util.UserError(str(e))
env_file = tempfile.NamedTemporaryFile(mode="w",
delete=False,
suffix=".yml")
try:
pyver = str(self._python).replace(".", "")[:2]
oggm_env = OGGM_CONDA_ENVS[pyver]
req = requests.get(OGGM_CONDA_ENV_URL.format(oggm_env))
req.raise_for_status()
env_text = req.text
for line in env_text.splitlines():
if line.startswith("prefix:") or self._has_requirement(line):
continue
elif line.startswith("name:"):
env_file.write("name: {0}\n".format(self.name))
else:
env_file.write(line + "\n")
conda_args, pip_args = self._get_requirements(conda)
env_file.writelines((' - %s\n' % s for s in conda_args))
if pip_args:
env_file.write(' - pip:\n')
env_file.writelines((' - %s\n' % s for s in pip_args))
env_file.close()
util.check_output([conda] + [
'env', 'create', '-f', env_file.name, '-p', self._path,
'--force'
])
except Exception as exc:
if os.path.isfile(env_file.name):
with open(env_file.name, "r") as f:
text = f.read()
log.info(
"oggm conda env create failed: in {} with:\n{}".format(
self._path, text))
raise
finally:
os.unlink(env_file.name)
def _build_project(self, repo, commit_hash, build_dir):
# at "build" time, we build the environment from the provided lockfile
self.run_executable(
_find_conda(),
[
"install",
"-y",
"-p",
self._path,
"--file",
f"{build_dir}/{self._lockfile_path}",
],
)
log.info(
f"Environment {self.name} updated to spec at {commit_hash[:8]}"
)
log.debug(
self.run_executable(_find_conda(), ["list", "-p", self._path])
)
return super()._build_project(repo, commit_hash, build_dir)
def _setup(self) -> None:
"""Used for initial environment creation - mimics parent method where possible."""
try:
self.conda = _find_conda()
except IOError as e:
raise asv_util.UserError(str(e))
if find_spec("nox") is None:
raise asv_util.UserError("Module not found: nox")
message = f"Creating Nox-Conda environment for {self.name} ."
log.info(message)
try:
self._nox_prep_env(setup=True)
except Exception:
raise
finally:
# No longer need the setup checkout now that the environment has been built.
self.project_temp_checkout.cleanup()
# Create an environment.yml file from the requirements in asv.conf.json.
# No default dependencies to specify - unlike parent - because Nox
# includes these by default.
conda_args, pip_args = self._get_requirements(self.conda)
if conda_args or pip_args:
with self._extra_reqs_path.open("w") as req_file:
req_file.write(f"name: {self.name}\n")
req_file.write("channels:\n")
req_file.writelines(
[f" - {channel}\n" for channel in self._conda_channels])
req_file.write("dependencies:\n")
# Categorise and write dependencies based on pip vs. conda.
req_file.writelines(
[f" - {package}\n" for package in conda_args])
if pip_args:
# And now specify the packages that are to be installed in the
# pip subsection.
req_file.write(" - pip:\n")
req_file.writelines(
[f" - {package}\n" for package in pip_args])
def _setup(self) -> None:
"""Used for initial environment creation - mimics parent method where possible."""
try:
self.conda = _find_conda()
except IOError as e:
raise asv_util.UserError(str(e))
if find_spec("nox") is None:
raise asv_util.UserError("Module not found: nox")
message = f"Creating Nox-Conda environment for {self.name} ."
log.info(message)
try:
self._nox_prep_env(setup=True)
finally:
# No longer need the setup checkout now that the environment has been built.
self.project_temp_checkout.cleanup()
conda_args, pip_args = self._get_requirements(self.conda)
if conda_args or pip_args:
message = (
"Ignoring user input package requirements. Benchmark "
"environment management is exclusively performed by Nox.")
log.warning(message)
def _setup(self):
# create the shell of a conda environment, that includes no packages
log.info("Creating conda environment for {0}".format(self.name))
self.run_executable(
_find_conda(), ["create", "-y", "-p", self._path, "--force"]
)
def _nox_prep_env(self, setup: bool = False) -> None:
message = f"Running Nox environment update for: {self.name}"
log.info(message)
build_root_path = Path(self._build_root)
env_path = Path(self._path)
def copy_asv_files(src_parent: Path, dst_parent: Path) -> None:
"""For copying between self._path and a temporary cache."""
asv_files = list(src_parent.glob("asv*"))
# build_root_path.name usually == "project" .
asv_files += [src_parent / build_root_path.name]
for src_path in asv_files:
dst_path = dst_parent / src_path.name
if not dst_path.exists():
# Only cache-ing in case Nox has rebuilt the env @
# self._path. If the dst_path already exists: rebuilding
# hasn't happened. Also a non-issue when copying in the
# reverse direction because the cache dir is temporary.
if src_path.is_dir():
func = copytree
else:
func = copy2
func(src_path, dst_path)
with TemporaryDirectory(prefix="nox_asv_cache_") as asv_cache:
asv_cache_path = Path(asv_cache)
if setup:
noxfile_path = self.setup_noxfile
else:
# Cache all of ASV's files as Nox may remove and re-build the environment.
copy_asv_files(env_path, asv_cache_path)
# Get location of noxfile in cache.
noxfile_path_build = (build_root_path / self._repo_subdir /
NOXFILE_REL_PATH)
noxfile_path = asv_cache_path / noxfile_path_build.relative_to(
build_root_path.parent)
nox_cmd = [
"nox",
f"--noxfile={noxfile_path}",
f"--envdir={env_path.parent}",
f"--session={SESSION_NAME}",
f"--python={self._python}",
"--install-only",
"--no-error-on-external-run",
"--verbose",
]
_ = asv_util.check_output(nox_cmd)
if not env_path.is_dir():
message = f"Expected Nox environment not found: {env_path}"
log.error(message)
if not setup:
# Restore ASV's files from the cache (if necessary).
copy_asv_files(asv_cache_path, env_path)
if (not setup) and self._extra_reqs_path.is_file():
# No need during initial ASV setup - this will be run again before
# any benchmarks are run.
cmd = f"{self.conda} env update -f {self._extra_reqs_path} -p {env_path}"
asv_util.check_output(cmd.split(" "))
def checkout_project(self, repo: Repo, commit_hash: str) -> None:
"""Check out the working tree of the project at given commit hash."""
super().checkout_project(repo, commit_hash)
self._nox_prep_env()
log.info(
f"Environment {self.name} updated to spec at {commit_hash[:8]}")
def _prep_env(self) -> None:
"""Run the custom environment script(s) and switch to using that environment."""
message = f"Running delegated environment management for: {self.name}"
log.info(message)
env_path = Path(self._path)
def copy_asv_files(src_parent: Path, dst_parent: Path) -> None:
"""For copying between self._path and a temporary cache."""
asv_files = list(src_parent.glob("asv*"))
# build_root_path.name usually == "project" .
asv_files += [src_parent / Path(self._build_root).name]
for src_path in asv_files:
dst_path = dst_parent / src_path.name
if not dst_path.exists():
# Only caching in case the environment has been rebuilt.
# If the dst_path already exists: rebuilding hasn't
# happened. Also a non-issue when copying in the reverse
# direction because the cache dir is temporary.
if src_path.is_dir():
func = copytree
else:
func = copy2
func(src_path, dst_path)
with TemporaryDirectory(prefix="delegated_asv_cache_") as asv_cache:
asv_cache_path = Path(asv_cache)
# Cache all of ASV's files as delegated command may remove and
# re-build the environment.
copy_asv_files(env_path.resolve(), asv_cache_path)
# Adapt the build_dir to the cache location.
build_root_path = Path(self._build_root)
build_dir_original = build_root_path / self._repo_subdir
build_dir_subpath = build_dir_original.relative_to(
build_root_path.parent)
build_dir = asv_cache_path / build_dir_subpath
# Run the script(s) for delegated environment creation/updating.
# (An adaptation of self._interpolate_and_run_commands).
for command, env, return_codes, cwd in self._env_commands:
local_envs = dict(environ)
local_envs.update(env)
if cwd is None:
cwd = str(build_dir)
_ = asv_util.check_output(
command,
timeout=self._install_timeout,
cwd=cwd,
env=local_envs,
valid_return_codes=return_codes,
)
# Replace the env that ASV created with a symlink to the env
# created/updated by the custom script.
delegated_env_path = sorted(
self._delegated_env_parent.glob("*"),
key=getmtime,
reverse=True,
)[0]
if env_path.resolve() != delegated_env_path:
try:
env_path.unlink(missing_ok=True)
except IsADirectoryError:
rmtree(env_path)
env_path.symlink_to(delegated_env_path,
target_is_directory=True)
# Check that environment exists.
try:
env_path.resolve(strict=True)
except FileNotFoundError:
message = f"Path does not resolve to environment: {env_path}"
log.error(message)
raise RuntimeError(message)
# Restore ASV's files from the cache (if necessary).
copy_asv_files(asv_cache_path, env_path.resolve())
# Record new environment information in properties.
self._update_info()
