def _setup(self): log.info("Creating oggm conda environment for {0}".format(self.name)) env_file = tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".yml") try: pyver = str(self._python).replace(".", "")[:2] oggm_env = OGGM_CONDA_ENVS[pyver] req = requests.get(OGGM_CONDA_ENV_URL.format(oggm_env)) req.raise_for_status() for line in req.text.splitlines(): if line.startswith("prefix:"): continue elif line.startswith("name:"): env_file.write("name: {0}\n".format(self.name)) else: env_file.write(line + "\n") env_file.close() self._conda_channels = ["conda-forge", "defaults"] self._conda_environment_file = env_file.name return super()._setup() except Exception as exc: if os.path.isfile(env_file.name): with open(env_file.name, "r") as f: text = f.read() log.info("oggm conda env create failed: in {} with:\n{}" .format(self._path, text)) raise finally: os.unlink(env_file.name)
def _nox_prep_env(self, setup: bool = False) -> None: message = f"Running Nox environment update for: {self.name}" log.info(message) build_root_path = Path(self._build_root) env_path = Path(self._path) def copy_asv_files(src_parent: Path, dst_parent: Path) -> None: """For copying between self._path and a temporary cache.""" asv_files = list(src_parent.glob("asv*")) # build_root_path.name usually == "project" . asv_files += [src_parent / build_root_path.name] for src_path in asv_files: dst_path = dst_parent / src_path.name if not dst_path.exists(): # Only cache-ing in case Nox has rebuilt the env @ # self._path. If the dst_path already exists: rebuilding # hasn't happened. Also a non-issue when copying in the # reverse direction because the cache dir is temporary. if src_path.is_dir(): func = copytree else: func = copy2 func(src_path, dst_path) with TemporaryDirectory(prefix="nox_asv_cache_") as asv_cache: asv_cache_path = Path(asv_cache) if setup: noxfile = self.setup_noxfile else: # Cache all of ASV's files as Nox may remove and re-build the environment. copy_asv_files(env_path, asv_cache_path) # Get location of noxfile in cache. noxfile_original = (build_root_path / self._repo_subdir / self.noxfile_rel_path) noxfile_subpath = noxfile_original.relative_to( build_root_path.parent) noxfile = asv_cache_path / noxfile_subpath nox_cmd = [ "nox", f"--noxfile={noxfile}", # Place the env in the ASV env directory, instead of the default. f"--envdir={env_path.parent}", f"--session={self.nox_session_name}", f"--python={self._python}", "--install-only", "--no-error-on-external-run", "--verbose", ] _ = asv_util.check_output(nox_cmd) if not env_path.is_dir(): message = f"Expected Nox environment not found: {env_path}" log.error(message) raise RuntimeError(message) if not setup: # Restore ASV's files from the cache (if necessary). copy_asv_files(asv_cache_path, env_path)
def _uninstall_project(self): installed_hash = self._get_installed_commit_hash() if installed_hash: log.info(f"Clearing conda environment for {self.name}") self._cache._remove_cache_dir(installed_hash) # we can only run the uninstall command if an environment has already # been made before, otherwise there is no python to use to uninstall super()._uninstall_project()
def _uninstall_project(self): self._set_installed_commit_hash(None) log.info('Uninstalling from {}'.format(self.name)) if not self._uninstall_command: log.info('Nothing to uninstall') return for cmd in self._uninstall_command: self._run(cmd, shell=True, cwd=self._env_dir, check=True, timeout=self._install_timeout)
def _install_project(self, repo, commit_hash, build_dir): commit_name = repo.get_decorated_hash(commit_hash, 8) log.info('Installing {} into {}'.format(commit_name, self.name)) if not self._install_command: log.info('Nothing to install') return for cmd in self._install_command: self._run(cmd, shell=True, cwd=build_dir, check=True, timeout=self._install_timeout)
def test_log_indent(capsys): log.set_nitems(0) log.info("First\nSecond") out, err = capsys.readouterr() lines = out.lstrip().splitlines() assert lines[0].index('First') == lines[1].index('Second') log.set_nitems(1) log.info("First\nSecond") out, err = capsys.readouterr() lines = out.lstrip().splitlines() assert lines[0].index('First') == lines[1].index('Second')
def _setup(self): log.info("Creating oggm conda environment for {0}".format(self.name)) try: conda = _find_conda() except IOError as e: raise util.UserError(str(e)) env_file = tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".yml") try: pyver = str(self._python).replace(".", "")[:2] oggm_env = OGGM_CONDA_ENVS[pyver] req = requests.get(OGGM_CONDA_ENV_URL.format(oggm_env)) req.raise_for_status() env_text = req.text for line in env_text.splitlines(): if line.startswith("prefix:") or self._has_requirement(line): continue elif line.startswith("name:"): env_file.write("name: {0}\n".format(self.name)) else: env_file.write(line + "\n") conda_args, pip_args = self._get_requirements(conda) env_file.writelines((' - %s\n' % s for s in conda_args)) if pip_args: env_file.write(' - pip:\n') env_file.writelines((' - %s\n' % s for s in pip_args)) env_file.close() util.check_output([conda] + [ 'env', 'create', '-f', env_file.name, '-p', self._path, '--force' ]) except Exception as exc: if os.path.isfile(env_file.name): with open(env_file.name, "r") as f: text = f.read() log.info( "oggm conda env create failed: in {} with:\n{}".format( self._path, text)) raise finally: os.unlink(env_file.name)
def _build_project(self, repo, commit_hash, build_dir): # at "build" time, we build the environment from the provided lockfile self.run_executable( _find_conda(), [ "install", "-y", "-p", self._path, "--file", f"{build_dir}/{self._lockfile_path}", ], ) log.info( f"Environment {self.name} updated to spec at {commit_hash[:8]}" ) log.debug( self.run_executable(_find_conda(), ["list", "-p", self._path]) ) return super()._build_project(repo, commit_hash, build_dir)
def _setup(self) -> None: """Used for initial environment creation - mimics parent method where possible.""" try: self.conda = _find_conda() except IOError as e: raise asv_util.UserError(str(e)) if find_spec("nox") is None: raise asv_util.UserError("Module not found: nox") message = f"Creating Nox-Conda environment for {self.name} ." log.info(message) try: self._nox_prep_env(setup=True) except Exception: raise finally: # No longer need the setup checkout now that the environment has been built. self.project_temp_checkout.cleanup() # Create an environment.yml file from the requirements in asv.conf.json. # No default dependencies to specify - unlike parent - because Nox # includes these by default. conda_args, pip_args = self._get_requirements(self.conda) if conda_args or pip_args: with self._extra_reqs_path.open("w") as req_file: req_file.write(f"name: {self.name}\n") req_file.write("channels:\n") req_file.writelines( [f" - {channel}\n" for channel in self._conda_channels]) req_file.write("dependencies:\n") # Categorise and write dependencies based on pip vs. conda. req_file.writelines( [f" - {package}\n" for package in conda_args]) if pip_args: # And now specify the packages that are to be installed in the # pip subsection. req_file.write(" - pip:\n") req_file.writelines( [f" - {package}\n" for package in pip_args])
def _setup(self) -> None: """Used for initial environment creation - mimics parent method where possible.""" try: self.conda = _find_conda() except IOError as e: raise asv_util.UserError(str(e)) if find_spec("nox") is None: raise asv_util.UserError("Module not found: nox") message = f"Creating Nox-Conda environment for {self.name} ." log.info(message) try: self._nox_prep_env(setup=True) finally: # No longer need the setup checkout now that the environment has been built. self.project_temp_checkout.cleanup() conda_args, pip_args = self._get_requirements(self.conda) if conda_args or pip_args: message = ( "Ignoring user input package requirements. Benchmark " "environment management is exclusively performed by Nox.") log.warning(message)
def _setup(self): # create the shell of a conda environment, that includes no packages log.info("Creating conda environment for {0}".format(self.name)) self.run_executable( _find_conda(), ["create", "-y", "-p", self._path, "--force"] )
def _nox_prep_env(self, setup: bool = False) -> None: message = f"Running Nox environment update for: {self.name}" log.info(message) build_root_path = Path(self._build_root) env_path = Path(self._path) def copy_asv_files(src_parent: Path, dst_parent: Path) -> None: """For copying between self._path and a temporary cache.""" asv_files = list(src_parent.glob("asv*")) # build_root_path.name usually == "project" . asv_files += [src_parent / build_root_path.name] for src_path in asv_files: dst_path = dst_parent / src_path.name if not dst_path.exists(): # Only cache-ing in case Nox has rebuilt the env @ # self._path. If the dst_path already exists: rebuilding # hasn't happened. Also a non-issue when copying in the # reverse direction because the cache dir is temporary. if src_path.is_dir(): func = copytree else: func = copy2 func(src_path, dst_path) with TemporaryDirectory(prefix="nox_asv_cache_") as asv_cache: asv_cache_path = Path(asv_cache) if setup: noxfile_path = self.setup_noxfile else: # Cache all of ASV's files as Nox may remove and re-build the environment. copy_asv_files(env_path, asv_cache_path) # Get location of noxfile in cache. noxfile_path_build = (build_root_path / self._repo_subdir / NOXFILE_REL_PATH) noxfile_path = asv_cache_path / noxfile_path_build.relative_to( build_root_path.parent) nox_cmd = [ "nox", f"--noxfile={noxfile_path}", f"--envdir={env_path.parent}", f"--session={SESSION_NAME}", f"--python={self._python}", "--install-only", "--no-error-on-external-run", "--verbose", ] _ = asv_util.check_output(nox_cmd) if not env_path.is_dir(): message = f"Expected Nox environment not found: {env_path}" log.error(message) if not setup: # Restore ASV's files from the cache (if necessary). copy_asv_files(asv_cache_path, env_path) if (not setup) and self._extra_reqs_path.is_file(): # No need during initial ASV setup - this will be run again before # any benchmarks are run. cmd = f"{self.conda} env update -f {self._extra_reqs_path} -p {env_path}" asv_util.check_output(cmd.split(" "))
def checkout_project(self, repo: Repo, commit_hash: str) -> None: """Check out the working tree of the project at given commit hash.""" super().checkout_project(repo, commit_hash) self._nox_prep_env() log.info( f"Environment {self.name} updated to spec at {commit_hash[:8]}")
def _prep_env(self) -> None: """Run the custom environment script(s) and switch to using that environment.""" message = f"Running delegated environment management for: {self.name}" log.info(message) env_path = Path(self._path) def copy_asv_files(src_parent: Path, dst_parent: Path) -> None: """For copying between self._path and a temporary cache.""" asv_files = list(src_parent.glob("asv*")) # build_root_path.name usually == "project" . asv_files += [src_parent / Path(self._build_root).name] for src_path in asv_files: dst_path = dst_parent / src_path.name if not dst_path.exists(): # Only caching in case the environment has been rebuilt. # If the dst_path already exists: rebuilding hasn't # happened. Also a non-issue when copying in the reverse # direction because the cache dir is temporary. if src_path.is_dir(): func = copytree else: func = copy2 func(src_path, dst_path) with TemporaryDirectory(prefix="delegated_asv_cache_") as asv_cache: asv_cache_path = Path(asv_cache) # Cache all of ASV's files as delegated command may remove and # re-build the environment. copy_asv_files(env_path.resolve(), asv_cache_path) # Adapt the build_dir to the cache location. build_root_path = Path(self._build_root) build_dir_original = build_root_path / self._repo_subdir build_dir_subpath = build_dir_original.relative_to( build_root_path.parent) build_dir = asv_cache_path / build_dir_subpath # Run the script(s) for delegated environment creation/updating. # (An adaptation of self._interpolate_and_run_commands). for command, env, return_codes, cwd in self._env_commands: local_envs = dict(environ) local_envs.update(env) if cwd is None: cwd = str(build_dir) _ = asv_util.check_output( command, timeout=self._install_timeout, cwd=cwd, env=local_envs, valid_return_codes=return_codes, ) # Replace the env that ASV created with a symlink to the env # created/updated by the custom script. delegated_env_path = sorted( self._delegated_env_parent.glob("*"), key=getmtime, reverse=True, )[0] if env_path.resolve() != delegated_env_path: try: env_path.unlink(missing_ok=True) except IsADirectoryError: rmtree(env_path) env_path.symlink_to(delegated_env_path, target_is_directory=True) # Check that environment exists. try: env_path.resolve(strict=True) except FileNotFoundError: message = f"Path does not resolve to environment: {env_path}" log.error(message) raise RuntimeError(message) # Restore ASV's files from the cache (if necessary). copy_asv_files(asv_cache_path, env_path.resolve()) # Record new environment information in properties. self._update_info()