def execute(self): session_friendly_name = self.signature or self.name logger.warning('Running session {}'.format(session_friendly_name)) try: # Set up the SessionConfig object (which session functions refer # to as `session`) and then the virtualenv. self._create_config() self._create_venv() # Run the actual commands prescribed by the session. cwd = py.path.local(os.getcwd()).as_cwd() with cwd: self._run_commands() # Nothing went wrong; return a success. return Result(self, Status.SUCCESS) except _SessionQuit: return Result(self, Status.ABORTED) except _SessionSkip: return Result(self, Status.SKIPPED) except CommandFailed: return Result(self, Status.FAILED) except KeyboardInterrupt: logger.error('Session {} interrupted.'.format(self)) raise
def execute(self): logger.warning('Running session {}'.format(self.signature or self.name)) try: self._create_config() self._create_venv() self._install_dependencies() if self.config._dir != '.': logger.info('Changing directory to {}'.format( self.config._dir)) cwd = py.path.local(self.config._dir).as_cwd() with cwd: self._run_commands() logger.success('Session {} successful. :)'.format(self.name)) return True except CommandFailed: logger.error('Session {} failed. :('.format(self.name)) return False except KeyboardInterrupt: logger.error('Session {} interrupted.'.format(self.name)) raise
def execute(self): session_friendly_name = self.signature or self.name logger.warning('Running session {}'.format(session_friendly_name)) try: if not self._create_config(): logger.error( 'Session {} aborted.'.format(session_friendly_name)) return False self._create_venv() cwd = py.path.local(os.getcwd()).as_cwd() with cwd: self._run_commands() logger.success( 'Session {} successful. :)'.format(session_friendly_name)) return True except CommandFailed: logger.error('Session {} failed. :('.format(session_friendly_name)) return False except KeyboardInterrupt: logger.error( 'Session {} interrupted.'.format(session_friendly_name)) raise
def _normalize_path(envdir, path): """Normalizes a string to be a "safe" filesystem path for a virtualenv.""" if isinstance(path, six.binary_type): path = path.decode('utf-8') path = unicodedata.normalize('NFKD', path).encode('ascii', 'ignore') path = path.decode('ascii') path = re.sub('[^\w\s-]', '-', path).strip().lower() path = re.sub('[-\s]+', '-', path) path = path.strip('-') full_path = os.path.join(envdir, path) if len(full_path) > 100 - len('bin/pythonX.Y'): if len(envdir) < 100 - 9: path = hashlib.sha1(path.encode('ascii')).hexdigest()[:8] full_path = os.path.join(envdir, path) logger.warning( 'The virtualenv name was hashed to avoid being too long.') else: logger.error( 'The virtualenv path {} is too long and will cause issues on ' 'some environments. Use the --envdir path to modify where ' 'nox stores virtualenvs.'.format(full_path)) return full_path
def _normalize_path(envdir, path): """Normalizes a string to be a "safe" filesystem path for a virtualenv.""" if isinstance(path, bytes): path = path.decode("utf-8") path = unicodedata.normalize("NFKD", path).encode("ascii", "ignore") path = path.decode("ascii") path = re.sub(r"[^\w\s-]", "-", path).strip().lower() path = re.sub(r"[-\s]+", "-", path) path = path.strip("-") full_path = os.path.join(envdir, path) if len(full_path) > 100 - len("bin/pythonX.Y"): if len(envdir) < 100 - 9: path = hashlib.sha1(path.encode("ascii")).hexdigest()[:8] full_path = os.path.join(envdir, path) logger.warning("The virtualenv name was hashed to avoid being too long.") else: logger.error( "The virtualenv path {} is too long and will cause issues on " "some environments. Use the --envdir path to modify where " "nox stores virtualenvs.".format(full_path) ) return full_path
def print_summary(results: List[Result], global_config: Namespace) -> List[Result]: """Print a summary of the results. Args: results (Sequence[~nox.sessions.Result]): A list of Result objects. global_config (~nox.main.GlobalConfig): The global configuration. Returns: results (Sequence[~nox.sessions.Result]): The results passed to this function, unmodified. """ # Sanity check: Do not print results if there was only one session run. if len(results) <= 1: return results # Iterate over the results and print the result for each in a # human-readable way. logger.warning("Ran multiple sessions:") for result in results: result.log("* {name}: {status}".format( name=result.session.friendly_name, status=result.status.name.lower())) # Return the results that were sent to this function. return results
def print_summary(results, global_config): """Print a summary of the results. Args: results (Sequence[~nox.sessions.Result]): A list of Result objects. global_config (~nox.main.GlobalConfig): The global configuration. Returns: results (Sequence[~nox.sessions.Result]): The results passed to this function, unmodified. """ # Sanity check: Do not print results if there was only one session run. if len(results) <= 1: return results # Iterate over the results and print the result for each in a # human-readable way. logger.warning("Ran multiple sessions:") for result in results: result.log( "* {name}: {status}".format( name=result.session.friendly_name, status=result.status.name.lower() ) ) # Return the results that were sent to this function. return results
def print_summary(results): logger.warning('Ran multiple sessions:') for session, result in results: log = logger.success if result else logger.error log('* {}: {}'.format( session.signature or session.name, 'passed' if result else 'failed'))
def run(args, *, env=None, silent=False, path=None, success_codes=None, log=True, external=False, **popen_kws): """Run a command-line program.""" if success_codes is None: success_codes = [0] cmd, args = args[0], args[1:] full_cmd = "{} {}".format(cmd, " ".join(args)) cmd_path = which(cmd, path) if log: logger.info(full_cmd) is_external_tool = path is not None and not cmd_path.startswith(path) if is_external_tool: if external == "error": logger.error( "Error: {} is not installed into the virtualenv, it is located at {}. " "Pass external=True into run() to explicitly allow this.". format(cmd, cmd_path)) raise CommandFailed("External program disallowed.") elif external is False: logger.warning( "Warning: {} is not installed into the virtualenv, it is located at {}. This might cause issues! " "Pass external=True into run() to silence this message.". format(cmd, cmd_path)) env = _clean_env(env) try: return_code, output = popen([cmd_path] + list(args), silent=silent, env=env, **popen_kws) if return_code not in success_codes: logger.error("Command {} failed with exit code {}{}".format( full_cmd, return_code, ":" if silent else "")) if silent: sys.stderr.write(output) raise CommandFailed("Returned code {}".format(return_code)) return output if silent else True except KeyboardInterrupt: logger.error("Interrupted...") raise
def _tests(session): """ Run tests """ if CI_RUN or IS_WINDOWS: env = None else: env = {"PYTHONPATH": str(REPO_ROOT)} if SKIP_REQUIREMENTS_INSTALL is False: # Always have the wheel package installed session.install("wheel", silent=PIP_INSTALL_SILENT) session.install(COVERAGE_VERSION_REQUIREMENT, silent=PIP_INSTALL_SILENT) session.install(SALT_REQUIREMENT, silent=PIP_INSTALL_SILENT) if CI_RUN or IS_WINDOWS: session.install("-e", ".", silent=PIP_INSTALL_SILENT) pip_list = session_run_always(session, "pip", "list", "--format=json", silent=True, log=False) if pip_list: for requirement in json.loads(pip_list.splitlines()[0]): if requirement["name"] == "msgpack-python": logger.warning( "Found msgpack-python installed. Installing msgpack to override it" ) session.install("msgpack=={}".format( requirement["version"])) break session.install("-r", os.path.join("requirements", "tests.txt"), silent=PIP_INSTALL_SILENT) session.run("coverage", "erase") args = [ "--rootdir", str(REPO_ROOT), "--log-file={}".format(RUNTESTS_LOGFILE), "--log-file-level=debug", "--show-capture=no", "--junitxml={}".format(JUNIT_REPORT), "-ra", "-s", ] if session._runner.global_config.forcecolor: args.append("--color=yes") if not session.posargs: args.append("tests/") else: for arg in session.posargs: if arg.startswith("--color") and args[0].startswith("--color"): args.pop(0) args.append(arg) session.run("coverage", "run", "-m", "pytest", *args, env=env) session.notify("coverage")
def execute(self) -> Result: logger.warning(f"Running session {self.friendly_name}") try: # By default, Nox should quietly change to the directory where # the noxfile.py file is located. cwd = py.path.local( os.path.realpath(os.path.dirname( self.global_config.noxfile))).as_cwd() with cwd: self._create_venv() session = Session(self) self.func(session) # Nothing went wrong; return a success. return Result(self, Status.SUCCESS) except nox.virtualenv.InterpreterNotFound as exc: if self.global_config.error_on_missing_interpreters: return Result(self, Status.FAILED, reason=str(exc)) else: logger.warning( "Missing interpreters will error by default on CI systems." ) return Result(self, Status.SKIPPED, reason=str(exc)) except _SessionQuit as exc: return Result(self, Status.ABORTED, reason=str(exc)) except _SessionSkip as exc: return Result(self, Status.SKIPPED, reason=str(exc)) except nox.command.CommandFailed: return Result(self, Status.FAILED) except KeyboardInterrupt: logger.error(f"Session {self.friendly_name} interrupted.") raise except Exception as exc: logger.exception( f"Session {self.friendly_name} raised exception {exc!r}") return Result(self, Status.FAILED)
def execute(self): logger.warning("Running session {}".format(self.friendly_name)) try: # By default, nox should quietly change to the directory where # the noxfile.py file is located. cwd = py.path.local( os.path.realpath(os.path.dirname(self.global_config.noxfile)) ).as_cwd() with cwd: self._create_venv() session = Session(self) self.func(session) # Nothing went wrong; return a success. return Result(self, Status.SUCCESS) except nox.virtualenv.InterpreterNotFound as exc: if self.global_config.error_on_missing_interpreters: return Result(self, Status.FAILED, reason=str(exc)) else: return Result(self, Status.SKIPPED, reason=str(exc)) except _SessionQuit as exc: return Result(self, Status.ABORTED, reason=str(exc)) except _SessionSkip as exc: return Result(self, Status.SKIPPED, reason=str(exc)) except nox.command.CommandFailed: return Result(self, Status.FAILED) except KeyboardInterrupt: logger.error("Session {} interrupted.".format(self.friendly_name)) raise except Exception as exc: logger.exception( "Session {} raised exception {!r}".format(self.friendly_name, exc) ) return Result(self, Status.FAILED)
def run_manifest(manifest: Manifest, global_config: Namespace) -> List[Result]: """Run the full manifest of sessions. Args: manifest (~.Manifest): The manifest of sessions to be run. global_config (~nox.main.GlobalConfig): The global configuration. Returns: tuple[~nox.sessions.Session,~.SessionStatus]: A two-tuple of the sessions and the result of each session that was run. """ results = [] # Iterate over each session in the manifest, and execute it. # # Note that it is possible for the manifest to be altered in any given # iteration. for session in manifest: # possibly raise warnings associated with this session if WARN_PYTHONS_IGNORED in session.func.should_warn: logger.warning( "Session {} is set to run with venv_backend='none', IGNORING its python={} parametrization. ".format( session.name, session.func.should_warn[WARN_PYTHONS_IGNORED] ) ) result = session.execute() result.log( "Session {name} {status}.".format( name=session.friendly_name, status=result.imperfect ) ) results.append(result) # Sanity check: If we are supposed to stop on the first error case, # the abort now. if not result and global_config.stop_on_first_error: return results # The entire manifest has been processed; return the results. return results
def execute(self): session_friendly_name = self.signature or self.name logger.warning("Running session {}".format(session_friendly_name)) try: # By default, nox should quietly change to the directory where # the noxfile.py file is located. cwd = py.path.local( os.path.realpath(os.path.dirname( self.global_config.noxfile))).as_cwd() with cwd: self._create_venv() session = Session(self) self.func(session) # Nothing went wrong; return a success. return Result(self, Status.SUCCESS) except _SessionQuit: return Result(self, Status.ABORTED) except _SessionSkip: return Result(self, Status.SKIPPED) except nox.command.CommandFailed: return Result(self, Status.FAILED) except KeyboardInterrupt: logger.error("Session {} interrupted.".format(self)) raise except Exception as exc: logger.exception("Session {} raised exception {!r}".format( self, exc)) return Result(self, Status.FAILED)
def conda_install( self, *args: str, auto_offline: bool = True, **kwargs: Any ) -> None: """Install invokes `conda install`_ to install packages inside of the session's environment. To install packages directly:: session.conda_install('pandas') session.conda_install('numpy', 'scipy') session.conda_install('--channel=conda-forge', 'dask==2.1.0') To install packages from a ``requirements.txt`` file:: session.conda_install('--file', 'requirements.txt') session.conda_install('--file', 'requirements-dev.txt') By default this method will detect when internet connection is not available and will add the `--offline` flag automatically in that case. To disable this behaviour, set `auto_offline=False`. To install the current package without clobbering conda-installed dependencies:: session.install('.', '--no-deps') # Install in editable mode. session.install('-e', '.', '--no-deps') Additional keyword args are the same as for :meth:`run`. .. _conda install: """ venv = self._runner.venv prefix_args = () # type: Tuple[str, ...] if isinstance(venv, CondaEnv): prefix_args = ("--prefix", venv.location) elif not isinstance(venv, PassthroughEnv): # pragma: no cover raise ValueError( "A session without a conda environment can not install dependencies from conda." ) if not args: raise ValueError("At least one argument required to install().") if self._runner.global_config.no_install and venv._reused: return None # Escape args that should be (conda-specific; pip install does not need this) args = _dblquote_pkg_install_args(args) if "silent" not in kwargs: kwargs["silent"] = True extraopts = () # type: Tuple[str, ...] if auto_offline and venv.is_offline(): logger.warning( "Automatically setting the `--offline` flag as conda repo seems unreachable." ) extraopts = ("--offline",) self._run( "conda", "install", "--yes", *extraopts, *prefix_args, *args, external="error", **kwargs, )
def tests(session): """ Run tests """ env = {} system_install = False if session.python is False: # This is running against the system logger.warning( "Adding SALT_FACTORIES_SYSTEM_INSTALL=1 to the environ so that tests run against the sytem python" ) env["SALT_FACTORIES_SYSTEM_INSTALL"] = "1" system_install = True if SKIP_REQUIREMENTS_INSTALL is False: # Always have the wheel package installed session.install("wheel", silent=PIP_INSTALL_SILENT) session.install(COVERAGE_VERSION_REQUIREMENT, silent=PIP_INSTALL_SILENT) if session.python is not False and system_install is False: session.install(SALT_REQUIREMENT, silent=PIP_INSTALL_SILENT) session.install("-e", ".", silent=PIP_INSTALL_SILENT) pip_list = session_run_always( session, "pip", "list", "--format=json", silent=True, log=False, stderr=None ) if pip_list: for requirement in json.loads(pip_list.splitlines()[0]): if requirement["name"] == "msgpack-python": logger.warning( "Found msgpack-python installed. Installing msgpack to override it" ) session.install("msgpack=={}".format(requirement["version"])) break session.install("-r", os.path.join("requirements", "tests.txt"), silent=PIP_INSTALL_SILENT) if EXTRA_REQUIREMENTS_INSTALL: session.log( "Installing the following extra requirements because the EXTRA_REQUIREMENTS_INSTALL " "environment variable was set: EXTRA_REQUIREMENTS_INSTALL='%s'", EXTRA_REQUIREMENTS_INSTALL, ) install_command = ["--progress-bar=off"] install_command += [req.strip() for req in EXTRA_REQUIREMENTS_INSTALL.split()] session.install(*install_command, silent=PIP_INSTALL_SILENT) session.run("coverage", "erase") python_path_env_var = os.environ.get("PYTHONPATH") or None if python_path_env_var is None: python_path_env_var = SITECUSTOMIZE_DIR else: python_path_entries = python_path_env_var.split(os.pathsep) if SITECUSTOMIZE_DIR in python_path_entries: python_path_entries.remove(SITECUSTOMIZE_DIR) python_path_entries.insert(0, SITECUSTOMIZE_DIR) python_path_env_var = os.pathsep.join(python_path_entries) env.update( { # The updated python path so that sitecustomize is importable "PYTHONPATH": python_path_env_var, # The full path to the .coverage data file. Makes sure we always write # them to the same directory "COVERAGE_FILE": str(COVERAGE_REPORT_DB), # Instruct sub processes to also run under coverage "COVERAGE_PROCESS_START": str(REPO_ROOT / ".coveragerc"), } ) args = [ "--rootdir", str(REPO_ROOT), "--log-file={}".format(RUNTESTS_LOGFILE), "--log-file-level=debug", "--show-capture=no", "--junitxml={}".format(JUNIT_REPORT), "--showlocals", "--strict-markers", "--lsof", "-ra", "-s", ] if session._runner.global_config.forcecolor: args.append("--color=yes") if not session.posargs: args.append("tests/") else: for arg in session.posargs: if arg.startswith("--color") and session._runner.global_config.forcecolor: args.remove("--color=yes") args.append(arg) session.run("coverage", "run", "-m", "pytest", *args, env=env) # Always combine and generate the XML coverage report try: session.run("coverage", "combine") except CommandFailed: # Sometimes some of the coverage files are corrupt which would # trigger a CommandFailed exception pass # Generate report for saltfactories code coverage session.run( "coverage", "xml", "-o", str(COVERAGE_REPORT_SALTFACTORIES), "--omit=tests/*", "--include=src/saltfactories/*", ) # Generate report for tests code coverage session.run( "coverage", "xml", "-o", str(COVERAGE_REPORT_TESTS), "--omit=src/saltfactories/*", "--include=tests/*", ) try: cmdline = ["coverage", "report", "--show-missing", "--include=src/saltfactories/*,tests/*"] if system_install is False: cmdline.append("--fail-under={}".format(COVERAGE_FAIL_UNDER_PERCENT)) session.run(*cmdline) finally: if COVERAGE_REPORT_DB.exists(): shutil.copyfile(str(COVERAGE_REPORT_DB), str(ARTIFACTS_DIR / ".coverage"))
def run( args, *, env=None, silent=False, path=None, success_codes=None, log=True, external=False ): """Run a command-line program.""" if success_codes is None: success_codes = [0] cmd, args = args[0], args[1:] full_cmd = "{} {}".format(cmd, " ".join(args)) cmd_path = which(cmd, path) if log: logger.info(full_cmd) is_external_tool = path is not None and not cmd_path.startswith(path) if is_external_tool: if external == "error": logger.error( "Error: {} is not installed into the virtualenv, it is located at {}. " "Pass external=True into run() to explicitly allow this.".format( cmd, cmd_path ) ) raise CommandFailed("External program disallowed.") elif external is False: logger.warning( "Warning: {} is not installed into the virtualenv, is it located at {}. This might cause issues! " "Pass external=True into run() to silence this message.".format( cmd, cmd_path ) ) env = _clean_env(env) try: return_code, output = popen([cmd_path] + list(args), silent=silent, env=env) if return_code not in success_codes: logger.error( "Command {} failed with exit code {}{}".format( full_cmd, return_code, ":" if silent else "" ) ) if silent: sys.stderr.write(output) raise CommandFailed("Returned code {}".format(return_code)) return output if silent else True except KeyboardInterrupt: logger.error("Interrupted...") raise
def _tests(session): """ Run tests """ if SKIP_REQUIREMENTS_INSTALL is False: # Always have the wheel package installed session.install("wheel", silent=PIP_INSTALL_SILENT) session.install(COVERAGE_VERSION_REQUIREMENT, silent=PIP_INSTALL_SILENT) session.install(SALT_REQUIREMENT, silent=PIP_INSTALL_SILENT) session.install("-e", ".", silent=PIP_INSTALL_SILENT) pip_list = session_run_always(session, "pip", "list", "--format=json", silent=True, log=False) if pip_list: for requirement in json.loads(pip_list.splitlines()[0]): if requirement["name"] == "msgpack-python": logger.warning( "Found msgpack-python installed. Installing msgpack to override it" ) session.install("msgpack=={}".format( requirement["version"])) break session.install("-r", os.path.join("requirements", "tests.txt"), silent=PIP_INSTALL_SILENT) if EXTRA_REQUIREMENTS_INSTALL: session.log( "Installing the following extra requirements because the EXTRA_REQUIREMENTS_INSTALL " "environment variable was set: EXTRA_REQUIREMENTS_INSTALL='%s'", EXTRA_REQUIREMENTS_INSTALL, ) install_command = ["--progress-bar=off"] install_command += [ req.strip() for req in EXTRA_REQUIREMENTS_INSTALL.split() ] session.install(*install_command, silent=PIP_INSTALL_SILENT) session.run("coverage", "erase") args = [ "--rootdir", str(REPO_ROOT), "--log-file={}".format(RUNTESTS_LOGFILE), "--log-file-level=debug", "--show-capture=no", "--junitxml={}".format(JUNIT_REPORT), "--showlocals", "-ra", "-s", ] if session._runner.global_config.forcecolor: args.append("--color=yes") if not session.posargs: args.append("tests/") else: for arg in session.posargs: if arg.startswith("--color") and args[0].startswith("--color"): args.pop(0) args.append(arg) session.run("coverage", "run", "-m", "pytest", *args) session.notify("coverage")
def skip(self, *args, **kwargs): """Immediately skips the session and optionally logs a warning.""" if args or kwargs: logger.warning(*args, **kwargs) raise _SessionSkip()
def run( args: Sequence[str], *, env: Optional[dict] = None, silent: bool = False, paths: Optional[List[str]] = None, success_codes: Optional[Iterable[int]] = None, log: bool = True, external: Union[Literal["error"], bool] = False, **popen_kws: Any, ) -> Union[str, bool]: """Run a command-line program.""" if success_codes is None: success_codes = [0] cmd, args = args[0], args[1:] full_cmd = f"{cmd} {_shlex_join(args)}" cmd_path = which(cmd, paths) if log: logger.info(full_cmd) is_external_tool = paths is not None and not any( cmd_path.startswith(path) for path in paths) if is_external_tool: if external == "error": logger.error( f"Error: {cmd} is not installed into the virtualenv, it is located at {cmd_path}. " "Pass external=True into run() to explicitly allow this.") raise CommandFailed("External program disallowed.") elif external is False: logger.warning( f"Warning: {cmd} is not installed into the virtualenv, it is located at {cmd_path}. This might cause issues! " "Pass external=True into run() to silence this message.") env = _clean_env(env) try: return_code, output = popen([cmd_path] + list(args), silent=silent, env=env, **popen_kws) if return_code not in success_codes: suffix = ":" if silent else "" logger.error( f"Command {full_cmd} failed with exit code {return_code}{suffix}" ) if silent: sys.stderr.write(output) raise CommandFailed(f"Returned code {return_code}") if output: logger.output(output) return output if silent else True except KeyboardInterrupt: logger.error("Interrupted...") raise
def warn(self, *args: Any, **kwargs: Any) -> None: """Outputs a warning during the session.""" logger.warning(*args, **kwargs)