def tox_testenv_create(venv, action): config_interpreter = venv.getsupportedinterpreter() args = [sys.executable, "-m", "virtualenv"] if venv.envconfig.sitepackages: args.append("--system-site-packages") if venv.envconfig.alwayscopy: args.append("--always-copy") if not venv.envconfig.download: args.append("--no-download") # add interpreter explicitly, to prevent using default (virtualenv.ini) args.extend(["--python", str(config_interpreter)]) cleanup_for_venv(venv) base_path = venv.path.dirpath() base_path.ensure(dir=1) args.append(venv.path.basename) if not _SKIP_VENV_CREATION: try: venv._pcall( args, venv=False, action=action, cwd=base_path, redirect=reporter.verbosity() < reporter.Verbosity.DEBUG, ) except KeyboardInterrupt: venv.status = "keyboardinterrupt" raise return True # Return non-None to indicate plugin has completed
def tox_testenv_create(venv, action): config_interpreter = venv.getsupportedinterpreter() args = [sys.executable, "-m", "virtualenv"] if venv.envconfig.sitepackages: args.append("--system-site-packages") if venv.envconfig.alwayscopy: args.append("--always-copy") if NO_DOWNLOAD: args.append("--no-download") # add interpreter explicitly, to prevent using default (virtualenv.ini) args.extend(["--python", str(config_interpreter)]) within_parallel = PARALLEL_ENV_VAR_KEY in os.environ if within_parallel: if venv.path.exists(): # do not delete the log folder as that's used by parent for content in venv.path.listdir(): if not content.basename == "log": content.remove(rec=1, ignore_errors=True) else: ensure_empty_dir(venv.path) basepath = venv.path.dirpath() basepath.ensure(dir=1) args.append(venv.path.basename) if not _SKIP_VENV_CREATION: venv._pcall( args, venv=False, action=action, cwd=basepath, redirect=reporter.verbosity() < reporter.Verbosity.DEBUG, ) return True # Return non-None to indicate plugin has completed
def run_install_command(self, packages, action, options=()): def expand(val): # expand an install command if val == "{packages}": for package in packages: yield package elif val == "{opts}": for opt in options: yield opt else: yield val cmd = list( chain.from_iterable( expand(val) for val in self.envconfig.install_command)) env = self._get_os_environ() self.ensure_pip_os_environ_ok(env) old_stdout = sys.stdout sys.stdout = codecs.getwriter("utf8")(sys.stdout) try: self._pcall( cmd, cwd=self.envconfig.config.toxinidir, action=action, redirect=reporter.verbosity() < reporter.Verbosity.DEBUG, env=env, ) finally: sys.stdout = old_stdout
def show_config(config): parser = configparser.ConfigParser() if not config.envlist_explicit or reporter.verbosity() >= reporter.Verbosity.INFO: tox_info(config, parser) version_info(parser) tox_envs_info(config, parser) content = StringIO() parser.write(content) value = content.getvalue().rstrip() reporter.verbosity0(value)
def install_pkg(self, dir, action, name, is_develop=False): assert action is not None if getattr(self, "just_created", False): action.setactivity(name, dir) self.finish() pip_flags = ["--exists-action", "w"] else: if is_develop and not self._needs_reinstall(dir, action): action.setactivity("{}-noop".format(name), dir) return action.setactivity("{}-nodeps".format(name), dir) pip_flags = ["--no-deps"] + ([] if is_develop else ["-U"]) pip_flags.extend(["-v"] * min(3, reporter.verbosity() - 2)) if self.envconfig.extras: dir += "[{}]".format(",".join(self.envconfig.extras)) target = [dir] if is_develop: target.insert(0, "-e") self._install(target, extraopts=pip_flags, action=action)
def get_pip_install(venv, packages, use_develop=False): flags = ["-e"] if use_develop else [] flags.extend(["-v"] * min(3, reporter.verbosity() - 2)) if not packages: return None pip_install = list(venv.envconfig.install_command) try: index = pip_install.index("{opts}") pip_install = list( chain(pip_install[:index], flags, pip_install[index + 1:])) except ValueError: # pragma: no cover pass # pragma: no cover try: index = pip_install.index("{packages}") pip_install = list( chain(pip_install[:index], (i.name for i in packages), pip_install[index + 1:])) except ValueError: # pragma: no cover pass # pragma: no cover if "PIP_INDEX_URL" in venv.env: index_url = venv.env["PIP_INDEX_URL"] pip_install.extend(("-i", index_url)) return pip_install
def run_parallel(config, venv_dict): """here we'll just start parallel sub-processes""" live_out = config.option.parallel_live disable_spinner = bool(os.environ.get("TOX_PARALLEL_NO_SPINNER") == "1") args = [sys.executable, MAIN_FILE] + config.args try: position = args.index("--") except ValueError: position = len(args) max_parallel = config.option.parallel if max_parallel is None: max_parallel = len(venv_dict) semaphore = Semaphore(max_parallel) finished = Event() show_progress = (not disable_spinner and not live_out and reporter.verbosity() > reporter.Verbosity.QUIET) with Spinner(enabled=show_progress) as spinner: def run_in_thread(tox_env, os_env, processes): output = None print_out = None env_name = tox_env.envconfig.envname status = "skipped tests" if config.option.notest else None try: os_env[str(PARALLEL_ENV_VAR_KEY_PRIVATE)] = str(env_name) os_env[str(PARALLEL_ENV_VAR_KEY_PUBLIC)] = str(env_name) args_sub = list(args) if hasattr(tox_env, "package"): args_sub.insert(position, str(tox_env.package)) args_sub.insert(position, "--installpkg") if tox_env.get_result_json_path(): result_json_index = args_sub.index("--result-json") args_sub[result_json_index + 1] = "{}".format( tox_env.get_result_json_path()) with tox_env.new_action("parallel {}".format( tox_env.name)) as action: def collect_process(process): processes[tox_env] = (action, process) print_out = not live_out and tox_env.envconfig.parallel_show_output output = action.popen( args=args_sub, env=os_env, redirect=not live_out, capture_err=print_out, callback=collect_process, returnout=print_out, ) except InvocationError as err: status = "parallel child exit code {}".format(err.exit_code) finally: semaphore.release() finished.set() tox_env.status = status done.add(env_name) outcome = spinner.succeed if config.option.notest: outcome = spinner.skip elif status is not None: outcome = spinner.fail outcome(env_name) if print_out and output is not None: reporter.verbosity0(output) threads = deque() processes = {} todo_keys = set(venv_dict.keys()) todo = OrderedDict((n, todo_keys & set(v.envconfig.depends)) for n, v in venv_dict.items()) done = set() try: while todo: for name, depends in list(todo.items()): if depends - done: # skip if has unfinished dependencies continue del todo[name] venv = venv_dict[name] semaphore.acquire(blocking=True) spinner.add(name) thread = Thread( target=run_in_thread, args=(venv, os.environ.copy(), processes), ) thread.daemon = True thread.start() threads.append(thread) if todo: # wait until someone finishes and retry queuing jobs finished.wait() finished.clear() while threads: threads = [ thread for thread in threads if not thread.join(0.1) and thread.is_alive() ] except KeyboardInterrupt: reporter.verbosity0( "[{}] KeyboardInterrupt parallel - stopping children".format( os.getpid()), ) while True: # do not allow to interrupt until children interrupt try: # putting it inside a thread so it's not interrupted stopper = Thread(target=_stop_child_processes, args=(processes, threads)) stopper.start() stopper.join() except KeyboardInterrupt: continue raise KeyboardInterrupt
def run_parallel(config, venv_dict): """here we'll just start parallel sub-processes""" live_out = config.option.parallel_live args = [sys.executable, "-m", "tox"] + config.args try: position = args.index("--") except ValueError: position = len(args) try: parallel_at = args[0:position].index("--parallel") del args[parallel_at] position -= 1 except ValueError: pass max_parallel = config.option.parallel if max_parallel is None: max_parallel = len(venv_dict) semaphore = Semaphore(max_parallel) finished = Event() sink = None if live_out else subprocess.PIPE show_progress = not live_out and reporter.verbosity() > reporter.Verbosity.QUIET with Spinner(enabled=show_progress) as spinner: def run_in_thread(tox_env, os_env): res = None env_name = tox_env.envconfig.envname try: os_env[str(PARALLEL_ENV_VAR_KEY)] = str(env_name) args_sub = list(args) if hasattr(tox_env, "package"): args_sub.insert(position, str(tox_env.package)) args_sub.insert(position, "--installpkg") process = subprocess.Popen( args_sub, env=os_env, stdout=sink, stderr=sink, stdin=None, universal_newlines=True, ) res = process.wait() finally: semaphore.release() finished.set() tox_env.status = ( "skipped tests" if config.option.notest else ("parallel child exit code {}".format(res) if res else res) ) done.add(env_name) outcome = spinner.succeed if config.option.notest: outcome = spinner.skip elif res: outcome = spinner.fail outcome(env_name) if not live_out: out, err = process.communicate() if res or tox_env.envconfig.parallel_show_output: outcome = ( "Failed {} under process {}, stdout:\n".format(env_name, process.pid) if res else "" ) message = "{}{}{}".format( outcome, out, "\nstderr:\n{}".format(err) if err else "" ).rstrip() reporter.quiet(message) threads = [] todo_keys = set(venv_dict.keys()) todo = OrderedDict((n, todo_keys & set(v.envconfig.depends)) for n, v in venv_dict.items()) done = set() while todo: for name, depends in list(todo.items()): if depends - done: # skip if has unfinished dependencies continue del todo[name] venv = venv_dict[name] semaphore.acquire(blocking=True) spinner.add(name) thread = Thread(target=run_in_thread, args=(venv, os.environ.copy())) thread.start() threads.append(thread) if todo: # wait until someone finishes and retry queuing jobs finished.wait() finished.clear() for thread in threads: thread.join()