def tox_testenv_create(venv, action):
config_interpreter = venv.getsupportedinterpreter()
args = [sys.executable, "-m", "virtualenv"]
if venv.envconfig.sitepackages:
args.append("--system-site-packages")
if venv.envconfig.alwayscopy:
args.append("--always-copy")
if not venv.envconfig.download:
args.append("--no-download")
# add interpreter explicitly, to prevent using default (virtualenv.ini)
args.extend(["--python", str(config_interpreter)])
cleanup_for_venv(venv)
base_path = venv.path.dirpath()
base_path.ensure(dir=1)
args.append(venv.path.basename)
if not _SKIP_VENV_CREATION:
try:
venv._pcall(
args,
venv=False,
action=action,
cwd=base_path,
redirect=reporter.verbosity() < reporter.Verbosity.DEBUG,
)
except KeyboardInterrupt:
venv.status = "keyboardinterrupt"
raise
return True # Return non-None to indicate plugin has completed
def tox_testenv_create(venv, action):
config_interpreter = venv.getsupportedinterpreter()
args = [sys.executable, "-m", "virtualenv"]
if venv.envconfig.sitepackages:
args.append("--system-site-packages")
if venv.envconfig.alwayscopy:
args.append("--always-copy")
if NO_DOWNLOAD:
args.append("--no-download")
# add interpreter explicitly, to prevent using default (virtualenv.ini)
args.extend(["--python", str(config_interpreter)])
within_parallel = PARALLEL_ENV_VAR_KEY in os.environ
if within_parallel:
if venv.path.exists():
# do not delete the log folder as that's used by parent
for content in venv.path.listdir():
if not content.basename == "log":
content.remove(rec=1, ignore_errors=True)
else:
ensure_empty_dir(venv.path)
basepath = venv.path.dirpath()
basepath.ensure(dir=1)
args.append(venv.path.basename)
if not _SKIP_VENV_CREATION:
venv._pcall(
args,
venv=False,
action=action,
cwd=basepath,
redirect=reporter.verbosity() < reporter.Verbosity.DEBUG,
)
return True # Return non-None to indicate plugin has completed
def run_install_command(self, packages, action, options=()):
def expand(val):
# expand an install command
if val == "{packages}":
for package in packages:
yield package
elif val == "{opts}":
for opt in options:
yield opt
else:
yield val
cmd = list(
chain.from_iterable(
expand(val) for val in self.envconfig.install_command))
env = self._get_os_environ()
self.ensure_pip_os_environ_ok(env)
old_stdout = sys.stdout
sys.stdout = codecs.getwriter("utf8")(sys.stdout)
try:
self._pcall(
cmd,
cwd=self.envconfig.config.toxinidir,
action=action,
redirect=reporter.verbosity() < reporter.Verbosity.DEBUG,
env=env,
)
finally:
sys.stdout = old_stdout
def show_config(config):
parser = configparser.ConfigParser()
if not config.envlist_explicit or reporter.verbosity() >= reporter.Verbosity.INFO:
tox_info(config, parser)
version_info(parser)
tox_envs_info(config, parser)
content = StringIO()
parser.write(content)
value = content.getvalue().rstrip()
reporter.verbosity0(value)
def install_pkg(self, dir, action, name, is_develop=False):
assert action is not None
if getattr(self, "just_created", False):
action.setactivity(name, dir)
self.finish()
pip_flags = ["--exists-action", "w"]
else:
if is_develop and not self._needs_reinstall(dir, action):
action.setactivity("{}-noop".format(name), dir)
return
action.setactivity("{}-nodeps".format(name), dir)
pip_flags = ["--no-deps"] + ([] if is_develop else ["-U"])
pip_flags.extend(["-v"] * min(3, reporter.verbosity() - 2))
if self.envconfig.extras:
dir += "[{}]".format(",".join(self.envconfig.extras))
target = [dir]
if is_develop:
target.insert(0, "-e")
self._install(target, extraopts=pip_flags, action=action)
def get_pip_install(venv, packages, use_develop=False):
flags = ["-e"] if use_develop else []
flags.extend(["-v"] * min(3, reporter.verbosity() - 2))
if not packages:
return None
pip_install = list(venv.envconfig.install_command)
try:
index = pip_install.index("{opts}")
pip_install = list(
chain(pip_install[:index], flags, pip_install[index + 1:]))
except ValueError: # pragma: no cover
pass # pragma: no cover
try:
index = pip_install.index("{packages}")
pip_install = list(
chain(pip_install[:index], (i.name for i in packages),
pip_install[index + 1:]))
except ValueError: # pragma: no cover
pass # pragma: no cover
if "PIP_INDEX_URL" in venv.env:
index_url = venv.env["PIP_INDEX_URL"]
pip_install.extend(("-i", index_url))
return pip_install
def run_parallel(config, venv_dict):
"""here we'll just start parallel sub-processes"""
live_out = config.option.parallel_live
disable_spinner = bool(os.environ.get("TOX_PARALLEL_NO_SPINNER") == "1")
args = [sys.executable, MAIN_FILE] + config.args
try:
position = args.index("--")
except ValueError:
position = len(args)
max_parallel = config.option.parallel
if max_parallel is None:
max_parallel = len(venv_dict)
semaphore = Semaphore(max_parallel)
finished = Event()
show_progress = (not disable_spinner and not live_out
and reporter.verbosity() > reporter.Verbosity.QUIET)
with Spinner(enabled=show_progress) as spinner:
def run_in_thread(tox_env, os_env, processes):
output = None
print_out = None
env_name = tox_env.envconfig.envname
status = "skipped tests" if config.option.notest else None
try:
os_env[str(PARALLEL_ENV_VAR_KEY_PRIVATE)] = str(env_name)
os_env[str(PARALLEL_ENV_VAR_KEY_PUBLIC)] = str(env_name)
args_sub = list(args)
if hasattr(tox_env, "package"):
args_sub.insert(position, str(tox_env.package))
args_sub.insert(position, "--installpkg")
if tox_env.get_result_json_path():
result_json_index = args_sub.index("--result-json")
args_sub[result_json_index + 1] = "{}".format(
tox_env.get_result_json_path())
with tox_env.new_action("parallel {}".format(
tox_env.name)) as action:
def collect_process(process):
processes[tox_env] = (action, process)
print_out = not live_out and tox_env.envconfig.parallel_show_output
output = action.popen(
args=args_sub,
env=os_env,
redirect=not live_out,
capture_err=print_out,
callback=collect_process,
returnout=print_out,
)
except InvocationError as err:
status = "parallel child exit code {}".format(err.exit_code)
finally:
semaphore.release()
finished.set()
tox_env.status = status
done.add(env_name)
outcome = spinner.succeed
if config.option.notest:
outcome = spinner.skip
elif status is not None:
outcome = spinner.fail
outcome(env_name)
if print_out and output is not None:
reporter.verbosity0(output)
threads = deque()
processes = {}
todo_keys = set(venv_dict.keys())
todo = OrderedDict((n, todo_keys & set(v.envconfig.depends))
for n, v in venv_dict.items())
done = set()
try:
while todo:
for name, depends in list(todo.items()):
if depends - done:
# skip if has unfinished dependencies
continue
del todo[name]
venv = venv_dict[name]
semaphore.acquire(blocking=True)
spinner.add(name)
thread = Thread(
target=run_in_thread,
args=(venv, os.environ.copy(), processes),
)
thread.daemon = True
thread.start()
threads.append(thread)
if todo:
# wait until someone finishes and retry queuing jobs
finished.wait()
finished.clear()
while threads:
threads = [
thread for thread in threads
if not thread.join(0.1) and thread.is_alive()
]
except KeyboardInterrupt:
reporter.verbosity0(
"[{}] KeyboardInterrupt parallel - stopping children".format(
os.getpid()), )
while True:
# do not allow to interrupt until children interrupt
try:
# putting it inside a thread so it's not interrupted
stopper = Thread(target=_stop_child_processes,
args=(processes, threads))
stopper.start()
stopper.join()
except KeyboardInterrupt:
continue
raise KeyboardInterrupt
def run_parallel(config, venv_dict):
"""here we'll just start parallel sub-processes"""
live_out = config.option.parallel_live
args = [sys.executable, "-m", "tox"] + config.args
try:
position = args.index("--")
except ValueError:
position = len(args)
try:
parallel_at = args[0:position].index("--parallel")
del args[parallel_at]
position -= 1
except ValueError:
pass
max_parallel = config.option.parallel
if max_parallel is None:
max_parallel = len(venv_dict)
semaphore = Semaphore(max_parallel)
finished = Event()
sink = None if live_out else subprocess.PIPE
show_progress = not live_out and reporter.verbosity() > reporter.Verbosity.QUIET
with Spinner(enabled=show_progress) as spinner:
def run_in_thread(tox_env, os_env):
res = None
env_name = tox_env.envconfig.envname
try:
os_env[str(PARALLEL_ENV_VAR_KEY)] = str(env_name)
args_sub = list(args)
if hasattr(tox_env, "package"):
args_sub.insert(position, str(tox_env.package))
args_sub.insert(position, "--installpkg")
process = subprocess.Popen(
args_sub,
env=os_env,
stdout=sink,
stderr=sink,
stdin=None,
universal_newlines=True,
)
res = process.wait()
finally:
semaphore.release()
finished.set()
tox_env.status = (
"skipped tests"
if config.option.notest
else ("parallel child exit code {}".format(res) if res else res)
)
done.add(env_name)
outcome = spinner.succeed
if config.option.notest:
outcome = spinner.skip
elif res:
outcome = spinner.fail
outcome(env_name)
if not live_out:
out, err = process.communicate()
if res or tox_env.envconfig.parallel_show_output:
outcome = (
"Failed {} under process {}, stdout:\n".format(env_name, process.pid)
if res
else ""
)
message = "{}{}{}".format(
outcome, out, "\nstderr:\n{}".format(err) if err else ""
).rstrip()
reporter.quiet(message)
threads = []
todo_keys = set(venv_dict.keys())
todo = OrderedDict((n, todo_keys & set(v.envconfig.depends)) for n, v in venv_dict.items())
done = set()
while todo:
for name, depends in list(todo.items()):
if depends - done:
# skip if has unfinished dependencies
continue
del todo[name]
venv = venv_dict[name]
semaphore.acquire(blocking=True)
spinner.add(name)
thread = Thread(target=run_in_thread, args=(venv, os.environ.copy()))
thread.start()
threads.append(thread)
if todo:
# wait until someone finishes and retry queuing jobs
finished.wait()
finished.clear()
for thread in threads:
thread.join()