def conda_envs():
"""Creates two conda env with different requests versions."""
conda_path = get_conda_bin_executable("conda")
init_cmd = f". {os.path.dirname(conda_path)}" f"/../etc/profile.d/conda.sh"
def delete_env(env_name):
subprocess.run(["conda", "remove", "--name", env_name, "--all", "-y"])
def create_package_env(env_name, package_version: str):
delete_env(env_name)
subprocess.run(
["conda", "create", "-n", env_name, "-y", f"python={_current_py_version()}"]
)
_inject_ray_to_conda_site(get_conda_env_dir(env_name))
ray_deps: List[str] = _resolve_install_from_source_ray_dependencies()
ray_deps.append(f"requests=={package_version}")
with tempfile.NamedTemporaryFile("w") as f:
f.writelines([line + "\n" for line in ray_deps])
f.flush()
commands = [
init_cmd,
f"conda activate {env_name}",
f"python -m pip install -r {f.name}",
"conda deactivate",
]
proc = subprocess.run(
[" && ".join(commands)],
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
if proc.returncode != 0:
print("pip install failed")
print(proc.stdout.decode())
print(proc.stderr.decode())
assert False
for package_version in REQUEST_VERSIONS:
create_package_env(
env_name=f"package-{package_version}", package_version=package_version
)
yield
for package_version in REQUEST_VERSIONS:
delete_env(env_name=f"package-{package_version}")
def conda_envs():
"""Creates two copies of current conda env with different tf versions."""
ray.init()
conda_path = get_conda_bin_executable("conda")
init_cmd = (f". {os.path.dirname(conda_path)}"
f"/../etc/profile.d/conda.sh")
subprocess.run([f"{init_cmd} && conda activate"], shell=True)
current_conda_env = os.environ.get("CONDA_DEFAULT_ENV")
assert current_conda_env is not None
def delete_env(env_name):
subprocess.run(["conda", "remove", "--name", env_name, "--all", "-y"])
# Cloning the env twice may take minutes, so parallelize with Ray.
@ray.remote
def create_tf_env(tf_version: str):
env_name = f"tf-{tf_version}"
delete_env(env_name)
subprocess.run([
"conda", "create", "-n", env_name, "--clone", current_conda_env,
"-y"
])
commands = [
init_cmd, f"conda activate {env_name}",
f"python -m pip install tensorflow=={tf_version}",
"conda deactivate"
]
command_separator = " && "
command_str = command_separator.join(commands)
subprocess.run([command_str], shell=True)
tf_versions = ["2.2.0", "2.3.0"]
ray.get([create_tf_env.remote(version) for version in tf_versions])
ray.shutdown()
yield
ray.init()
for tf_version in tf_versions:
delete_env(env_name=f"tf-{tf_version}")
subprocess.run([f"{init_cmd} && conda deactivate"], shell=True)
ray.shutdown()
def conda_envs(tmp_path_factory):
"""Creates two conda env with different requests versions."""
conda_path = get_conda_bin_executable("conda")
init_cmd = f". {os.path.dirname(conda_path)}" f"/../etc/profile.d/conda.sh"
def delete_env(env_name):
subprocess.run(["conda", "remove", "--name", env_name, "--all", "-y"])
def create_package_env(env_name, package_version: str):
delete_env(env_name)
proc = subprocess.run(
[
"conda",
"create",
"-n",
env_name,
"-y",
f"python={_current_py_version()}",
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
if proc.returncode != 0:
print("conda create failed, returned %d" % proc.returncode)
print(proc.stdout.decode())
print(proc.stderr.decode())
assert False
_inject_ray_to_conda_site(get_conda_env_dir(env_name))
ray_deps: List[str] = _resolve_install_from_source_ray_dependencies()
ray_deps.append(f"requests=={package_version}")
reqs = tmp_path_factory.mktemp("reqs") / "requirements.txt"
with reqs.open("wt") as fid:
for line in ray_deps:
fid.write(line)
fid.write("\n")
commands = [
f"conda activate {env_name}",
f"python -m pip install -r {str(reqs)}",
"conda deactivate",
]
if _WIN32:
# as a string
command = " && ".join(commands)
else:
commands.insert(0, init_cmd)
# as a list
command = [" && ".join(commands)]
proc = subprocess.run(
command,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
if proc.returncode != 0:
print("conda/pip install failed, returned %d" % proc.returncode)
print("command", command)
print(proc.stdout.decode())
print(proc.stderr.decode())
assert False
for package_version in REQUEST_VERSIONS:
create_package_env(
env_name=f"package-{package_version}", package_version=package_version
)
yield
for package_version in REQUEST_VERSIONS:
delete_env(env_name=f"package-{package_version}")
