def sample_build_commands():
retval = networkx.DiGraph()
node1 = build_tree.DependencyNode(packages=["package1a", "package1b"],
build_command=build_tree.BuildCommand(
"recipe1",
"repo1", ["package1a", "package1b"],
python="2.6",
build_type="cuda",
mpi_type="openmpi",
cudatoolkit="10.2"))
node2 = build_tree.DependencyNode(packages=["package2a"],
build_command=build_tree.BuildCommand(
"recipe2",
"repo2", ["package2a"],
python="2.6",
build_type="cpu",
mpi_type="openmpi",
cudatoolkit="10.2"))
node3 = build_tree.DependencyNode(packages=["package3a", "package3b"],
build_command=build_tree.BuildCommand(
"recipe3", "repo3",
["package3a", "package3b"]))
retval.add_node(node1)
retval.add_node(node2)
retval.add_node(node3)
retval.add_edge(node1, node2)
retval.add_edge(node1, node3)
retval.add_edge(node3, node2)
return retval
def test_get_installable_package_for_non_runtime_package():
'''
Tests that `get_installable_package` doesn't return the packages marked as
non-runtime i.e. build command with runtime_package=False.
'''
build_commands = [
build_tree.BuildCommand("recipe1",
"repo1", ["package1a"],
runtime_package=False,
python="2.6",
build_type="cuda",
mpi_type="openmpi",
cudatoolkit="10.2",
build_command_dependencies=[1]),
build_tree.BuildCommand("recipe2",
"repo2", ["package2a"],
python="2.6",
build_type="cpu",
mpi_type="openmpi",
cudatoolkit="10.2",
build_command_dependencies=[])
]
external_deps = [
"external_pac1 1.2", "external_pack2", "external_pack3=1.2.3"
]
packages = build_tree.get_installable_packages(build_commands,
external_deps)
assert not "package1a" in packages
def test_get_installable_package_with_no_duplicates():
'''
This test verifies that get_installable_package doesn't return duplicate dependencies.
'''
build_commands = graph.OpenCEGraph()
node1 = build_tree.DependencyNode(
packages=["package1a"],
build_command=build_tree.BuildCommand(
"recipe1",
"repo1", ["package1a"],
runtime_package=False,
output_files=["package1a-1.0-py37_4.tar.bz2"],
run_dependencies=["python 3.7", "pack1a==1.0"]))
node2 = build_tree.DependencyNode(
packages=["package2a"],
build_command=build_tree.BuildCommand(
"recipe2",
"repo2", ["package2a"],
output_files=["package2a-2.0-py37.tar.bz2"],
run_dependencies=[
"python 3.7", "pack1 ==1.0", "pack1", "pack2 <=2.0",
"pack2 2.0", "pack3 3.0.*", "pack2"
]))
node3 = build_tree.DependencyNode(packages=["package3a", "package3b"],
build_command=build_tree.BuildCommand(
"recipe3",
"repo3", ["package3a", "package3b"],
output_files=[
"package3a-1.5-cpu.tar.bz2",
"package3b-1.5-cpu.tar.bz2"
],
run_dependencies=[
"pack1 >=1.0", "pack1",
"pack4 <=2.0", "pack2 2.0",
"pack3 3.0.*", "pack4"
]))
build_commands.add_node(node1)
build_commands.add_node(node2)
build_commands.add_node(node3)
external_deps = ["external_pac1 1.2"]
for dep in external_deps:
build_commands.add_node(build_tree.DependencyNode({dep}))
packages = build_tree.get_installable_packages(build_commands,
external_deps)
assert not "package1a" in str(packages)
assert not "pack1a" in str(packages)
print("Packages: ", packages)
expected_packages = [
"package2a 2.0.* py37", "python 3.7.*", "pack1 ==1.0.*", "pack2 <=2.0",
"pack3 3.0.*", "package3a 1.5.* cpu", "package3b 1.5.* cpu",
"pack4 <=2.0", "external_pac1 1.2.*"
]
assert Counter(packages) == Counter(expected_packages)
def test_build_tree_cycle_fail():
'''
Tests that a cycle is detected in a build_tree.
'''
cycle_build_commands = networkx.DiGraph()
node1 = build_tree.DependencyNode(packages=["package1a", "package1b"],
build_command=build_tree.BuildCommand(
"recipe1",
"repo1", ["package1a", "package1b"],
python="2.6",
build_type="cuda",
mpi_type="openmpi",
cudatoolkit="10.2"))
node2 = build_tree.DependencyNode(packages=["package2a"],
build_command=build_tree.BuildCommand(
"recipe2",
"repo2", ["package2a"],
python="2.6",
build_type="cpu",
mpi_type="openmpi",
cudatoolkit="10.2"))
node3 = build_tree.DependencyNode(packages=["package3a", "package3b"],
build_command=build_tree.BuildCommand(
"recipe3", "repo3",
["package3a", "package3b"]))
cycle_build_commands.add_node(node1)
cycle_build_commands.add_node(node2)
cycle_build_commands.add_node(node3)
cycle_build_commands.add_edge(node1, node2)
cycle_build_commands.add_edge(node1, node3)
cycle_build_commands.add_edge(node2, node1)
cycle_build_commands.add_edge(node3, node2)
mock_build_tree = TestBuildTree([], "3.6", "cpu", "openmpi", "10.2")
mock_build_tree._tree = sample_build_commands()
mock_build_tree._detect_cycle() #Make sure there isn't a false positive.
mock_build_tree._tree = cycle_build_commands
with pytest.raises(OpenCEError) as exc:
mock_build_tree._detect_cycle()
assert "Build dependencies should form a Directed Acyclic Graph." in str(
exc.value)
assert any([
"recipe1 -> recipe2 -> recipe1" in str(exc.value),
"recipe2 -> recipe1 -> recipe2" in str(exc.value),
"recipe1 -> recipe3 -> recipe2 -> recipe1" in str(exc.value),
"recipe2 -> recipe1 -> recipe3 -> recipe2" in str(exc.value),
"recipe3 -> recipe2 -> recipe1 -> recipe2" in str(exc.value)
])
def sample_build_commands():
retval = graph.OpenCEGraph()
node1 = build_tree.DependencyNode(packages=["package1a", "package1b"],
build_command=build_tree.BuildCommand(
"recipe1",
"repo1", ["package1a", "package1b"],
python="2.6",
build_type="cuda",
mpi_type="openmpi",
cudatoolkit="10.2",
output_files=[
"package1a-py26-cuda-openmpi",
"package1b-py26-cuda-openmpi"
]))
node2 = build_tree.DependencyNode(packages=["package2a"],
build_command=build_tree.BuildCommand(
"recipe2",
"repo2", ["package2a"],
python="2.6",
build_type="cpu",
mpi_type="openmpi",
cudatoolkit="10.2",
output_files=["package2a-noarch"]))
node3 = build_tree.DependencyNode(packages=["package3a", "package3b"],
build_command=build_tree.BuildCommand(
"recipe3",
"repo3", ["package3a", "package3b"],
output_files=[
"package3a-py26-cpu-openmpi",
"package3b-py26-cpu-openmpi"
]))
# This node is a duplicate and nothing should happen when it's added.
node4 = build_tree.DependencyNode(packages=["package2a"],
build_command=build_tree.BuildCommand(
"recipe2",
"repo2", ["package2a"],
python="3.6",
build_type="cuda",
mpi_type="system",
cudatoolkit="10.2",
output_files=["package2a-noarch"]))
retval.add_node(node1)
retval.add_node(node2)
retval.add_node(node3)
retval.add_node(node4)
retval.add_edge(node1, node4)
retval.add_edge(node1, node3)
retval.add_edge(node3, node2)
return retval
def test_feedstock_args():
'''
Tests that feedstock_args creates the correct arguments.
'''
build_commands = [
build_tree.BuildCommand("recipe", "repo", {"pkg1", "pkg2"}),
build_tree.BuildCommand(
"recipe2",
"repo2", {"pkg1", "pkg2"},
python="3.2",
mpi_type="system",
build_type="cuda",
cudatoolkit="10.0",
conda_build_configs=["conda_build_config5.yaml"])
]
for build_command in build_commands:
build_string = " ".join(build_command.feedstock_args())
assert "--working_directory {}".format(
build_command.repository) in build_string
if build_command.recipe:
assert "--recipes {}".format(build_command.recipe) in build_string
if build_command.channels:
for channel in build_command.channels:
assert "--channels {}".format(channel) in build_string
if build_command.python:
assert "--python_versions {}".format(
build_command.python) in build_string
if build_command.build_type:
assert "--build_types {}".format(
build_command.build_type) in build_string
if build_command.mpi_type:
assert "--mpi_types {}".format(
build_command.mpi_type) in build_string
if build_command.cudatoolkit:
assert "--cuda_versions {}".format(
build_command.cudatoolkit) in build_string
if build_command.conda_build_configs:
assert "--conda_build_configs \'{}\'".format(",".join(
build_command.conda_build_configs)) in build_string
def test_get_installable_package_for_non_runtime_package():
'''
Tests that `get_installable_package` doesn't return the packages marked as
non-runtime i.e. build command with runtime_package=False.
'''
build_commands = graph.OpenCEGraph()
node1 = build_tree.DependencyNode(
packages=["package1a"],
build_command=build_tree.BuildCommand(
"recipe1",
"repo1", ["package1a"],
runtime_package=False,
python="2.6",
build_type="cuda",
mpi_type="openmpi",
cudatoolkit="10.2",
output_files=["package1a-py26-cuda-openmpi"]))
node2 = build_tree.DependencyNode(
packages=["package2a"],
build_command=build_tree.BuildCommand(
"recipe2",
"repo2", ["package2a"],
python="2.6",
build_type="cpu",
mpi_type="openmpi",
cudatoolkit="10.2",
output_files=["package2a-py26-cuda-openmpi"]))
build_commands.add_node(node1)
build_commands.add_node(node2)
build_commands.add_edge(node1, node2)
external_deps = [
"external_pac1 1.2", "external_pack2", "external_pack3=1.2.3"
]
for dep in external_deps:
build_commands.add_node(build_tree.DependencyNode({dep}))
packages = build_tree.get_installable_packages(build_commands,
external_deps)
assert not "package1a" in packages
def test_build_tree_cycle_fail():
'''
Tests that a cycle is detected in a build_tree.
'''
cycle_build_commands = [
build_tree.BuildCommand("recipe1",
"repo1", ["package1a", "package1b"],
python="2.6",
build_type="cuda",
mpi_type="openmpi",
cudatoolkit="10.2",
build_command_dependencies=[1, 2]),
build_tree.BuildCommand("recipe2",
"repo2", ["package2a"],
python="2.6",
build_type="cpu",
mpi_type="openmpi",
cudatoolkit="10.2",
build_command_dependencies=[0]),
build_tree.BuildCommand("recipe3",
"repo3", ["package3a", "package3b"],
build_command_dependencies=[1])
]
mock_build_tree = TestBuildTree([], "3.6", "cpu", "openmpi", "10.2")
mock_build_tree.build_commands = sample_build_commands
mock_build_tree._detect_cycle() #Make sure there isn't a false positive.
mock_build_tree.build_commands = cycle_build_commands
with pytest.raises(OpenCEError) as exc:
mock_build_tree._detect_cycle()
assert "Build dependencies should form a Directed Acyclic Graph." in str(
exc.value)
assert "recipe1 -> recipe2 -> recipe1" in str(exc.value)
assert "recipe1 -> recipe3 -> recipe2 -> recipe1" in str(exc.value)
assert "recipe2 -> recipe1 -> recipe3 -> recipe2" in str(exc.value)
assert "recipe3 -> recipe2 -> recipe1 -> recipe2" in str(exc.value)
def test_get_installable_package_with_no_duplicates():
'''
This test verifies that get_installable_package doesn't return duplicate dependencies.
'''
build_commands = [
build_tree.BuildCommand("recipe1",
"repo1", ["package1a"],
runtime_package=False,
run_dependencies=["python 3.7",
"pack1a==1.0"]),
build_tree.BuildCommand("recipe2",
"repo2", ["package2a"],
run_dependencies=[
"python 3.7", "pack1 ==1.0", "pack1",
"pack2 <=2.0", "pack2 2.0",
"pack3 3.0.*", "pack2"
]),
build_tree.BuildCommand("recipe3",
"repo3", ["package3a", "package3b"],
run_dependencies=[
"pack1 >=1.0", "pack1", "pack4 <=2.0",
"pack2 2.0", "pack3 3.0.*", "pack4"
])
]
external_deps = ["external_pac1 1.2"]
packages = build_tree.get_installable_packages(build_commands,
external_deps)
assert not "package1a" in packages
assert not "pack1a" in packages
print("Packages: ", packages)
expected_packages = [
"package2a", "python 3.7.*", "pack1 ==1.0.*", "pack2 <=2.0",
"pack3 3.0.*", "package3a", "package3b", "pack4 <=2.0",
"external_pac1 1.2.*"
]
assert Counter(packages) == Counter(expected_packages)
def sample_build_commands():
retval = networkx.DiGraph()
node1 = build_tree.DependencyNode(
packages=["package1a", "package1b"],
build_command=build_tree.BuildCommand(
"recipe1",
"repo1", ["package1a", "package1b"],
python="3.6",
build_type="cuda",
mpi_type="openmpi",
cudatoolkit="10.2",
output_files=[
"package1a-1.0-py36_cuda10.2.tar.bz2",
"package1b-1.0-py36_cuda10.2.tar.bz2"
],
run_dependencies=[
"python >=3.6", "pack1 1.0", "pack2 >=2.0", "pack3 9b"
]))
node2 = build_tree.DependencyNode(
packages=["package2a"],
build_command=build_tree.BuildCommand(
"recipe2",
"repo2", ["package2a"],
python="3.6",
build_type="cpu",
mpi_type="system",
cudatoolkit="10.2",
output_files=["package2a-1.0-py36_cuda10.2.tar.bz2"],
run_dependencies=[
"python ==3.6", "pack1 >=1.0", "pack2 ==2.0",
"pack3 3.3 build"
]))
node3 = build_tree.DependencyNode(
packages=["package3a", "package3b"],
build_command=build_tree.BuildCommand(
"recipe3",
"repo3", ["package3a", "package3b"],
python="3.7",
build_type="cpu",
mpi_type="openmpi",
cudatoolkit="10.2",
output_files=[
"package3a-1.0-py37_cuda10.2.tar.bz2",
"package3b-1.0-py37_cuda10.2.tar.bz2"
],
run_dependencies=[
"python 3.7", "pack1==1.0", "pack2 <=2.0", "pack3 3.0.*",
"pack4=1.15.0=py38h6ffa863_0"
]))
node4 = build_tree.DependencyNode(
packages=["package4a", "package4b"],
build_command=build_tree.BuildCommand(
"recipe4",
"repo4", ["package4a", "package4b"],
python="3.7",
build_type="cuda",
mpi_type="system",
cudatoolkit="10.2",
output_files=[
"package4a-1.0-py37_cuda.tar.bz2",
"package4b-1.0-py37_cuda.tar.bz2"
],
run_dependencies=["pack1==1.0", "pack2 <=2.0",
"pack3-suffix 3.0"]))
retval.add_node(node1)
retval.add_node(node2)
retval.add_node(node3)
retval.add_node(node4)
for dep in external_deps:
retval.add_node(build_tree.DependencyNode({dep}))
return retval
def test_build_tree_duplicates():
'''
Tests that `build_tree._add_build_command_dependencies` removes duplicate build_commands
and sets the `build_command_dependencies` accordingly.
'''
initial_build_commands = [
build_tree.BuildCommand("recipe1",
"repo1", ["package1a"],
output_files=["linux/package1a.tar.gz"],
python="2.6",
build_type="cuda",
mpi_type="openmpi",
build_command_dependencies=[],
run_dependencies=[],
build_dependencies=[],
host_dependencies=[],
test_dependencies=[]),
build_tree.BuildCommand("recipe2",
"repo2", ["package2a"],
output_files=["linux/package2a.tar.gz"],
python="2.6",
build_type="cuda",
mpi_type="openmpi",
build_command_dependencies=[0],
run_dependencies=[],
build_dependencies=[],
host_dependencies=[],
test_dependencies=[])
]
duplicate_build_commands = [
build_tree.BuildCommand("recipe2",
"repo2", ["package2a"],
output_files=["linux/package2a.tar.gz"],
python="2.6",
build_type="cuda",
mpi_type="openmpi",
build_command_dependencies=[],
run_dependencies=[],
build_dependencies=[],
host_dependencies=[],
test_dependencies=[]),
build_tree.BuildCommand("recipe1",
"repo1", ["package1a"],
output_files=["linux/package1a.tar.gz"],
python="2.6",
build_type="cuda",
mpi_type="openmpi",
build_command_dependencies=[],
run_dependencies=[],
build_dependencies=[],
host_dependencies=[],
test_dependencies=[]),
build_tree.BuildCommand("recipe3",
"repo3", ["package3a"],
output_files=["linux/package3a.tar.gz"],
python="2.6",
build_type="cpu",
mpi_type="openmpi",
build_command_dependencies=[1],
run_dependencies=[],
build_dependencies=["package1a"],
host_dependencies=[],
test_dependencies=[])
]
additional_build_commands = [
build_tree.BuildCommand("recipe4",
"repo4", ["package4a"],
output_files=["linux/package4a.tar.gz"],
python="2.6",
build_type="cpu",
mpi_type="openmpi",
build_command_dependencies=[],
run_dependencies=[],
build_dependencies=[],
host_dependencies=[],
test_dependencies=[])
]
out_commands, _ = build_tree._add_build_command_dependencies(
additional_build_commands, initial_build_commands,
len(initial_build_commands))
assert len(
out_commands) == 1 # Make sure the non-duplicates are not removed
out_commands, _ = build_tree._add_build_command_dependencies(
duplicate_build_commands, initial_build_commands,
len(initial_build_commands))
assert len(out_commands) == 1
for build_command in out_commands:
assert build_command.build_command_dependencies == [0]
# test-env1.yaml has defined "recipe_path" as "package11_recipe_path" for "package11".
env_config_data_list = env_config.load_env_config_files([env_file],
variant)
for env_config_data in env_config_data_list:
packages = env_config_data.get(env_config.Key.packages.name, [])
for package in packages:
if package.get(env_config.Key.feedstock.name) == "package11":
assert package.get(env_config.Key.recipe_path.name
) == "package11_recipe_path"
sample_build_commands = [
build_tree.BuildCommand("recipe1",
"repo1", ["package1a", "package1b"],
python="2.6",
build_type="cuda",
mpi_type="openmpi",
cudatoolkit="10.2",
build_command_dependencies=[1, 2]),
build_tree.BuildCommand("recipe2",
"repo2", ["package2a"],
python="2.6",
build_type="cpu",
mpi_type="openmpi",
cudatoolkit="10.2",
build_command_dependencies=[]),
build_tree.BuildCommand("recipe3",
"repo3", ["package3a", "package3b"],
build_command_dependencies=[1])
]
import os
import pathlib
from collections import Counter
test_dir = pathlib.Path(__file__).parent.absolute()
import open_ce.build_tree as build_tree
import open_ce.conda_env_file_generator as conda_env_file_generator
import open_ce.utils as utils
sample_build_commands = [
build_tree.BuildCommand("recipe1",
"repo1", ["package1a", "package1b"],
python="3.6",
build_type="cuda",
mpi_type="openmpi",
cudatoolkit="10.2",
run_dependencies=[
"python >=3.6", "pack1 1.0",
"pack2 >=2.0", "pack3 9b"
]),
build_tree.BuildCommand("recipe2",
"repo2", ["package2a"],
python="3.6",
build_type="cpu",
mpi_type="system",
cudatoolkit="10.2",
run_dependencies=[
"python ==3.6", "pack1 >=1.0", "pack2 ==2.0",
"pack3 3.3 build"
]),
build_tree.BuildCommand("recipe3",