def sample_build_commands():
retval = networkx.DiGraph()
node1 = build_tree.DependencyNode(packages=["package1a", "package1b"],
build_command=build_tree.BuildCommand(
"recipe1",
"repo1", ["package1a", "package1b"],
python="2.6",
build_type="cuda",
mpi_type="openmpi",
cudatoolkit="10.2"))
node2 = build_tree.DependencyNode(packages=["package2a"],
build_command=build_tree.BuildCommand(
"recipe2",
"repo2", ["package2a"],
python="2.6",
build_type="cpu",
mpi_type="openmpi",
cudatoolkit="10.2"))
node3 = build_tree.DependencyNode(packages=["package3a", "package3b"],
build_command=build_tree.BuildCommand(
"recipe3", "repo3",
["package3a", "package3b"]))
retval.add_node(node1)
retval.add_node(node2)
retval.add_node(node3)
retval.add_edge(node1, node2)
retval.add_edge(node1, node3)
retval.add_edge(node3, node2)
return retval
def test_get_installable_package_with_no_duplicates():
'''
This test verifies that get_installable_package doesn't return duplicate dependencies.
'''
build_commands = graph.OpenCEGraph()
node1 = build_tree.DependencyNode(
packages=["package1a"],
build_command=build_tree.BuildCommand(
"recipe1",
"repo1", ["package1a"],
runtime_package=False,
output_files=["package1a-1.0-py37_4.tar.bz2"],
run_dependencies=["python 3.7", "pack1a==1.0"]))
node2 = build_tree.DependencyNode(
packages=["package2a"],
build_command=build_tree.BuildCommand(
"recipe2",
"repo2", ["package2a"],
output_files=["package2a-2.0-py37.tar.bz2"],
run_dependencies=[
"python 3.7", "pack1 ==1.0", "pack1", "pack2 <=2.0",
"pack2 2.0", "pack3 3.0.*", "pack2"
]))
node3 = build_tree.DependencyNode(packages=["package3a", "package3b"],
build_command=build_tree.BuildCommand(
"recipe3",
"repo3", ["package3a", "package3b"],
output_files=[
"package3a-1.5-cpu.tar.bz2",
"package3b-1.5-cpu.tar.bz2"
],
run_dependencies=[
"pack1 >=1.0", "pack1",
"pack4 <=2.0", "pack2 2.0",
"pack3 3.0.*", "pack4"
]))
build_commands.add_node(node1)
build_commands.add_node(node2)
build_commands.add_node(node3)
external_deps = ["external_pac1 1.2"]
for dep in external_deps:
build_commands.add_node(build_tree.DependencyNode({dep}))
packages = build_tree.get_installable_packages(build_commands,
external_deps)
assert not "package1a" in str(packages)
assert not "pack1a" in str(packages)
print("Packages: ", packages)
expected_packages = [
"package2a 2.0.* py37", "python 3.7.*", "pack1 ==1.0.*", "pack2 <=2.0",
"pack3 3.0.*", "package3a 1.5.* cpu", "package3b 1.5.* cpu",
"pack4 <=2.0", "external_pac1 1.2.*"
]
assert Counter(packages) == Counter(expected_packages)
def test_build_tree_cycle_fail():
'''
Tests that a cycle is detected in a build_tree.
'''
cycle_build_commands = networkx.DiGraph()
node1 = build_tree.DependencyNode(packages=["package1a", "package1b"],
build_command=build_tree.BuildCommand(
"recipe1",
"repo1", ["package1a", "package1b"],
python="2.6",
build_type="cuda",
mpi_type="openmpi",
cudatoolkit="10.2"))
node2 = build_tree.DependencyNode(packages=["package2a"],
build_command=build_tree.BuildCommand(
"recipe2",
"repo2", ["package2a"],
python="2.6",
build_type="cpu",
mpi_type="openmpi",
cudatoolkit="10.2"))
node3 = build_tree.DependencyNode(packages=["package3a", "package3b"],
build_command=build_tree.BuildCommand(
"recipe3", "repo3",
["package3a", "package3b"]))
cycle_build_commands.add_node(node1)
cycle_build_commands.add_node(node2)
cycle_build_commands.add_node(node3)
cycle_build_commands.add_edge(node1, node2)
cycle_build_commands.add_edge(node1, node3)
cycle_build_commands.add_edge(node2, node1)
cycle_build_commands.add_edge(node3, node2)
mock_build_tree = TestBuildTree([], "3.6", "cpu", "openmpi", "10.2")
mock_build_tree._tree = sample_build_commands()
mock_build_tree._detect_cycle() #Make sure there isn't a false positive.
mock_build_tree._tree = cycle_build_commands
with pytest.raises(OpenCEError) as exc:
mock_build_tree._detect_cycle()
assert "Build dependencies should form a Directed Acyclic Graph." in str(
exc.value)
assert any([
"recipe1 -> recipe2 -> recipe1" in str(exc.value),
"recipe2 -> recipe1 -> recipe2" in str(exc.value),
"recipe1 -> recipe3 -> recipe2 -> recipe1" in str(exc.value),
"recipe2 -> recipe1 -> recipe3 -> recipe2" in str(exc.value),
"recipe3 -> recipe2 -> recipe1 -> recipe2" in str(exc.value)
])
def test_search_package_priority(mocker):
'''
Test remote package priority.
'''
mock_build_tree = TestBuildTree([],
"3.6",
"cpu",
"openmpi",
"10.2",
channels=["defaults"])
dep_graph = sample_build_commands()
external_node = build_tree.DependencyNode(packages=["external_package"])
nodes = list(dep_graph.nodes())
parent_node = nodes[0]
#defaults is the highest priority conda channel, a package should only come from conda_forge if there are none in defaults, regardless of version.
parent_node.build_command.channels = ["defaults", "conda_forge"]
dep_graph.add_node(external_node)
dep_graph.add_edge(parent_node, external_node)
def mocked_search(*arguments, **_):
search_results = {
("external_package", "conda_forge"):
make_search_result(package_name="external_package",
package_version="2.0",
deps=["conda_forge_dep"]),
("external_package", "defaults"):
make_search_result(package_name="external_package",
package_version="1.0",
deps=["defaults_dep"]),
("defaults_dep", "conda_forge"):
make_search_result(package_name="defaults_dep",
deps=["defaults_dep_conda_forge_dep"]),
("defaults_dep", "defaults"):
empty_search_result("defaults_dep")
}
package = arguments[1][1]
channel = arguments[1][4] if "-c" in arguments[1] else ""
if (package, channel) in search_results:
return search_results[(package, channel)]
else:
return make_search_result(package_name=package)
mocker.patch('conda.cli.python_api.run_command', side_effect=mocked_search)
dep_graph = mock_build_tree._create_remote_deps(dep_graph)
assert build_tree.DependencyNode(packages={"defaults_dep"}) in list(dep_graph.successors(external_node)), \
"defaults_dep should be found, even though the conda_forge package is a newer version"
assert build_tree.DependencyNode(packages={"defaults_dep_conda_forge_dep"}) in list(dep_graph.successors(build_tree.DependencyNode(packages={"defaults_dep"}))), \
"There is no defaults_dep package in the defaults channel, so conda_forge should be used."
def sample_build_commands():
retval = graph.OpenCEGraph()
node1 = build_tree.DependencyNode(packages=["package1a", "package1b"],
build_command=build_tree.BuildCommand(
"recipe1",
"repo1", ["package1a", "package1b"],
python="2.6",
build_type="cuda",
mpi_type="openmpi",
cudatoolkit="10.2",
output_files=[
"package1a-py26-cuda-openmpi",
"package1b-py26-cuda-openmpi"
]))
node2 = build_tree.DependencyNode(packages=["package2a"],
build_command=build_tree.BuildCommand(
"recipe2",
"repo2", ["package2a"],
python="2.6",
build_type="cpu",
mpi_type="openmpi",
cudatoolkit="10.2",
output_files=["package2a-noarch"]))
node3 = build_tree.DependencyNode(packages=["package3a", "package3b"],
build_command=build_tree.BuildCommand(
"recipe3",
"repo3", ["package3a", "package3b"],
output_files=[
"package3a-py26-cpu-openmpi",
"package3b-py26-cpu-openmpi"
]))
# This node is a duplicate and nothing should happen when it's added.
node4 = build_tree.DependencyNode(packages=["package2a"],
build_command=build_tree.BuildCommand(
"recipe2",
"repo2", ["package2a"],
python="3.6",
build_type="cuda",
mpi_type="system",
cudatoolkit="10.2",
output_files=["package2a-noarch"]))
retval.add_node(node1)
retval.add_node(node2)
retval.add_node(node3)
retval.add_node(node4)
retval.add_edge(node1, node4)
retval.add_edge(node1, node3)
retval.add_edge(node3, node2)
return retval
def test_search_channels(mocker):
'''
Test that recipe specific channels are used for remote dependency discovery.
'''
mock_build_tree = TestBuildTree([],
"3.6",
"cpu",
"openmpi",
"10.2",
channels=["defaults"])
dep_graph = sample_build_commands()
external_node = build_tree.DependencyNode(packages=["external_package"])
nodes = list(dep_graph.nodes())
parent_node = nodes[0]
parent_node.build_command.channels = ["conda_forge"]
dep_graph.add_node(external_node)
dep_graph.add_edge(parent_node, external_node)
def validate_channels(channels, package):
assert "defaults" in channels
if package == "external_package":
assert "conda_forge" in channels, \
"The conda_forge channel should come from the parent node."
assert channels.index("conda_forge") < channels.index("defaults"), \
"The conda_forge channel should be higher priority, since it is from a parent node and the defaults channel is from the env file."
return {'dependencies': []}
mocker.patch('open_ce.conda_utils.get_latest_package_info',
side_effect=validate_channels)
mock_build_tree._create_remote_deps(dep_graph)
def test_get_installable_package_for_non_runtime_package():
'''
Tests that `get_installable_package` doesn't return the packages marked as
non-runtime i.e. build command with runtime_package=False.
'''
build_commands = graph.OpenCEGraph()
node1 = build_tree.DependencyNode(
packages=["package1a"],
build_command=build_tree.BuildCommand(
"recipe1",
"repo1", ["package1a"],
runtime_package=False,
python="2.6",
build_type="cuda",
mpi_type="openmpi",
cudatoolkit="10.2",
output_files=["package1a-py26-cuda-openmpi"]))
node2 = build_tree.DependencyNode(
packages=["package2a"],
build_command=build_tree.BuildCommand(
"recipe2",
"repo2", ["package2a"],
python="2.6",
build_type="cpu",
mpi_type="openmpi",
cudatoolkit="10.2",
output_files=["package2a-py26-cuda-openmpi"]))
build_commands.add_node(node1)
build_commands.add_node(node2)
build_commands.add_edge(node1, node2)
external_deps = [
"external_pac1 1.2", "external_pack2", "external_pack3=1.2.3"
]
for dep in external_deps:
build_commands.add_node(build_tree.DependencyNode({dep}))
packages = build_tree.get_installable_packages(build_commands,
external_deps)
assert not "package1a" in packages
def test_dag_cleanup():
'''
Test that external packages are removed during cleanup.
'''
mock_build_tree = TestBuildTree([], "3.6", "cpu", "openmpi", "10.2")
mock_build_tree._tree = sample_build_commands()
external_node = build_tree.DependencyNode(packages=["external_package"])
nodes = list(mock_build_tree._tree.nodes())
parent_node = nodes[0]
child_node = nodes[1]
mock_build_tree._tree.add_node(external_node)
mock_build_tree._tree.add_edge(parent_node, external_node)
mock_build_tree._tree.add_edge(external_node, child_node)
mock_build_tree.remove_external_deps_from_dag()
assert not external_node in mock_build_tree._tree.nodes()
assert child_node in mock_build_tree._tree.successors(parent_node)
assert parent_node in mock_build_tree._tree.predecessors(child_node)
def sample_build_commands():
retval = networkx.DiGraph()
node1 = build_tree.DependencyNode(
packages=["package1a", "package1b"],
build_command=build_tree.BuildCommand(
"recipe1",
"repo1", ["package1a", "package1b"],
python="3.6",
build_type="cuda",
mpi_type="openmpi",
cudatoolkit="10.2",
output_files=[
"package1a-1.0-py36_cuda10.2.tar.bz2",
"package1b-1.0-py36_cuda10.2.tar.bz2"
],
run_dependencies=[
"python >=3.6", "pack1 1.0", "pack2 >=2.0", "pack3 9b"
]))
node2 = build_tree.DependencyNode(
packages=["package2a"],
build_command=build_tree.BuildCommand(
"recipe2",
"repo2", ["package2a"],
python="3.6",
build_type="cpu",
mpi_type="system",
cudatoolkit="10.2",
output_files=["package2a-1.0-py36_cuda10.2.tar.bz2"],
run_dependencies=[
"python ==3.6", "pack1 >=1.0", "pack2 ==2.0",
"pack3 3.3 build"
]))
node3 = build_tree.DependencyNode(
packages=["package3a", "package3b"],
build_command=build_tree.BuildCommand(
"recipe3",
"repo3", ["package3a", "package3b"],
python="3.7",
build_type="cpu",
mpi_type="openmpi",
cudatoolkit="10.2",
output_files=[
"package3a-1.0-py37_cuda10.2.tar.bz2",
"package3b-1.0-py37_cuda10.2.tar.bz2"
],
run_dependencies=[
"python 3.7", "pack1==1.0", "pack2 <=2.0", "pack3 3.0.*",
"pack4=1.15.0=py38h6ffa863_0"
]))
node4 = build_tree.DependencyNode(
packages=["package4a", "package4b"],
build_command=build_tree.BuildCommand(
"recipe4",
"repo4", ["package4a", "package4b"],
python="3.7",
build_type="cuda",
mpi_type="system",
cudatoolkit="10.2",
output_files=[
"package4a-1.0-py37_cuda.tar.bz2",
"package4b-1.0-py37_cuda.tar.bz2"
],
run_dependencies=["pack1==1.0", "pack2 <=2.0",
"pack3-suffix 3.0"]))
retval.add_node(node1)
retval.add_node(node2)
retval.add_node(node3)
retval.add_node(node4)
for dep in external_deps:
retval.add_node(build_tree.DependencyNode({dep}))
return retval