Exemple #1
0
def test_pipeline_adding_tools():
    p = Pipeline()
    touch = p.add(Touch())
    split = p.add(Split())
    assert len(p.tools) == 2
    assert p.tools["Touch"] == touch
    assert p.tools["Split"] == split
Exemple #2
0
def test_edge_equality():
    p = Pipeline()
    a = p.run('nop')
    b = p.run('nop')
    assert p.add_edge(a, b) is not None
    assert p.add_edge(a, b) is not None
    assert len(p._edges) == 1
Exemple #3
0
def test_gt_to_option():
    tool_1 = Tool(tool_1_def, "T1")
    tool_2 = Tool(tool_1_def, "T2")
    tool_3 = Tool(tool_1_def, "T3")
    p = Pipeline()
    node_1 = p.add(tool_1)
    node_2 = p.add(tool_2)
    node_3 = p.add(tool_3)
    assert len(list(node_1.outgoing())) == 0
    assert len(list(node_2.outgoing())) == 0
    assert len(list(node_3.outgoing())) == 0
    assert node_1._tool.options['output'] == sys.stdout
    assert node_2._tool.options['input'] == sys.stdin
    assert node_3._tool.options['input'] == sys.stdin
    (node_1 > node_2.input) > node_3.input

    n_1_out = node_1._tool.options['output'].raw()
    n_2_in = node_2._tool.options['input'].raw()
    n_2_out = node_2._tool.options['output'].raw()
    n_3_in = node_3._tool.options['output'].raw()
    assert n_1_out == n_2_in
    assert n_2_out == n_3_in
    assert len(list(node_1.outgoing())) == 1
    assert len(list(node_2.incoming())) == 1
    assert len(list(node_2.outgoing())) == 1
    assert len(list(node_3.outgoing())) == 0
def test_gt_to_file_name():
    tool_1 = Tool(tool_1_def, "T1")
    p = Pipeline()
    node_1 = p.add(tool_1)
    assert node_1._tool.options['output'] == sys.stdout
    node_1 > "A.txt"
    assert node_1._tool.options['output'] == "A.txt"
Exemple #5
0
def test_lt_from_file_name():
    tool_1 = Tool(tool_1_def, "T1")
    p = Pipeline()
    node_1 = p.add(tool_1)
    assert node_1.input == sys.stdin
    node_1 < "A.txt"
    assert node_1.input == "A.txt"
def test_lt_from_file_name():
    tool_1 = Tool(tool_1_def, "T1")
    p = Pipeline()
    node_1 = p.add(tool_1)
    assert node_1.input == sys.stdin
    node_1 < "A.txt"
    assert node_1.input == "A.txt"
def test_gt_to_option():
    tool_1 = Tool(tool_1_def, "T1")
    tool_2 = Tool(tool_1_def, "T2")
    tool_3 = Tool(tool_1_def, "T3")
    p = Pipeline()
    node_1 = p.add(tool_1)
    node_2 = p.add(tool_2)
    node_3 = p.add(tool_3)
    assert len(list(node_1.outgoing())) == 0
    assert len(list(node_2.outgoing())) == 0
    assert len(list(node_3.outgoing())) == 0
    assert node_1._tool.options['output'] == sys.stdout
    assert node_2._tool.options['input'] == sys.stdin
    assert node_3._tool.options['input'] == sys.stdin
    (node_1 > node_2.input) > node_3.input

    n_1_out = node_1._tool.options['output'].raw()
    n_2_in = node_2._tool.options['input'].raw()
    n_2_out = node_2._tool.options['output'].raw()
    n_3_in = node_3._tool.options['output'].raw()
    assert n_1_out == n_2_in
    assert n_2_out == n_3_in
    assert len(list(node_1.outgoing())) == 1
    assert len(list(node_2.incoming())) == 1
    assert len(list(node_2.outgoing())) == 1
    assert len(list(node_3.outgoing())) == 0
Exemple #8
0
def test_gt_to_file_name():
    tool_1 = Tool(tool_1_def, "T1")
    p = Pipeline()
    node_1 = p.add(tool_1)
    assert node_1._tool.options['output'] == sys.stdout
    node_1 > "A.txt"
    assert node_1._tool.options['output'] == "A.txt"
def test_edge_equality():
    p = Pipeline()
    a = p.run('nop')
    b = p.run('nop')
    assert p.add_edge(a, b) is not None
    assert p.add_edge(a, b) is not None
    assert len(p._edges) == 1
Exemple #10
0
def test_get_node_properties():
    tool = Tool(tool_1_def)
    p = Pipeline()
    node = p.add(tool)
    assert isinstance(node.input, Option)
    with pytest.raises(AttributeError) as ex:
        node.xxx
    assert str(ex.value) == "Option 'xxx' not found in tools"
def test_find_fanout_options():
    tool = Tool(tool_1_def)
    p = Pipeline()
    node = p.add(tool)
    node.input = ["test_1.txt", "test_2.txt"]
    assert len(node.input.value) == 2
    assert len(node.input) == 2
    assert p._get_fanout_options(node) == [node.input]
def test_graph_create():
    p = Pipeline()
    a = p.run('nop')
    b = p.run('nop')
    p.run('nop')
    assert len(p._nodes) == 3
    assert p.add_edge(a, b) is not None
    assert len(p._edges) == 1
def test_get_node_properties():
    tool = Tool(tool_1_def)
    p = Pipeline()
    node = p.add(tool)
    assert isinstance(node.input, Option)
    with pytest.raises(AttributeError) as ex:
        node.xxx
    assert str(ex.value) == "Option 'xxx' not found in tools"
Exemple #14
0
def test_find_fanout_options():
    tool = Tool(tool_1_def)
    p = Pipeline()
    node = p.add(tool)
    node.input = ["test_1.txt", "test_2.txt"]
    assert len(node.input.value) == 2
    assert len(node.input) == 2
    assert p._get_fanout_options(node) == [node.input]
Exemple #15
0
def test_pipeline_circular_dependencies_direct_loop():
    p = Pipeline()
    a = p.add(Touch("a"))
    b = p.add(Touch("b"))
    b.name = a.name

    with pytest.raises(CircularDependencyException) as excinfo:
        a.name = b.name
    assert excinfo.value.circle == [a, a]
Exemple #16
0
def test_skip_last_node():
    tool_1 = Tool(tool_1_def, "T1")
    tool_2 = Tool(tool_1_def, "T2")
    tool_3 = Tool(tool_1_def, "T3")

    p = Pipeline()
    node_1 = p.run(tool_1, input='/infile.txt')
    node_2 = p.run(tool_2)
    node_3 = p.run(tool_3, output='/outfile.txt')

    node_1 | node_2 | node_3
    groups = list(p.groups())
    assert len(groups) == 1
    assert node_1._tool.options['input'].get() == '/infile.txt'
    assert node_1._tool.options['output'].raw() == sys.stdout
    assert node_2._tool.options['input'].raw() == sys.stdout  # output of 1
    assert node_2._tool.options['output'].raw() == sys.stdout
    assert node_3._tool.options['input'].raw() == sys.stdout
    assert node_3._tool.options['output'].get() == '/outfile.txt'

    p.skip(node_3)
    assert len(list(p.nodes())) == 2
    assert node_1._tool.options['input'].get() == '/infile.txt'
    assert node_1._tool.options['output'].raw() == sys.stdout
    assert node_2._tool.options['input'].raw() == sys.stdout
    assert node_2._tool.options['output'].get() == '/outfile.txt'
def test_three_groups():
    tool_1 = Tool(tool_1_def, "T1")
    tool_2 = Tool(tool_1_def, "T2")
    tool_3 = Tool(tool_1_def, "T3")
    p = Pipeline()
    node_1 = p.add(tool_1)
    node_2 = p.add(tool_2)
    node_3 = p.add(tool_3)
    node_1 >> (node_2 + node_3)
    groups = list(p.groups())
    assert len(groups) == 3
Exemple #18
0
def test_three_groups():
    tool_1 = Tool(tool_1_def, "T1")
    tool_2 = Tool(tool_1_def, "T2")
    tool_3 = Tool(tool_1_def, "T3")
    p = Pipeline()
    node_1 = p.add(tool_1)
    node_2 = p.add(tool_2)
    node_3 = p.add(tool_3)
    node_1 >> (node_2 + node_3)
    groups = list(p.groups())
    assert len(groups) == 3
Exemple #19
0
def test_groups_all_in_one():
    tool_1 = Tool(tool_1_def, "T1")
    tool_2 = Tool(tool_1_def, "T2")
    tool_3 = Tool(tool_1_def, "T3")
    p = Pipeline()
    node_1 = p.add(tool_1)
    node_2 = p.add(tool_2)
    node_3 = p.add(tool_3)
    node_1 | (node_2 + node_3)
    groups = list(p.groups())
    assert len(groups) == 1
    assert set(groups[0]) == set([node_1, node_2, node_3])
def test_set_node_properties():
    tool = Tool(tool_1_def)
    p = Pipeline()
    node = p.add(tool)
    opt = node.input
    assert isinstance(opt, Option)
    node.input = "test.txt"
    assert opt.raw() == "test.txt"

    with pytest.raises(AttributeError) as ex:
        node.xxx = "A"
    assert str(ex.value) == "Option 'xxx' not found in tools"
Exemple #21
0
def test_set_node_properties():
    tool = Tool(tool_1_def)
    p = Pipeline()
    node = p.add(tool)
    opt = node.input
    assert isinstance(opt, Option)
    node.input = "test.txt"
    assert opt.raw() == "test.txt"

    with pytest.raises(AttributeError) as ex:
        node.xxx = "A"
    assert str(ex.value) == "Option 'xxx' not found in tools"
def test_two_groups():
    tool_1 = Tool(tool_1_def, "T1")
    tool_2 = Tool(tool_1_def, "T2")
    tool_3 = Tool(tool_1_def, "T3")
    p = Pipeline()
    p.add(tool_1)
    node_2 = p.add(tool_2)
    node_3 = p.add(tool_3)

    node_2 | node_3
    groups = list(p.groups())
    assert len(groups) == 2
def test_groups_all_in_one():
    tool_1 = Tool(tool_1_def, "T1")
    tool_2 = Tool(tool_1_def, "T2")
    tool_3 = Tool(tool_1_def, "T3")
    p = Pipeline()
    node_1 = p.add(tool_1)
    node_2 = p.add(tool_2)
    node_3 = p.add(tool_3)
    node_1 | (node_2 + node_3)
    groups = list(p.groups())
    assert len(groups) == 1
    assert set(groups[0]) == set([node_1, node_2, node_3])
def test_expand_two_nodes_both_fan_out():
    tool_1 = Tool(tool_1_def, "T1")
    tool_2 = Tool(tool_1_def, "T2")
    p = Pipeline()
    node_1 = p.add(tool_1)
    node_1.input = ["test_1.txt", "test_2.txt"]
    node_2 = p.add(tool_2)
    node_2.input = node_1.output
    assert len(p._nodes) == 2
    assert len(p._edges) == 1
    p.expand(validate=False)
    assert len(p._nodes) == 4
    assert len(p._edges) == 2
def test_delegate_singleton_node_default_option():
    tool_1 = Tool(tool_1_def)
    tool_2 = Tool(tool_1_def)
    p = Pipeline()
    node_1 = p.add(tool_1)
    node_2 = p.add(tool_2)

    node_2.input = node_1
    assert len(p._nodes) == 2
    assert len(p._edges) == 1
    edge = p.get_edge(node_1, node_2)
    assert edge is not None
    assert len(edge._links) == 1
def test_expand_single_node():
    tool = Tool(tool_1_def)
    p = Pipeline()
    node = p.add(tool)
    node.input = ["test_1.txt", "test_2.txt"]
    p.expand(validate=False)
    assert len(p._nodes) == 2
    assert len(p._edges) == 0
    inputs = []
    for node in p.nodes():
        inputs.append(node.input.get())
    assert sorted(inputs) == [os.path.join(os.getcwd(), "test_1.txt"),
                              os.path.join(os.getcwd(), "test_2.txt")]
Exemple #27
0
def test_delegate_singleton_node_default_option():
    tool_1 = Tool(tool_1_def)
    tool_2 = Tool(tool_1_def)
    p = Pipeline()
    node_1 = p.add(tool_1)
    node_2 = p.add(tool_2)

    node_2.input = node_1
    assert len(p._nodes) == 2
    assert len(p._edges) == 1
    edge = p.get_edge(node_1, node_2)
    assert edge is not None
    assert len(edge._links) == 1
def test_pipe_and_plus_operator():
    tool_1 = Tool(tool_1_def, "T1")
    tool_2 = Tool(tool_1_def, "T2")
    tool_3 = Tool(tool_1_def, "T3")
    p = Pipeline()
    node_1 = p.add(tool_1)
    node_2 = p.add(tool_2)
    node_3 = p.add(tool_3)

    node_1 | node_2 | node_3
    assert len(p._edges) == 2
    assert list(node_1.children()) == [node_2]
    assert list(node_2.children()) == [node_3]
Exemple #29
0
def test_pipe_and_plus_operator():
    tool_1 = Tool(tool_1_def, "T1")
    tool_2 = Tool(tool_1_def, "T2")
    tool_3 = Tool(tool_1_def, "T3")
    p = Pipeline()
    node_1 = p.add(tool_1)
    node_2 = p.add(tool_2)
    node_3 = p.add(tool_3)

    node_1 | node_2 | node_3
    assert len(p._edges) == 2
    assert list(node_1.children()) == [node_2]
    assert list(node_2.children()) == [node_3]
def test_skip_last_node():
    tool_1 = Tool(tool_1_def, "T1")
    tool_2 = Tool(tool_1_def, "T2")
    tool_3 = Tool(tool_1_def, "T3")

    p = Pipeline()
    node_1 = p.run(tool_1, input='/infile.txt')
    node_2 = p.run(tool_2)
    node_3 = p.run(tool_3, output='/outfile.txt')

    node_1 | node_2 | node_3
    groups = list(p.groups())
    assert len(groups) == 1
    assert node_1._tool.options['input'].get() == '/infile.txt'
    assert node_1._tool.options['output'].raw() == sys.stdout
    assert node_2._tool.options['input'].raw() == sys.stdout  # output of 1
    assert node_2._tool.options['output'].raw() == sys.stdout
    assert node_3._tool.options['input'].raw() == sys.stdout
    assert node_3._tool.options['output'].get() == '/outfile.txt'

    p.skip(node_3)
    assert len(list(p.nodes())) == 2
    assert node_1._tool.options['input'].get() == '/infile.txt'
    assert node_1._tool.options['output'].raw() == sys.stdout
    assert node_2._tool.options['input'].raw() == sys.stdout
    assert node_2._tool.options['output'].get() == '/outfile.txt'
Exemple #31
0
def test_pipeline_configure_tools():
    p = Pipeline()
    touch = p.add(Touch())
    split = p.add(Split())
    touch.name= "myfile.txt"
    split.file = touch.file
    split.prefix = "split"
    split.count = 2

    assert p.get_configuration(touch) == {"name": "myfile.txt", "file": "myfile.txt", "job": touch.job}
    assert p.get_configuration(split) == {"file": "myfile.txt",
                                    "prefix": "split",
                                    "count": 2,
                                          "files": ["split-0", "split-1"], "job": split.job}
Exemple #32
0
def test_left_shift_operator():
    tool_1 = Tool(tool_1_def, "T1")
    tool_2 = Tool(tool_1_def, "T2")
    tool_3 = Tool(tool_1_def, "T3")
    p = Pipeline()
    node_1 = p.add(tool_1)
    node_2 = p.add(tool_2)
    node_3 = p.add(tool_3)

    node_3 << node_1
    node_3 << node_2
    assert len(p._edges) == 2
    assert list(node_1.children()) == [node_3]
    assert list(node_2.children()) == [node_3]
    assert list(node_3.children()) == []
Exemple #33
0
def test_add_operator():
    tool_1 = Tool(tool_1_def, "T1")
    tool_2 = Tool(tool_1_def, "T2")
    tool_3 = Tool(tool_1_def, "T3")
    p = Pipeline()
    node_1 = p.add(tool_1)
    node_2 = p.add(tool_2)
    node_3 = p.add(tool_3)

    node_1 | (node_2 + node_3)
    assert len(p._edges) == 2
    assert set(node_1.children()) == set([node_2, node_3])
    assert len(node_2._tool.options['input']) == 1
    assert len(node_2._tool.options['input']) == 1
    assert node_2._tool.options['input'] == node_3._tool.options['input']
def test_left_shift_operator():
    tool_1 = Tool(tool_1_def, "T1")
    tool_2 = Tool(tool_1_def, "T2")
    tool_3 = Tool(tool_1_def, "T3")
    p = Pipeline()
    node_1 = p.add(tool_1)
    node_2 = p.add(tool_2)
    node_3 = p.add(tool_3)

    node_3 << node_1
    node_3 << node_2
    assert len(p._edges) == 2
    assert list(node_1.children()) == [node_3]
    assert list(node_2.children()) == [node_3]
    assert list(node_3.children()) == []
def test_add_operator():
    tool_1 = Tool(tool_1_def, "T1")
    tool_2 = Tool(tool_1_def, "T2")
    tool_3 = Tool(tool_1_def, "T3")
    p = Pipeline()
    node_1 = p.add(tool_1)
    node_2 = p.add(tool_2)
    node_3 = p.add(tool_3)

    node_1 | (node_2 + node_3)
    assert len(p._edges) == 2
    assert set(node_1.children()) == set([node_2, node_3])
    assert len(node_2._tool.options['input']) == 1
    assert len(node_2._tool.options['input']) == 1
    assert node_2._tool.options['input'] == node_3._tool.options['input']
Exemple #36
0
def test_remove_node():
    p = Pipeline()
    a = p.run('nop')
    b = p.run('nop')
    c = p.run('nop')
    p.add_edge(c, b)
    p.add_edge(b, a)
    p.remove(b)
    assert len(p._nodes) == 2
    assert len(p._edges) == 0
    for node in p.nodes():
        assert len(node._edges) == 0
Exemple #37
0
def test_pipeline_dependencies():
    p = Pipeline()
    touch = p.add(Touch())
    split = p.add(Split())
    touch_b = p.add(Touch(), "b")
    touch_b.name = "myfile.txt"
    touch.name= touch_b.name
    split.file = touch.file
    split.prefix = "split"
    split.count = 2

    assert len(touch_b.get_dependencies()) == 0
    assert len(touch.get_dependencies()) == 1
    assert len(split.get_dependencies()) == 1
    assert list(split.get_dependencies()) == [touch]
    assert p.get_sorted_tools() == [touch_b, touch, split]
Exemple #38
0
def test_pipeline_circular_dependencies_complex_loop():
    p = Pipeline()
    a = p.add(Touch("a"))
    b = p.add(Touch("b"))
    c = p.add(Touch("c"))
    d = p.add(Touch("d"))

    a.name = "myfile.txt"
    a.file = "outfile"
    b.name = a.file
    c.name = b.file
    d.name = c.file

    with pytest.raises(CircularDependencyException) as excinfo:
        a.name = d.file
    print excinfo.value.circle == [a, b, c, d]
def test_delegate_list_node_default_option():
    tool_1 = Tool(tool_1_def)
    tool_2 = Tool(tool_1_def)
    tool_3 = Tool(tool_1_def)
    p = Pipeline()
    node_1 = p.add(tool_1)
    node_2 = p.add(tool_2)
    node_3 = p.add(tool_3)

    node_3.input = [node_1, node_2]
    assert len(p._edges) == 2
    edge = p.get_edge(node_1, node_3)
    assert edge is not None
    assert len(edge._links) == 1
    edge_2 = p.get_edge(node_2, node_3)
    assert edge_2 is not None
    assert len(edge_2._links) == 1
def test_lt_from_option():
    tool_1 = Tool(tool_1_def, "T1")
    tool_2 = Tool(tool_1_def, "T2")
    tool_3 = Tool(tool_1_def, "T3")
    p = Pipeline()
    node_1 = p.add(tool_1)
    node_2 = p.add(tool_2)
    node_3 = p.add(tool_3)
    assert len(list(node_1.outgoing())) == 0
    assert len(list(node_2.outgoing())) == 0
    assert len(list(node_3.outgoing())) == 0
    assert node_1.output == sys.stdout
    assert node_2.input == sys.stdin
    assert node_3.input == sys.stdin
    (node_1 < node_2.output) < node_3.output

    assert not node_3.has_incoming()
    assert node_2.has_incoming(node_3, ('output', 'input'), True)
    assert node_1.has_incoming(node_2, ('output', 'input'), True)
Exemple #41
0
def test_lt_from_option():
    tool_1 = Tool(tool_1_def, "T1")
    tool_2 = Tool(tool_1_def, "T2")
    tool_3 = Tool(tool_1_def, "T3")
    p = Pipeline()
    node_1 = p.add(tool_1)
    node_2 = p.add(tool_2)
    node_3 = p.add(tool_3)
    assert len(list(node_1.outgoing())) == 0
    assert len(list(node_2.outgoing())) == 0
    assert len(list(node_3.outgoing())) == 0
    assert node_1.output == sys.stdout
    assert node_2.input == sys.stdin
    assert node_3.input == sys.stdin
    (node_1 < node_2.output) < node_3.output

    assert not node_3.has_incoming()
    assert node_2.has_incoming(node_3, ('output', 'input'), True)
    assert node_1.has_incoming(node_2, ('output', 'input'), True)
def test_remove_node():
    p = Pipeline()
    a = p.run('nop')
    b = p.run('nop')
    c = p.run('nop')
    p.add_edge(c, b)
    p.add_edge(b, a)
    p.remove(b)
    assert len(p._nodes) == 2
    assert len(p._edges) == 0
    for node in p.nodes():
        assert len(node._edges) == 0
def test_right_shift_operator():
    tool_1 = Tool(tool_1_def, "T1")
    tool_2 = Tool(tool_1_def, "T2")
    tool_3 = Tool(tool_1_def, "T3")
    p = Pipeline()
    node_1 = p.add(tool_1)
    node_2 = p.add(tool_2)
    node_3 = p.add(tool_3)

    node_1.output = "A"
    node_2.output = "B"
    node_3.output = "C"

    node_1 >> node_3
    node_2 >> node_3
    assert len(p._edges) == 2
    assert list(node_1.children()) == [node_3]
    assert list(node_2.children()) == [node_3]
    assert list(node_3.children()) == []
    for e in node_3._edges:
        for link in e._links:
            assert not link[2]
Exemple #44
0
def test_right_shift_operator():
    tool_1 = Tool(tool_1_def, "T1")
    tool_2 = Tool(tool_1_def, "T2")
    tool_3 = Tool(tool_1_def, "T3")
    p = Pipeline()
    node_1 = p.add(tool_1)
    node_2 = p.add(tool_2)
    node_3 = p.add(tool_3)

    node_1.output = "A"
    node_2.output = "B"
    node_3.output = "C"

    node_1 >> node_3
    node_2 >> node_3
    assert len(p._edges) == 2
    assert list(node_1.children()) == [node_3]
    assert list(node_2.children()) == [node_3]
    assert list(node_3.children()) == []
    for e in node_3._edges:
        for link in e._links:
            assert not link[2]
Exemple #45
0
def test_topological_sort():
    p = Pipeline()
    a = p.run('nop')
    assert a.name == "nop"
    b = p.run('nop')
    assert a.name == "nop.0"
    assert b.name == "nop.1"
    c = p.run('nop')
    assert a.name == "nop.0"
    assert b.name == "nop.1"
    assert c.name == "nop.2"
    p.add_edge(c, b)
    p.add_edge(b, a)
    sorted_nodes = [n for n in p.topological_order()]
    assert sorted_nodes == [c, b, a]
Exemple #46
0
def test_graph_create():
    p = Pipeline()
    a = p.run('nop')
    b = p.run('nop')
    p.run('nop')
    assert len(p._nodes) == 3
    assert p.add_edge(a, b) is not None
    assert len(p._edges) == 1
def test_gt_to_option_no_blocks():
    tool_1 = Tool(tool_1_def, "T1")
    tool_2 = Tool(tool_1_def, "T2")
    tool_3 = Tool(tool_1_def, "T3")
    p = Pipeline()
    node_1 = p.add(tool_1)
    node_2 = p.add(tool_2)
    node_3 = p.add(tool_3)
    assert len(list(node_1.outgoing())) == 0
    assert len(list(node_2.outgoing())) == 0
    assert len(list(node_3.outgoing())) == 0
    assert node_1.output == sys.stdout
    assert node_2.input == sys.stdin
    assert node_3.input == sys.stdin
    node_1 > node_2.input  # this does not work in a single line!
    node_2 > node_3.input
    assert not node_3.input.raw() == sys.stdin
    # check the graph structure
    assert node_2.has_incoming(node_1, ('output', 'input'),
                               True, node_1.output)
    assert node_3.has_incoming(node_2, ('output', 'input'),
                               True, node_2.output)
    assert not node_3.has_outgoing()
Exemple #48
0
def test_gt_to_option_no_blocks():
    tool_1 = Tool(tool_1_def, "T1")
    tool_2 = Tool(tool_1_def, "T2")
    tool_3 = Tool(tool_1_def, "T3")
    p = Pipeline()
    node_1 = p.add(tool_1)
    node_2 = p.add(tool_2)
    node_3 = p.add(tool_3)
    assert len(list(node_1.outgoing())) == 0
    assert len(list(node_2.outgoing())) == 0
    assert len(list(node_3.outgoing())) == 0
    assert node_1.output == sys.stdout
    assert node_2.input == sys.stdin
    assert node_3.input == sys.stdin
    node_1 > node_2.input  # this does not work in a single line!
    node_2 > node_3.input
    assert not node_3.input.raw() == sys.stdin
    # check the graph structure
    assert node_2.has_incoming(node_1, ('output', 'input'), True,
                               node_1.output)
    assert node_3.has_incoming(node_2, ('output', 'input'), True,
                               node_2.output)
    assert not node_3.has_outgoing()
def test_topological_sort():
    p = Pipeline()
    a = p.run('nop')
    assert a.name == "nop"
    b = p.run('nop')
    assert a.name == "nop.0"
    assert b.name == "nop.1"
    c = p.run('nop')
    assert a.name == "nop.0"
    assert b.name == "nop.1"
    assert c.name == "nop.2"
    p.add_edge(c, b)
    p.add_edge(b, a)
    sorted_nodes = [n for n in p.topological_order()]
    assert sorted_nodes == [c, b, a]
Exemple #50
0
def test_two_groups():
    tool_1 = Tool(tool_1_def, "T1")
    tool_2 = Tool(tool_1_def, "T2")
    tool_3 = Tool(tool_1_def, "T3")
    p = Pipeline()
    p.add(tool_1)
    node_2 = p.add(tool_2)
    node_3 = p.add(tool_3)

    node_2 | node_3
    groups = list(p.groups())
    assert len(groups) == 2
Exemple #51
0
def test_expand_three_nodes_two_fan_out():
    tool_1 = Tool(tool_1_def, "T1")
    tool_2 = Tool(tool_1_def, "T2")
    tool_3 = Tool(tool_1_def, "T3")
    p = Pipeline()
    node_1 = p.add(tool_1)
    node_1.input = ["test_1.txt", "test_2.txt"]
    node_2 = p.add(tool_2)
    node_2.input = node_1.output
    node_3 = p.add(tool_3)
    node_3.x = "other"
    node_3 = p.add(tool_3)
    node_2.x = node_3.x

    assert len(p._nodes) == 3
    assert len(p._edges) == 2
    p.expand(validate=False)
    assert len(p._nodes) == 5
    assert len(p._edges) == 6
Exemple #52
0
def test_expand_two_nodes_both_fan_out():
    tool_1 = Tool(tool_1_def, "T1")
    tool_2 = Tool(tool_1_def, "T2")
    p = Pipeline()
    node_1 = p.add(tool_1)
    node_1.input = ["test_1.txt", "test_2.txt"]
    node_2 = p.add(tool_2)
    node_2.input = node_1.output
    assert len(p._nodes) == 2
    assert len(p._edges) == 1
    p.expand(validate=False)
    assert len(p._nodes) == 4
    assert len(p._edges) == 2
Exemple #53
0
def test_expand_single_node():
    tool = Tool(tool_1_def)
    p = Pipeline()
    node = p.add(tool)
    node.input = ["test_1.txt", "test_2.txt"]
    p.expand(validate=False)
    assert len(p._nodes) == 2
    assert len(p._edges) == 0
    inputs = []
    for node in p.nodes():
        inputs.append(node.input.get())
    assert sorted(inputs) == [
        os.path.join(os.getcwd(), "test_1.txt"),
        os.path.join(os.getcwd(), "test_2.txt")
    ]
def test_expand_three_nodes_two_fan_out():
    tool_1 = Tool(tool_1_def, "T1")
    tool_2 = Tool(tool_1_def, "T2")
    tool_3 = Tool(tool_1_def, "T3")
    p = Pipeline()
    node_1 = p.add(tool_1)
    node_1.input = ["test_1.txt", "test_2.txt"]
    node_2 = p.add(tool_2)
    node_2.input = node_1.output
    node_3 = p.add(tool_3)
    node_3.x = "other"
    node_3 = p.add(tool_3)
    node_2.x = node_3.x

    assert len(p._nodes) == 3
    assert len(p._edges) == 2
    p.expand(validate=False)
    assert len(p._nodes) == 5
    assert len(p._edges) == 6
Exemple #55
0
def test_delegate_list_node_default_option():
    tool_1 = Tool(tool_1_def)
    tool_2 = Tool(tool_1_def)
    tool_3 = Tool(tool_1_def)
    p = Pipeline()
    node_1 = p.add(tool_1)
    node_2 = p.add(tool_2)
    node_3 = p.add(tool_3)

    node_3.input = [node_1, node_2]
    assert len(p._edges) == 2
    edge = p.get_edge(node_1, node_3)
    assert edge is not None
    assert len(edge._links) == 1
    edge_2 = p.get_edge(node_2, node_3)
    assert edge_2 is not None
    assert len(edge_2._links) == 1
Exemple #56
0
def test_missing_node_for_edge_insert():
    p = Pipeline()
    assert p.add_edge("A", "B") is None
Exemple #57
0
def test_node_naming_and_auto_indexing_no_names_assigned():
    p = Pipeline()
    p.run('bash', cmd="ls")
    p.run('bash', cmd="ls")
    assert p.get("bash.0") is not None
    assert p.get("bash.1") is not None
Exemple #58
0
def test_node_naming_and_auto_indexing_job_names_assigned():
    p = Pipeline()
    p.job("1").run('bash', cmd="ls")
    p.job("2").run('bash', cmd="ls")
    assert p.get("1") is not None
    assert p.get("2") is not None
Exemple #59
0
def test_node_naming_in_simple_multiplex():
    p = Pipeline()
    j = p.job("1").run('bash', cmd="ls")
    assert p.get("1") == j
    j.input = ["A", "B", "C"]
    p.expand(validate=False)

    with pytest.raises(LookupError):
        p.get("1")
    assert len(p) == 3
    assert p.get("1.0") is not None
    assert p.get("1.0").input.get() == os.path.join(os.getcwd(), "A")

    assert p.get("1.1") is not None
    assert p.get("1.1").input.get() == os.path.join(os.getcwd(), "B")

    assert p.get("1.2") is not None
    assert p.get("1.2").input.get() == os.path.join(os.getcwd(), "C")
Exemple #60
0
def test_node_equality():
    p = Pipeline()
    tool = Tool(tool_1_def)
    p.add(tool)
    p.add(tool)
    assert len(p._nodes) == 1