def test_gt_to_file_name():
tool_1 = Tool(tool_1_def, "T1")
p = Pipeline()
node_1 = p.add(tool_1)
assert node_1._tool.options['output'] == sys.stdout
node_1 > "A.txt"
assert node_1._tool.options['output'] == "A.txt"
def test_lt_from_file_name():
tool_1 = Tool(tool_1_def, "T1")
p = Pipeline()
node_1 = p.add(tool_1)
assert node_1.input == sys.stdin
node_1 < "A.txt"
assert node_1.input == "A.txt"
def test_skip_last_node():
tool_1 = Tool(tool_1_def, "T1")
tool_2 = Tool(tool_1_def, "T2")
tool_3 = Tool(tool_1_def, "T3")
p = Pipeline()
node_1 = p.run(tool_1, input='/infile.txt')
node_2 = p.run(tool_2)
node_3 = p.run(tool_3, output='/outfile.txt')
node_1 | node_2 | node_3
groups = list(p.groups())
assert len(groups) == 1
assert node_1._tool.options['input'].get() == '/infile.txt'
assert node_1._tool.options['output'].raw() == sys.stdout
assert node_2._tool.options['input'].raw() == sys.stdout # output of 1
assert node_2._tool.options['output'].raw() == sys.stdout
assert node_3._tool.options['input'].raw() == sys.stdout
assert node_3._tool.options['output'].get() == '/outfile.txt'
p.skip(node_3)
assert len(list(p.nodes())) == 2
assert node_1._tool.options['input'].get() == '/infile.txt'
assert node_1._tool.options['output'].raw() == sys.stdout
assert node_2._tool.options['input'].raw() == sys.stdout
assert node_2._tool.options['output'].get() == '/outfile.txt'
def test_gt_to_option():
tool_1 = Tool(tool_1_def, "T1")
tool_2 = Tool(tool_1_def, "T2")
tool_3 = Tool(tool_1_def, "T3")
p = Pipeline()
node_1 = p.add(tool_1)
node_2 = p.add(tool_2)
node_3 = p.add(tool_3)
assert len(list(node_1.outgoing())) == 0
assert len(list(node_2.outgoing())) == 0
assert len(list(node_3.outgoing())) == 0
assert node_1._tool.options['output'] == sys.stdout
assert node_2._tool.options['input'] == sys.stdin
assert node_3._tool.options['input'] == sys.stdin
(node_1 > node_2.input) > node_3.input
n_1_out = node_1._tool.options['output'].raw()
n_2_in = node_2._tool.options['input'].raw()
n_2_out = node_2._tool.options['output'].raw()
n_3_in = node_3._tool.options['output'].raw()
assert n_1_out == n_2_in
assert n_2_out == n_3_in
assert len(list(node_1.outgoing())) == 1
assert len(list(node_2.incoming())) == 1
assert len(list(node_2.outgoing())) == 1
assert len(list(node_3.outgoing())) == 0
def test_edge_equality():
p = Pipeline()
a = p.run('nop')
b = p.run('nop')
assert p.add_edge(a, b) is not None
assert p.add_edge(a, b) is not None
assert len(p._edges) == 1
def test_find_fanout_options():
tool = Tool(tool_1_def)
p = Pipeline()
node = p.add(tool)
node.input = ["test_1.txt", "test_2.txt"]
assert len(node.input.value) == 2
assert len(node.input) == 2
assert p._get_fanout_options(node) == [node.input]
def test_graph_create():
p = Pipeline()
a = p.run('nop')
b = p.run('nop')
p.run('nop')
assert len(p._nodes) == 3
assert p.add_edge(a, b) is not None
assert len(p._edges) == 1
def test_get_node_properties():
tool = Tool(tool_1_def)
p = Pipeline()
node = p.add(tool)
assert isinstance(node.input, Option)
with pytest.raises(AttributeError) as ex:
node.xxx
assert str(ex.value) == "Option 'xxx' not found in tools"
def test_three_groups():
tool_1 = Tool(tool_1_def, "T1")
tool_2 = Tool(tool_1_def, "T2")
tool_3 = Tool(tool_1_def, "T3")
p = Pipeline()
node_1 = p.add(tool_1)
node_2 = p.add(tool_2)
node_3 = p.add(tool_3)
node_1 >> (node_2 + node_3)
groups = list(p.groups())
assert len(groups) == 3
def test_set_node_properties():
tool = Tool(tool_1_def)
p = Pipeline()
node = p.add(tool)
opt = node.input
assert isinstance(opt, Option)
node.input = "test.txt"
assert opt.raw() == "test.txt"
with pytest.raises(AttributeError) as ex:
node.xxx = "A"
assert str(ex.value) == "Option 'xxx' not found in tools"
def test_groups_all_in_one():
tool_1 = Tool(tool_1_def, "T1")
tool_2 = Tool(tool_1_def, "T2")
tool_3 = Tool(tool_1_def, "T3")
p = Pipeline()
node_1 = p.add(tool_1)
node_2 = p.add(tool_2)
node_3 = p.add(tool_3)
node_1 | (node_2 + node_3)
groups = list(p.groups())
assert len(groups) == 1
assert set(groups[0]) == set([node_1, node_2, node_3])
def test_two_groups():
tool_1 = Tool(tool_1_def, "T1")
tool_2 = Tool(tool_1_def, "T2")
tool_3 = Tool(tool_1_def, "T3")
p = Pipeline()
p.add(tool_1)
node_2 = p.add(tool_2)
node_3 = p.add(tool_3)
node_2 | node_3
groups = list(p.groups())
assert len(groups) == 2
def test_remove_node():
p = Pipeline()
a = p.run('nop')
b = p.run('nop')
c = p.run('nop')
p.add_edge(c, b)
p.add_edge(b, a)
p.remove(b)
assert len(p._nodes) == 2
assert len(p._edges) == 0
for node in p.nodes():
assert len(node._edges) == 0
def test_delegate_singleton_node_default_option():
tool_1 = Tool(tool_1_def)
tool_2 = Tool(tool_1_def)
p = Pipeline()
node_1 = p.add(tool_1)
node_2 = p.add(tool_2)
node_2.input = node_1
assert len(p._nodes) == 2
assert len(p._edges) == 1
edge = p.get_edge(node_1, node_2)
assert edge is not None
assert len(edge._links) == 1
def test_expand_two_nodes_both_fan_out():
tool_1 = Tool(tool_1_def, "T1")
tool_2 = Tool(tool_1_def, "T2")
p = Pipeline()
node_1 = p.add(tool_1)
node_1.input = ["test_1.txt", "test_2.txt"]
node_2 = p.add(tool_2)
node_2.input = node_1.output
assert len(p._nodes) == 2
assert len(p._edges) == 1
p.expand(validate=False)
assert len(p._nodes) == 4
assert len(p._edges) == 2
def test_pipe_and_plus_operator():
tool_1 = Tool(tool_1_def, "T1")
tool_2 = Tool(tool_1_def, "T2")
tool_3 = Tool(tool_1_def, "T3")
p = Pipeline()
node_1 = p.add(tool_1)
node_2 = p.add(tool_2)
node_3 = p.add(tool_3)
node_1 | node_2 | node_3
assert len(p._edges) == 2
assert list(node_1.children()) == [node_2]
assert list(node_2.children()) == [node_3]
def test_expand_single_node():
tool = Tool(tool_1_def)
p = Pipeline()
node = p.add(tool)
node.input = ["test_1.txt", "test_2.txt"]
p.expand(validate=False)
assert len(p._nodes) == 2
assert len(p._edges) == 0
inputs = []
for node in p.nodes():
inputs.append(node.input.get())
assert sorted(inputs) == [
os.path.join(os.getcwd(), "test_1.txt"),
os.path.join(os.getcwd(), "test_2.txt")
]
def test_topological_sort():
p = Pipeline()
a = p.run('nop')
assert a.name == "nop"
b = p.run('nop')
assert a.name == "nop.0"
assert b.name == "nop.1"
c = p.run('nop')
assert a.name == "nop.0"
assert b.name == "nop.1"
assert c.name == "nop.2"
p.add_edge(c, b)
p.add_edge(b, a)
sorted_nodes = [n for n in p.topological_order()]
assert sorted_nodes == [c, b, a]
def test_add_operator():
tool_1 = Tool(tool_1_def, "T1")
tool_2 = Tool(tool_1_def, "T2")
tool_3 = Tool(tool_1_def, "T3")
p = Pipeline()
node_1 = p.add(tool_1)
node_2 = p.add(tool_2)
node_3 = p.add(tool_3)
node_1 | (node_2 + node_3)
assert len(p._edges) == 2
assert set(node_1.children()) == set([node_2, node_3])
assert len(node_2._tool.options['input']) == 1
assert len(node_2._tool.options['input']) == 1
assert node_2._tool.options['input'] == node_3._tool.options['input']
def test_left_shift_operator():
tool_1 = Tool(tool_1_def, "T1")
tool_2 = Tool(tool_1_def, "T2")
tool_3 = Tool(tool_1_def, "T3")
p = Pipeline()
node_1 = p.add(tool_1)
node_2 = p.add(tool_2)
node_3 = p.add(tool_3)
node_3 << node_1
node_3 << node_2
assert len(p._edges) == 2
assert list(node_1.children()) == [node_3]
assert list(node_2.children()) == [node_3]
assert list(node_3.children()) == []
def test_delegate_list_node_default_option():
tool_1 = Tool(tool_1_def)
tool_2 = Tool(tool_1_def)
tool_3 = Tool(tool_1_def)
p = Pipeline()
node_1 = p.add(tool_1)
node_2 = p.add(tool_2)
node_3 = p.add(tool_3)
node_3.input = [node_1, node_2]
assert len(p._edges) == 2
edge = p.get_edge(node_1, node_3)
assert edge is not None
assert len(edge._links) == 1
edge_2 = p.get_edge(node_2, node_3)
assert edge_2 is not None
assert len(edge_2._links) == 1
def test_node_naming_in_simple_multiplex():
p = Pipeline()
j = p.job("1").run('bash', cmd="ls")
assert p.get("1") == j
j.input = ["A", "B", "C"]
p.expand(validate=False)
with pytest.raises(LookupError):
p.get("1")
assert len(p) == 3
assert p.get("1.0") is not None
assert p.get("1.0").input.get() == os.path.join(os.getcwd(), "A")
assert p.get("1.1") is not None
assert p.get("1.1").input.get() == os.path.join(os.getcwd(), "B")
assert p.get("1.2") is not None
assert p.get("1.2").input.get() == os.path.join(os.getcwd(), "C")
def test_expand_three_nodes_two_fan_out():
tool_1 = Tool(tool_1_def, "T1")
tool_2 = Tool(tool_1_def, "T2")
tool_3 = Tool(tool_1_def, "T3")
p = Pipeline()
node_1 = p.add(tool_1)
node_1.input = ["test_1.txt", "test_2.txt"]
node_2 = p.add(tool_2)
node_2.input = node_1.output
node_3 = p.add(tool_3)
node_3.x = "other"
node_3 = p.add(tool_3)
node_2.x = node_3.x
assert len(p._nodes) == 3
assert len(p._edges) == 2
p.expand(validate=False)
assert len(p._nodes) == 5
assert len(p._edges) == 6
def test_lt_from_option():
tool_1 = Tool(tool_1_def, "T1")
tool_2 = Tool(tool_1_def, "T2")
tool_3 = Tool(tool_1_def, "T3")
p = Pipeline()
node_1 = p.add(tool_1)
node_2 = p.add(tool_2)
node_3 = p.add(tool_3)
assert len(list(node_1.outgoing())) == 0
assert len(list(node_2.outgoing())) == 0
assert len(list(node_3.outgoing())) == 0
assert node_1.output == sys.stdout
assert node_2.input == sys.stdin
assert node_3.input == sys.stdin
(node_1 < node_2.output) < node_3.output
assert not node_3.has_incoming()
assert node_2.has_incoming(node_3, ('output', 'input'), True)
assert node_1.has_incoming(node_2, ('output', 'input'), True)
def test_right_shift_operator():
tool_1 = Tool(tool_1_def, "T1")
tool_2 = Tool(tool_1_def, "T2")
tool_3 = Tool(tool_1_def, "T3")
p = Pipeline()
node_1 = p.add(tool_1)
node_2 = p.add(tool_2)
node_3 = p.add(tool_3)
node_1.output = "A"
node_2.output = "B"
node_3.output = "C"
node_1 >> node_3
node_2 >> node_3
assert len(p._edges) == 2
assert list(node_1.children()) == [node_3]
assert list(node_2.children()) == [node_3]
assert list(node_3.children()) == []
for e in node_3._edges:
for link in e._links:
assert not link[2]
def test_gt_to_option_no_blocks():
tool_1 = Tool(tool_1_def, "T1")
tool_2 = Tool(tool_1_def, "T2")
tool_3 = Tool(tool_1_def, "T3")
p = Pipeline()
node_1 = p.add(tool_1)
node_2 = p.add(tool_2)
node_3 = p.add(tool_3)
assert len(list(node_1.outgoing())) == 0
assert len(list(node_2.outgoing())) == 0
assert len(list(node_3.outgoing())) == 0
assert node_1.output == sys.stdout
assert node_2.input == sys.stdin
assert node_3.input == sys.stdin
node_1 > node_2.input # this does not work in a single line!
node_2 > node_3.input
assert not node_3.input.raw() == sys.stdin
# check the graph structure
assert node_2.has_incoming(node_1, ('output', 'input'), True,
node_1.output)
assert node_3.has_incoming(node_2, ('output', 'input'), True,
node_2.output)
assert not node_3.has_outgoing()
def test_node_equality():
p = Pipeline()
tool = Tool(tool_1_def)
p.add(tool)
p.add(tool)
assert len(p._nodes) == 1
def test_node_naming_and_auto_indexing_job_names_assigned():
p = Pipeline()
p.job("1").run('bash', cmd="ls")
p.job("2").run('bash', cmd="ls")
assert p.get("1") is not None
assert p.get("2") is not None
def test_node_naming_and_auto_indexing_no_names_assigned():
p = Pipeline()
p.run('bash', cmd="ls")
p.run('bash', cmd="ls")
assert p.get("bash.0") is not None
assert p.get("bash.1") is not None
def test_missing_node_for_edge_insert():
p = Pipeline()
assert p.add_edge("A", "B") is None