def test_skip_last_node():
tool_1 = Tool(tool_1_def, "T1")
tool_2 = Tool(tool_1_def, "T2")
tool_3 = Tool(tool_1_def, "T3")
p = Pipeline()
node_1 = p.run(tool_1, input='/infile.txt')
node_2 = p.run(tool_2)
node_3 = p.run(tool_3, output='/outfile.txt')
node_1 | node_2 | node_3
groups = list(p.groups())
assert len(groups) == 1
assert node_1._tool.options['input'].get() == '/infile.txt'
assert node_1._tool.options['output'].raw() == sys.stdout
assert node_2._tool.options['input'].raw() == sys.stdout # output of 1
assert node_2._tool.options['output'].raw() == sys.stdout
assert node_3._tool.options['input'].raw() == sys.stdout
assert node_3._tool.options['output'].get() == '/outfile.txt'
p.skip(node_3)
assert len(list(p.nodes())) == 2
assert node_1._tool.options['input'].get() == '/infile.txt'
assert node_1._tool.options['output'].raw() == sys.stdout
assert node_2._tool.options['input'].raw() == sys.stdout
assert node_2._tool.options['output'].get() == '/outfile.txt'
def test_gt_to_option():
tool_1 = Tool(tool_1_def, "T1")
tool_2 = Tool(tool_1_def, "T2")
tool_3 = Tool(tool_1_def, "T3")
p = Pipeline()
node_1 = p.add(tool_1)
node_2 = p.add(tool_2)
node_3 = p.add(tool_3)
assert len(list(node_1.outgoing())) == 0
assert len(list(node_2.outgoing())) == 0
assert len(list(node_3.outgoing())) == 0
assert node_1._tool.options['output'] == sys.stdout
assert node_2._tool.options['input'] == sys.stdin
assert node_3._tool.options['input'] == sys.stdin
(node_1 > node_2.input) > node_3.input
n_1_out = node_1._tool.options['output'].raw()
n_2_in = node_2._tool.options['input'].raw()
n_2_out = node_2._tool.options['output'].raw()
n_3_in = node_3._tool.options['output'].raw()
assert n_1_out == n_2_in
assert n_2_out == n_3_in
assert len(list(node_1.outgoing())) == 1
assert len(list(node_2.incoming())) == 1
assert len(list(node_2.outgoing())) == 1
assert len(list(node_3.outgoing())) == 0
def test_three_groups():
tool_1 = Tool(tool_1_def, "T1")
tool_2 = Tool(tool_1_def, "T2")
tool_3 = Tool(tool_1_def, "T3")
p = Pipeline()
node_1 = p.add(tool_1)
node_2 = p.add(tool_2)
node_3 = p.add(tool_3)
node_1 >> (node_2 + node_3)
groups = list(p.groups())
assert len(groups) == 3
def test_groups_all_in_one():
tool_1 = Tool(tool_1_def, "T1")
tool_2 = Tool(tool_1_def, "T2")
tool_3 = Tool(tool_1_def, "T3")
p = Pipeline()
node_1 = p.add(tool_1)
node_2 = p.add(tool_2)
node_3 = p.add(tool_3)
node_1 | (node_2 + node_3)
groups = list(p.groups())
assert len(groups) == 1
assert set(groups[0]) == set([node_1, node_2, node_3])
def test_two_groups():
tool_1 = Tool(tool_1_def, "T1")
tool_2 = Tool(tool_1_def, "T2")
tool_3 = Tool(tool_1_def, "T3")
p = Pipeline()
p.add(tool_1)
node_2 = p.add(tool_2)
node_3 = p.add(tool_3)
node_2 | node_3
groups = list(p.groups())
assert len(groups) == 2
def test_expand_two_nodes_both_fan_out():
tool_1 = Tool(tool_1_def, "T1")
tool_2 = Tool(tool_1_def, "T2")
p = Pipeline()
node_1 = p.add(tool_1)
node_1.input = ["test_1.txt", "test_2.txt"]
node_2 = p.add(tool_2)
node_2.input = node_1.output
assert len(p._nodes) == 2
assert len(p._edges) == 1
p.expand(validate=False)
assert len(p._nodes) == 4
assert len(p._edges) == 2
def test_delegate_singleton_node_default_option():
tool_1 = Tool(tool_1_def)
tool_2 = Tool(tool_1_def)
p = Pipeline()
node_1 = p.add(tool_1)
node_2 = p.add(tool_2)
node_2.input = node_1
assert len(p._nodes) == 2
assert len(p._edges) == 1
edge = p.get_edge(node_1, node_2)
assert edge is not None
assert len(edge._links) == 1
def test_pipe_and_plus_operator():
tool_1 = Tool(tool_1_def, "T1")
tool_2 = Tool(tool_1_def, "T2")
tool_3 = Tool(tool_1_def, "T3")
p = Pipeline()
node_1 = p.add(tool_1)
node_2 = p.add(tool_2)
node_3 = p.add(tool_3)
node_1 | node_2 | node_3
assert len(p._edges) == 2
assert list(node_1.children()) == [node_2]
assert list(node_2.children()) == [node_3]
def test_add_operator():
tool_1 = Tool(tool_1_def, "T1")
tool_2 = Tool(tool_1_def, "T2")
tool_3 = Tool(tool_1_def, "T3")
p = Pipeline()
node_1 = p.add(tool_1)
node_2 = p.add(tool_2)
node_3 = p.add(tool_3)
node_1 | (node_2 + node_3)
assert len(p._edges) == 2
assert set(node_1.children()) == set([node_2, node_3])
assert len(node_2._tool.options['input']) == 1
assert len(node_2._tool.options['input']) == 1
assert node_2._tool.options['input'] == node_3._tool.options['input']
def test_left_shift_operator():
tool_1 = Tool(tool_1_def, "T1")
tool_2 = Tool(tool_1_def, "T2")
tool_3 = Tool(tool_1_def, "T3")
p = Pipeline()
node_1 = p.add(tool_1)
node_2 = p.add(tool_2)
node_3 = p.add(tool_3)
node_3 << node_1
node_3 << node_2
assert len(p._edges) == 2
assert list(node_1.children()) == [node_3]
assert list(node_2.children()) == [node_3]
assert list(node_3.children()) == []
def test_lt_from_file_name():
tool_1 = Tool(tool_1_def, "T1")
p = Pipeline()
node_1 = p.add(tool_1)
assert node_1.input == sys.stdin
node_1 < "A.txt"
assert node_1.input == "A.txt"
def test_gt_to_file_name():
tool_1 = Tool(tool_1_def, "T1")
p = Pipeline()
node_1 = p.add(tool_1)
assert node_1._tool.options['output'] == sys.stdout
node_1 > "A.txt"
assert node_1._tool.options['output'] == "A.txt"
def test_tool_validation_valid():
help_string = """\
Some Tool
Usage: tools [-i <input>] [-o <output>]
Options:
-i, --input <input> The input
[Default: stdin]
-o, --output <output> The output
[Default: stdout]
"""
tool = Tool(help_string)
assert tool.options is not None
assert len(tool.options) == 2
tool.validate()
def test_tool_empty_list_options():
help_string = """\
Some Tool
Usage: tools -i <input>... [-o <output>...]
Options:
-i, --input <input>... The input
-o, --output <output>... The output
"""
tool = Tool(help_string)
assert tool.options is not None
assert len(tool.options) == 2
tool.parse_args(['-i', 'A', 'B'])
assert tool.options['input'].raw() == ['A', 'B']
assert tool.options['output'].raw() == []
def test_get_node_properties():
tool = Tool(tool_1_def)
p = Pipeline()
node = p.add(tool)
assert isinstance(node.input, Option)
with pytest.raises(AttributeError) as ex:
node.xxx
assert str(ex.value) == "Option 'xxx' not found in tools"
def test_tool_validation_missing_required():
help_string = """\
Some Tool
Usage: tools -i <input> [-o <output>]
Options:
-i, --input <input> The input
-o, --output <output> The output
[Default: stdout]
"""
tool = Tool(help_string)
assert tool.options is not None
assert len(tool.options) == 2
with pytest.raises(ValidationError) as execinfo:
tool.validate()
assert "required" in str(execinfo.value)
def test_delegate_list_node_default_option():
tool_1 = Tool(tool_1_def)
tool_2 = Tool(tool_1_def)
tool_3 = Tool(tool_1_def)
p = Pipeline()
node_1 = p.add(tool_1)
node_2 = p.add(tool_2)
node_3 = p.add(tool_3)
node_3.input = [node_1, node_2]
assert len(p._edges) == 2
edge = p.get_edge(node_1, node_3)
assert edge is not None
assert len(edge._links) == 1
edge_2 = p.get_edge(node_2, node_3)
assert edge_2 is not None
assert len(edge_2._links) == 1
def test_find_fanout_options():
tool = Tool(tool_1_def)
p = Pipeline()
node = p.add(tool)
node.input = ["test_1.txt", "test_2.txt"]
assert len(node.input.value) == 2
assert len(node.input) == 2
assert p._get_fanout_options(node) == [node.input]
def test_tool_validation_unknown():
help_string = """\
Some Tool
Usage: tools [-i <input>] [-o <output>]
Options:
-i, --input <input> The input
[Default: stdin]
-o, --output <output> The output
[Default: stdout]
"""
tool = Tool(help_string)
assert tool.options is not None
assert len(tool.options) == 2
with pytest.raises(ParserException) as execinfo:
tool.parse_args(["-x"])
assert str(execinfo.value) == "tools :: unrecognized arguments: -x"
def test_expand_three_nodes_two_fan_out():
tool_1 = Tool(tool_1_def, "T1")
tool_2 = Tool(tool_1_def, "T2")
tool_3 = Tool(tool_1_def, "T3")
p = Pipeline()
node_1 = p.add(tool_1)
node_1.input = ["test_1.txt", "test_2.txt"]
node_2 = p.add(tool_2)
node_2.input = node_1.output
node_3 = p.add(tool_3)
node_3.x = "other"
node_3 = p.add(tool_3)
node_2.x = node_3.x
assert len(p._nodes) == 3
assert len(p._edges) == 2
p.expand(validate=False)
assert len(p._nodes) == 5
assert len(p._edges) == 6
def test_lt_from_option():
tool_1 = Tool(tool_1_def, "T1")
tool_2 = Tool(tool_1_def, "T2")
tool_3 = Tool(tool_1_def, "T3")
p = Pipeline()
node_1 = p.add(tool_1)
node_2 = p.add(tool_2)
node_3 = p.add(tool_3)
assert len(list(node_1.outgoing())) == 0
assert len(list(node_2.outgoing())) == 0
assert len(list(node_3.outgoing())) == 0
assert node_1.output == sys.stdout
assert node_2.input == sys.stdin
assert node_3.input == sys.stdin
(node_1 < node_2.output) < node_3.output
assert not node_3.has_incoming()
assert node_2.has_incoming(node_3, ('output', 'input'), True)
assert node_1.has_incoming(node_2, ('output', 'input'), True)
def test_set_node_properties():
tool = Tool(tool_1_def)
p = Pipeline()
node = p.add(tool)
opt = node.input
assert isinstance(opt, Option)
node.input = "test.txt"
assert opt.raw() == "test.txt"
with pytest.raises(AttributeError) as ex:
node.xxx = "A"
assert str(ex.value) == "Option 'xxx' not found in tools"
def test_right_shift_operator():
tool_1 = Tool(tool_1_def, "T1")
tool_2 = Tool(tool_1_def, "T2")
tool_3 = Tool(tool_1_def, "T3")
p = Pipeline()
node_1 = p.add(tool_1)
node_2 = p.add(tool_2)
node_3 = p.add(tool_3)
node_1.output = "A"
node_2.output = "B"
node_3.output = "C"
node_1 >> node_3
node_2 >> node_3
assert len(p._edges) == 2
assert list(node_1.children()) == [node_3]
assert list(node_2.children()) == [node_3]
assert list(node_3.children()) == []
for e in node_3._edges:
for link in e._links:
assert not link[2]
def test_gt_to_option_no_blocks():
tool_1 = Tool(tool_1_def, "T1")
tool_2 = Tool(tool_1_def, "T2")
tool_3 = Tool(tool_1_def, "T3")
p = Pipeline()
node_1 = p.add(tool_1)
node_2 = p.add(tool_2)
node_3 = p.add(tool_3)
assert len(list(node_1.outgoing())) == 0
assert len(list(node_2.outgoing())) == 0
assert len(list(node_3.outgoing())) == 0
assert node_1.output == sys.stdout
assert node_2.input == sys.stdin
assert node_3.input == sys.stdin
node_1 > node_2.input # this does not work in a single line!
node_2 > node_3.input
assert not node_3.input.raw() == sys.stdin
# check the graph structure
assert node_2.has_incoming(node_1, ('output', 'input'), True,
node_1.output)
assert node_3.has_incoming(node_2, ('output', 'input'), True,
node_2.output)
assert not node_3.has_outgoing()
def test_expand_single_node():
tool = Tool(tool_1_def)
p = Pipeline()
node = p.add(tool)
node.input = ["test_1.txt", "test_2.txt"]
p.expand(validate=False)
assert len(p._nodes) == 2
assert len(p._edges) == 0
inputs = []
for node in p.nodes():
inputs.append(node.input.get())
assert sorted(inputs) == [
os.path.join(os.getcwd(), "test_1.txt"),
os.path.join(os.getcwd(), "test_2.txt")
]
def test_tool_validate_input_file_not_found():
help_string = """\
Some Tool
Usage: tools [-i <input>] [-o <output>]
Options:
-i, --input <input> The input
[Default: stdin]
-o, --output <output> The output
[Default: stdout]
"""
tool = Tool(help_string)
tool.parse_args(["-i", "unknown"])
with pytest.raises(ValidationError) as execinfo:
tool.validate()
assert "Input file not found: unknown" in str(execinfo.value)
def validate(self, args, incoming=None):
Tool.validate(self, args, incoming)
# check that this is a gem index
if not args["index"].endswith(".gem"):
raise ValidationException({"index": "The given index does not seem to be a gem index!"})
def test_node_equality():
p = Pipeline()
tool = Tool(tool_1_def)
p.add(tool)
p.add(tool)
assert len(p._nodes) == 1
