def test_static_type_validation():
"""When initializing a port to a static value, the type is immediately
validated"""
graph = Graph()
with pytest.raises(FlowError):
graph.add_component("Repeat", Repeat, COUNT='foo')
def test_array_connections():
graph = Graph()
gen = graph.add_component("Generate", GenerateArray)
dis1 = graph.add_component("Discard1", Discard)
graph.add_component("Discard2", Discard)
assert gen.ports['OUT'].get_full_name() == 'Generate.OUT'
# non-fixed array ports delay element creation
assert gen.ports.OUT.fixed_size is None
assert names(gen.outports) == []
# assert list(gen.ports._ports.keys()) == ['OUT', 'NULL']
assert type(gen.ports['OUT']) is OutputArray
assert type(dis1.ports['IN']) is InputPort
# nothing is connected yet
# assert gen.ports['OUT'].is_connected() is False
assert dis1.ports['IN'].is_connected() is False
# make a connection
graph.connect("Generate.OUT[1]", "Discard1.IN")
assert names(gen.outports) == ['OUT[1]']
assert gen.ports['OUT'][1].is_connected() is True
# uses first unused index (index 0)
graph.connect("Generate.OUT", "Discard2.IN")
assert names(gen.outports) == ['OUT[0]', 'OUT[1]']
assert gen.ports['OUT'][0].is_connected() is True
assert type(gen.ports['OUT']) is OutputArray
assert type(gen.ports['OUT'][0]) is OutputPort
assert type(dis1.ports['IN']) is InputPort
# assert gen.ports['OUT'].is_connected() is False
assert dis1.ports['IN'].is_connected() is True
def test_optional_fixed_size_array_error():
"""if an array port specifies fixed_size, all elements must be
connected if the array port is required"""
graph = Graph()
gen = graph.add_component("Generate", GenerateFixedSizeArray)
assert gen.ports['OUT'].required is True
gen.init()
with pytest.raises(FlowError):
gen.ports.OUT.validate()
def test_stream_initialization():
graph = Graph()
# a single packet with a list of ints
passthru1 = graph.add_component("Pass1", SlowPass, IN=[1, 2, 3],
DELAY=0.1)
# a stream of int packets
passthru2 = graph.add_component("Pass2", SlowPass, IN=Stream([1, 2, 3]),
DELAY=0.1)
assert passthru1.ports.IN._connection._content == [[1, 2, 3]]
assert passthru2.ports.IN._connection._content == [1, 2, 3]
def test_get_spec():
sub = Graph()
sub.add_component('Head', Passthru)
sub.add_component('Tail', Passthru)
sub.connect('Head.OUT', 'Tail.IN')
sub.export('Head.IN', 'IN')
sub.export('Tail.OUT', 'OUT')
PassNet = make_subgraph(sub, name='PassNet')
spec = PassNet.get_spec()
assert spec['name'] == 'abc/PassNet'
assert len(spec['inPorts']) == 2
assert len(spec['outPorts']) == 2
def new_graph(self, graph_id, description=None, metadata=None):
"""
Create a new graph.
"""
self.logger.debug('Graph {}: Initializing'.format(graph_id))
self.add_graph(
graph_id,
Graph(name=graph_id, description=description, metadata=metadata))
def test_inport_default():
graph = Graph()
graph.add_component("Generate", GenerateTestData)
dis = graph.add_component("Discard", Discard)
graph.connect("Generate.OUT", "Discard.IN")
run_graph(graph)
assert dis.values == ['000001']
def _new_graph(self, graph_id):
"""
Create a new graph.
"""
self.logger.debug('Graph {}: Initializing'.format(graph_id))
# FIXME: set graph name to graph_id?
graph = Graph(graph_id)
self.add_graph(graph_id, graph)
return graph
def test_component_with_inheritance():
@inport('IN')
@outport('OUT')
class A(Component):
def execute(self):
pass
@inport('OPT', type=int)
class B(A):
pass
assert names(B.port_definitions().values(), include_null=True) == [
IN_NULL, 'IN', 'OPT', OUT_NULL, 'OUT']
graph = Graph()
b = graph.add_component('b', B)
assert names(b.ports, include_null=True) == [
IN_NULL, 'IN', 'OPT', OUT_NULL, 'OUT']
def test_network_apply():
graph = Graph()
graph.add_component('Add1', Add)
graph.add_component('Add2', Add)
graph.connect('Add1.OUT', 'Add2.IN1')
graph.export('Add1.IN1', 'IN1')
graph.export('Add1.IN2', 'IN2')
graph.export('Add2.IN2', 'IN3')
graph.export('Add2.OUT', 'OUT')
outputs = run_graph(graph, {
'IN1': 1,
'IN2': 3,
'IN3': 6
}, capture_results=True)
assert outputs['OUT'] == 10
def test_set_connection_metadata():
graph = Graph()
gen = graph.add_component("generate", GenerateTestData, COUNT=10)
dis = graph.add_component("dis", Discard)
graph.connect("generate.OUT", "dis.IN", metadata={'route': 4})
outport = graph.get_component_port(('generate', 'OUT'), kind='out')
inport = graph.get_component_port(('dis', 'IN'), kind='in')
metadata = inport._connection.metadata.get(outport)
assert metadata == {'route': 4}
graph.set_edge_metadata(outport, inport, metadata={
'label': 'test',
'route': None
})
metadata = inport._connection.metadata.get(outport)
assert metadata == {'label': 'test'}
def test_network_export():
graph = Graph()
passthru = graph.add_component("Pass", SlowPass, DELAY=0.1)
graph.export('Pass.OUT', 'OUT')
graph.export('Pass.IN', 'IN')
assert len(graph.inports.keys()) == 1
assert len(graph.outports.keys()) == 1
def _init_graph(cls):
"""
Initialize the graph.
This is a classmethod so that exported port definitions can be
inspected without instantiating the SubGraph, which is a requirement
for all Components.
"""
cls.subgraph = Graph()
cls.define(cls.subgraph)
assert cls.subgraph is not None
def test_name():
root = Graph(name='root')
passnet = root.add_component("Subnet", PassthruNet)
child = passnet.subgraph.component("Pass")
assert passnet._runner is None
assert passnet.get_parents() == []
assert passnet.get_full_name() == 'Subnet'
assert child._runner is None
assert child.get_parents() == []
assert child.get_full_name() == 'Pass'
# a component's full name is not available until all of the graphs' and
# sub-graphs' runners have been initialized.
# this is the result of two conflicting requirements:
# - graphs should not be modified during execution. in other words,
# there should be no side-effects. this means that a graph cannot have
# a parent graph attribute, we must track that relationship elsewhere.
# - delay building runners until absolutely necessary. this means that
# a component, which *can* have a parent, is not available until the
# graph is executed.
# we might want to revisit this if it becomes a nuisance.
net = Network(root)
net._build_runners()
assert passnet._runner is not None
assert passnet.get_parents() == [root]
# building the network does a deep copy of the graph and its components
snet = passnet._build_network()
assert snet.graph is not passnet.subgraph
assert child is passnet.subgraph.component('Pass')
child_copy = snet.graph.component('Pass')
assert child is not child_copy
assert type(child) is type(child_copy)
snet._build_runners()
assert child_copy._runner is not None
assert snet.parent_network is not None
assert child_copy.get_full_name() == 'root.Subnet.Pass'
def _new_graph(self, graph_id, description=None, metadata=None,
overwrite=True):
"""
Create a new graph.
"""
if not overwrite and self._graphs.get(graph_id, None):
raise FlowError('Graph already exists')
self.logger.debug('Graph {}: Initializing'.format(graph_id))
self.add_graph(graph_id, Graph(
name=graph_id,
description=description,
metadata=metadata
))
def test_network_deserialization(serialized_graph):
comp_map = {
'rill.components.basic/Counter': Counter,
'rill.components.merge/Group': Group,
'tests.components/Discard': Discard,
'tests.subnets/PassthruNet': PassthruNet,
'tests.components/GenerateArray': GenerateArray
}
graph = Graph.from_dict(serialized_graph, component_lookup=comp_map)
assert len(graph.get_components().keys()) == 5
Counter1 = graph.get_component('Counter1')
Discard1 = graph.get_component('Discard1')
Pass = graph.get_component('Pass')
Generate = graph.get_component('Generate')
Merge = graph.get_component('Merge')
assert Counter1.ports.OUT._connections[0].inport.component is Pass
assert Counter1.metadata == {
'x': 20.0,
'y': 300.5
}
assert Pass.ports.OUT._connections[0].inport.component is Discard1
assert Counter1.ports.IN._connection._content == [5]
assert (
Generate.ports.OUT.get_element(0)._connections[0].inport.component is Merge
)
assert (
Generate.ports.OUT.get_element(1)._connections[0].inport.component is Merge
)
assert (
Generate.ports.OUT.get_element(0)._connections[0].inport.index is 1
)
assert (
Generate.ports.OUT.get_element(1)._connections[0].inport.index is 2
)
expected = graph.to_dict()
# Order of connections array shouldn't matter
expected['connections'] = sorted(expected['connections'], key=str)
assert serialized_graph == expected
def test_required_array_error():
"""if fixed_size is provided and the array port is required, all elements
must be connected"""
graph = Graph()
gen = graph.add_component("Generate", GenerateFixedSizeArray)
assert gen.ports['OUT'].required is True
graph.add_component("Discard1", Discard)
graph.connect("Generate.OUT[0]", "Discard1.IN")
assert names(gen.outports) == ['OUT[0]', 'OUT[1]']
gen.init()
with pytest.raises(FlowError):
gen.ports.OUT.validate()
def test_add_component():
graph = Graph()
gen = graph.add_component("generate", GenerateTestData, COUNT=10)
assert type(gen.ports.COUNT) is InputPort
assert gen.ports.COUNT.is_initialized()
assert gen.ports.COUNT._connection._content == [10]
assert type(gen.ports.COUNT._connection._content) is list
with pytest.raises(FlowError):
# component already exists
graph.add_component("generate", GenerateTestData)
def foo(): pass
with pytest.raises(TypeError):
graph.add_component("not_component", foo)
def test_initialize_subnet():
@outport("OUT")
@inport("IN")
@subnet
def PassNet(sub):
sub.add_component('Head', Passthru)
sub.add_component('Tail', Passthru)
sub.connect('Head.OUT', 'Tail.IN')
sub.export('Head.IN', 'IN')
sub.export('Tail.OUT', 'OUT')
graph = Graph()
capture = graph.add_component('Capture', Capture)
graph.add_component('Pass', PassNet)
graph.initialize(5, 'Pass.IN')
graph.connect('Pass.OUT', 'Capture.IN')
run_graph(graph)
assert capture.value == 5
def test_subnet_decorator():
@outport("OUT")
@inport("IN", description='an input')
@subnet
def DecoratedPassNet(sub):
sub.add_component('Head', SlowPass, DELAY=0.01)
sub.add_component('Tail', SlowPass, DELAY=0.01)
sub.connect('Head.OUT', 'Tail.IN')
sub.export('Head.IN', 'IN')
sub.export('Tail.OUT', 'OUT')
assert issubclass(DecoratedPassNet, SubGraph)
assert DecoratedPassNet._inport_definitions[0].description == 'an input'
assert DecoratedPassNet.inport_definitions['IN'].description == 'an input'
graph = Graph()
gen = graph.add_component("Generate", GenSS, COUNT=5)
passnet = graph.add_component("Subnet", DecoratedPassNet)
dis = graph.add_component("Discard", Discard)
graph.connect("Generate.OUT", "Subnet.IN")
graph.connect("Subnet.OUT", "Discard.IN")
run_graph(graph)
assert dis.values == [
'',
'000005',
'000004',
'000003',
'000002',
'000001',
'',
]
def test_rename_exports():
graph = Graph()
graph.add_component('Head', Passthru)
graph.add_component('Tail', Passthru)
graph.connect('Head.OUT', 'Tail.IN')
graph.export('Head.IN', 'IN')
graph.export('Tail.OUT', 'OUT')
graph.set_inport_metadata('IN', {'x': 100})
graph.set_outport_metadata('OUT', {'y': 100})
graph.rename_inport('IN', 'HAPPY')
assert graph.inports['HAPPY'] == graph.get_component_port('Head.IN')
assert graph.inport_metadata['HAPPY'] == {'x': 100}
assert not graph.inports.get('IN', False)
assert not graph.inport_metadata.get('IN', False)
graph.rename_outport('OUT', 'SAD')
assert graph.outports['SAD'] == graph.get_component_port('Tail.OUT')
assert graph.outport_metadata['SAD'] == {'y': 100}
assert not graph.outports.get('OUT', False)
assert not graph.outport_metadata.get('OUT', False)
def graph(request):
return Graph(**request.param)
def test_network_apply_with_outputs():
graph = Graph()
graph.add_component('Add1', Add)
graph.add_component('Add2', Add)
graph.add_component('Kick', Kick)
graph.connect('Add1.OUT', 'Add2.IN1')
graph.export('Add1.IN1', 'IN1')
graph.export('Add1.IN2', 'IN2')
graph.export('Add2.IN2', 'IN3')
graph.export('Add2.OUT', 'OUT')
graph.export('Kick.OUT', 'Kick_OUT')
outputs = run_graph(graph, {
'IN1': 1,
'IN2': 3,
'IN3': 6
}, ['OUT'])
assert outputs == {'OUT': 10}
def test_fixed_array_connections():
graph = Graph()
gen = graph.add_component("Generate", GenerateFixedSizeArray)
dis1 = graph.add_component("Discard1", Discard)
graph.add_component("Discard2", Discard)
assert gen.ports['OUT'].get_full_name() == 'Generate.OUT'
# fixed array ports create their ports immediately
assert gen.ports.OUT.fixed_size == 2
assert names(gen.outports) == ['OUT[0]', 'OUT[1]']
# assert list(gen.ports._ports.keys()) == ['OUT', 'NULL']
assert type(gen.ports['OUT']) is OutputArray
assert type(dis1.ports['IN']) is InputPort
# nothing is connected yet
# assert gen.ports['OUT'].is_connected() is False
assert dis1.ports['IN'].is_connected() is False
# make a connection
graph.connect("Generate.OUT[1]", "Discard1.IN")
assert names(gen.outports) == ['OUT[0]', 'OUT[1]']
assert gen.ports['OUT'][1].is_connected() is True
# uses first unconnected index (index 0)
graph.connect("Generate.OUT", "Discard2.IN")
assert gen.ports['OUT'][0].is_connected() is True
# outports can only have more than one connection
graph.connect("Generate.OUT[1]", "Discard2.IN")
with pytest.raises(FlowError):
# cannot connect outside the fixed range
graph.connect("Generate.OUT[2]", "Discard2.IN")
assert type(gen.ports['OUT']) is OutputArray
assert type(gen.ports['OUT'][0]) is OutputPort
assert type(dis1.ports['IN']) is InputPort
# assert gen.ports['OUT'].is_connected() is False
assert dis1.ports['IN'].is_connected() is True
def test_required_port_error():
graph = Graph()
graph.add_component("Generate", GenerateFixedSizeArray)
graph.add_component("Discard1", Discard)
with pytest.raises(FlowError):
graph.validate()
def test_basic_connections():
graph = Graph()
count = graph.add_component("Count", Counter)
dis = graph.add_component("Discard1", Discard)
graph.add_component("Discard2", Discard)
# ports are stored in the order they are declared
assert names(count.outports) == ['OUT', 'COUNT']
# assert list(count.ports._ports.keys()) == ['OUT', 'COUNT', 'NULL']
# nothing is connected yet
assert count.ports['OUT'].is_connected() is False
assert dis.ports['IN'].is_connected() is False
with pytest.raises(FlowError):
# non-existent port
graph.connect("Count.FOO", "Discard1.IN")
with pytest.raises(FlowError):
# non-existent Component
graph.connect("Foo.FOO", "Discard1.IN")
# make a connection
graph.connect("Count.OUT", "Discard1.IN")
# outports can have more than one connection
graph.connect("Count.OUT", "Discard2.IN")
with pytest.raises(FlowError):
# connected ports cannot be initialized
graph.initialize(1, "Discard1.IN")
assert type(count.ports['OUT']) is OutputPort
assert type(dis.ports['IN']) is InputPort
assert count.ports['OUT'].is_connected() is True
assert dis.ports['IN'].is_connected() is True
# FIXME: move this to a different test
net = Network(graph)
net.reset()
net._build_runners()
net._open_ports()
assert count.ports['OUT'].component is count
assert isinstance(count.ports['OUT'].sender, ComponentRunner)
assert dis.ports['IN'].component is dis
assert isinstance(dis.ports['IN'].receiver, ComponentRunner)
def get_graph(graph_name):
graph = Graph(name=graph_name)
gen = graph.add_component('Generate', GenerateTestData)
gen.metadata['x'] = 5
gen.metadata['y'] = 5
passthru = graph.add_component('Pass', Passthru)
outside = graph.add_component('Outside', Passthru)
graph.connect('Generate.OUT', 'Pass.IN')
graph.connect('Outside.OUT', 'Pass.IN')
graph.initialize(5, 'Generate.COUNT')
graph.export('Pass.OUT', 'OUTPORT')
graph.export('Outside.IN', 'INPORT')
return graph, gen, passthru, outside
if line:
line += " "
line += word
if line:
# remainder
OUT.send(line)
@component
@outport("OUT", type=str)
@inport("IN", type=str)
def LineToWords(IN, OUT):
for line in IN.iter_contents():
words = line.split()
for word in words:
OUT.send(word)
net = Graph()
net.add_component("LineToWords", LineToWords, IN="HeLLo Goodbye World")
net.add_component("StartsWith", StartsWith, TEST='G')
net.add_component("WordsToLine", WordsToLine)
dis = net.add_component("Output", Output)
net.connect("LineToWords.OUT", "StartsWith.IN")
net.connect("StartsWith.REJ", "WordsToLine.IN")
net.connect("WordsToLine.OUT", "Output.IN")
net =Network(net)
net.go()
def test_make_subgraph():
sub = Graph()
sub.add_component('Head', Passthru)
sub.add_component('Tail', Passthru)
sub.connect('Head.OUT', 'Tail.IN')
sub.export('Head.IN', 'IN')
sub.export('Tail.OUT', 'OUT')
PassNet = make_subgraph(sub, name='PassNet')
assert len(PassNet.inport_definitions) == 2
assert len(PassNet.outport_definitions) == 2
graph = Graph()
capture = graph.add_component('Capture', Capture)
graph.add_component('Pass', PassNet)
graph.initialize(5, 'Pass.IN')
graph.connect('Pass.OUT', 'Capture.IN')
run_graph(graph)
assert capture.value == 5
import ast
from rill.engine.network import Graph, Network
from rill.components.hello_world import LineToWords, StartsWith, WordsToLine, Output
graph_file = open('example_serialized.txt')
graph_str = graph_file.read()
graph_file.close()
graph_from_file = ast.literal_eval(graph_str)
graph = Graph.from_dict(graph_from_file)
net = Network(graph)
net.go()
