def _build_network(self):
# the graph is a class attribute, so we have to make a deep copy to
# avoid side-effects
graph = self.subgraph.copy()
for (name, internal_port) in graph.inports.items():
subcomp = graph.add_component('_' + name, SubIn)
graph.initialize(self.ports[name], subcomp.ports.PROXIED)
graph.connect(subcomp.ports.OUT, internal_port)
for (name, internal_port) in graph.outports.items():
subcomp = graph.add_component('_' + name, SubOut)
graph.initialize(self.ports[name], subcomp.ports.PROXIED)
graph.connect(internal_port, subcomp.ports.IN)
# don't do deadlock testing in sub graphs - you need to consider
# the whole graph!
network = Network(graph, deadlock_test_interval=None)
# set the network parent. this allows runners within the network to
# walk up the network parents to the root
network.parent_network = self._runner.parent_network
return network
def test_name():
root = Graph(name='root')
passnet = root.add_component("Subnet", PassthruNet)
child = passnet.subgraph.component("Pass")
assert passnet._runner is None
assert passnet.get_parents() == []
assert passnet.get_full_name() == 'Subnet'
assert child._runner is None
assert child.get_parents() == []
assert child.get_full_name() == 'Pass'
# a component's full name is not available until all of the graphs' and
# sub-graphs' runners have been initialized.
# this is the result of two conflicting requirements:
# - graphs should not be modified during execution. in other words,
# there should be no side-effects. this means that a graph cannot have
# a parent graph attribute, we must track that relationship elsewhere.
# - delay building runners until absolutely necessary. this means that
# a component, which *can* have a parent, is not available until the
# graph is executed.
# we might want to revisit this if it becomes a nuisance.
net = Network(root)
net._build_runners()
assert passnet._runner is not None
assert passnet.get_parents() == [root]
# building the network does a deep copy of the graph and its components
snet = passnet._build_network()
assert snet.graph is not passnet.subgraph
assert child is passnet.subgraph.component('Pass')
child_copy = snet.graph.component('Pass')
assert child is not child_copy
assert type(child) is type(child_copy)
snet._build_runners()
assert child_copy._runner is not None
assert snet.parent_network is not None
assert child_copy.get_full_name() == 'root.Subnet.Pass'
def test_basic_connections():
graph = Graph()
count = graph.add_component("Count", Counter)
dis = graph.add_component("Discard1", Discard)
graph.add_component("Discard2", Discard)
# ports are stored in the order they are declared
assert names(count.outports) == ['OUT', 'COUNT']
# assert list(count.ports._ports.keys()) == ['OUT', 'COUNT', 'NULL']
# nothing is connected yet
assert count.ports['OUT'].is_connected() is False
assert dis.ports['IN'].is_connected() is False
with pytest.raises(FlowError):
# non-existent port
graph.connect("Count.FOO", "Discard1.IN")
with pytest.raises(FlowError):
# non-existent Component
graph.connect("Foo.FOO", "Discard1.IN")
# make a connection
graph.connect("Count.OUT", "Discard1.IN")
# outports can have more than one connection
graph.connect("Count.OUT", "Discard2.IN")
with pytest.raises(FlowError):
# connected ports cannot be initialized
graph.initialize(1, "Discard1.IN")
assert type(count.ports['OUT']) is OutputPort
assert type(dis.ports['IN']) is InputPort
assert count.ports['OUT'].is_connected() is True
assert dis.ports['IN'].is_connected() is True
# FIXME: move this to a different test
net = Network(graph)
net.reset()
net._build_runners()
net._open_ports()
assert count.ports['OUT'].component is count
assert isinstance(count.ports['OUT'].sender, ComponentRunner)
assert dis.ports['IN'].component is dis
assert isinstance(dis.ports['IN'].receiver, ComponentRunner)
def runtwice(grph, *pairs):
"""Run a network twice to ensure that it shuts down properly"""
network = Network(grph)
with gevent.Timeout(2):
network.go()
for real, ref in pairs:
assert real.values == ref
real.values = []
network.go()
for real, ref in pairs:
assert real.values == ref
def start(self, graph_id, done_callback):
"""
Execute a graph.
"""
self.logger.debug('Graph {}: Starting execution'.format(graph_id))
graph = self.get_graph(graph_id)
network = Network(graph)
executor = gevent.Greenlet(network.go)
# FIXME: should we delete the executor from self._executors on finish?
# this has an impact on the result returned from get_status(). Leaving
# it means that after completion it will be started:True, running:False
# until stop() is triggered, at which point it will be started:False,
# running:False
executor.link(lambda g: done_callback())
self._executors[graph_id] = (executor, network)
executor.start()
def test_pickle(graph):
graph.add_component("Generate", GenerateTestData, COUNT=5)
passthru = graph.add_component("Pass", SlowPass, DELAY=0.1)
count = graph.add_component("Counter", Counter)
dis1 = graph.add_component("Discard1", Discard)
dis2 = graph.add_component("Discard2", Discard)
graph.connect("Generate.OUT", "Pass.IN")
graph.connect("Pass.OUT", "Counter.IN")
graph.connect("Counter.COUNT", "Discard1.IN")
graph.connect("Counter.OUT", "Discard2.IN")
net = Network(graph)
netrunner = gevent.spawn(net.go)
try:
with gevent.Timeout(.35) as timeout:
gevent.wait([netrunner])
except gevent.Timeout:
print(count.execute)
assert count.count == 4
assert dis2.values == ['000005', '000004', '000003', '000002']
import pickle
# dump before terminating to get the runner statuses
data = pickle.dumps(net)
# FIXME: do we need to auto-terminate inside wait_for_all if there is an error?
net.terminate()
net.wait_for_all()
# gevent.wait([netrunner]) # this causes more packets to be sent. no good.
net2 = pickle.loads(data)
assert net2.graph.component('Counter').count == 4
assert net2.graph.component('Discard2').values == [
'000005', '000004', '000003', '000002'
]
net2.go(resume=True)
assert net2.graph.component('Counter').count == 5
assert net2.graph.component('Discard2').values == [
'000005', '000004', '000003', '000002', '000001'
]
if line:
line += " "
line += word
if line:
# remainder
OUT.send(line)
@component
@outport("OUT", type=str)
@inport("IN", type=str)
def LineToWords(IN, OUT):
for line in IN.iter_contents():
words = line.split()
for word in words:
OUT.send(word)
net = Graph()
net.add_component("LineToWords", LineToWords, IN="HeLLo Goodbye World")
net.add_component("StartsWith", StartsWith, TEST='G')
net.add_component("WordsToLine", WordsToLine)
dis = net.add_component("Output", Output)
net.connect("LineToWords.OUT", "StartsWith.IN")
net.connect("StartsWith.REJ", "WordsToLine.IN")
net.connect("WordsToLine.OUT", "Output.IN")
net =Network(net)
net.go()
import ast
from rill.engine.network import Graph, Network
from rill.components.hello_world import LineToWords, StartsWith, WordsToLine, Output
graph_file = open('example_serialized.txt')
graph_str = graph_file.read()
graph_file.close()
graph_from_file = ast.literal_eval(graph_str)
graph = Graph.from_dict(graph_from_file)
net = Network(graph)
net.go()