def initialize(self, static_value):
"""
Initialize a port to a static value.
Parameters
----------
static_value : Any
"""
if self.is_null():
raise FlowError("Cannot initialize null port: {}".format(self))
if self.is_connected():
if not self.is_initialized():
raise FlowError(
"Port cannot have both an initial packet and a "
"connection: {}".format(self))
else:
raise FlowError("Port is already initialized: {}".format(self))
# a Stream instance indicates that each item in the stream should be
# yielded as a separate packet. This is used to differentiate between
# a port which may expect a single packet to contain a list of data.
if isinstance(static_value, Stream):
if self.auto_receive:
raise FlowError("Static port initialized with a stream of "
"data: {}".format(self))
content = [self.validate_packet_contents(p) for p in static_value]
else:
content = [self.validate_packet_contents(static_value)]
self._connection = InitializationConnection(content, self)
def recv_message(self, msg):
"""
Handle a FBP graph message
Parameters
----------
msg: rill.runtime.plumbing.Message
"""
if msg.protocol == 'graph' and msg.command == 'clear':
payload = msg.payload.copy()
graph_id = payload.pop('id')
self._new_graph(graph_id, **payload)
elif msg.protocol == 'graph' and msg.command == 'addgraph':
payload = msg.payload.copy()
graph_id = payload.pop('id')
self._new_graph(graph_id, overwrite=False, **payload)
else:
# automatic handling
try:
method_name = self.COMMAND_TO_METHOD[msg.protocol][msg.command]
except KeyError:
raise FlowError("Unknown command '%s' for protocol '%s'" %
(msg.command, msg.protocol))
method = getattr(self, method_name)
method(**_toargs(msg.payload))
def resume(self):
"""
Resume a graph that has been suspended.
"""
self_starters = []
for runner in self.runners:
for port in runner.component.inports:
if port.is_connected() and not port.is_initialized() and \
port._connection._queue:
logger.info("Existing data in connection buffer: {}",
args=[port])
if runner.status in (StatusValues.TERMINATED, StatusValues.ERROR):
runner.kill()
continue
elif runner.self_starting or \
runner.status in (StatusValues.SUSP_RECV,
StatusValues.SUSP_SEND,
StatusValues.DORMANT,
StatusValues.ACTIVE):
self_starters.append(runner)
if not self_starters:
raise FlowError("No self-starters found")
for runner in self_starters:
runner.activate()
def connect(self, inport, outport, capacity, topics=['all']):
"""
Connect an ``InputPort`` to an ``OutputPort``.
Parameters
----------
inport : ``rill.engine.inputport.InputPort``
outport : ``rill.engine.outputport.OutputPort``
capacity : int
size of the buffer
"""
if self.outports:
if capacity != self.capacity():
# a previous connect specified a destination port with the same
# name and index.
raise FlowError(
"{}: Connection capacity does not agree with previous "
"specification".format(self))
else:
self._queue = deque(maxlen=capacity)
self.topics=topics
self.inport = inport
self.outports.add(outport)
outport._connections.append(self)
for topic in topics:
if not topic in outport._topic_map.keys():
outport._topic_map[topic]=[]
outport._topic_map[topic].append(self)
def validate(self):
self._check_port_types()
if not self.ports() and self.required:
raise FlowError("Required {} {} has no members".format(
self.port_class.__name__, self))
if self.fixed_size is not None and self.required:
missing = []
for port in self.ports():
if not port.is_connected():
missing.append(str(port))
if missing:
raise FlowError(
"Required {} {} has missing elements: {}".format(
self.port_class.__name__, self, ', '.join(missing)))
def validate(self):
"""
Runs prior to opening a port to validate that the port can be opened.
"""
if not self.is_connected() and self.required:
raise FlowError(
"{} port is required, but not connected".format(self))
def uninitialize(self):
"""
Remove static value initialization from the port.
Returns
-------
``InitializationConnection``
The removed initialization connection
"""
if self.is_null():
raise FlowError("Cannot uninitialize null port: {}".format(self))
if not self.is_initialized():
raise FlowError("Port is not initialized: {}".format(self))
conn = self._connection
self._connection = None
return conn
def port(self, port_name, kind=None):
"""
Get a port on the component.
Handles names with array indices (e.g. 'IN[0]')
Parameters
----------
port_name : str
kind : {'in', 'out'} or None
assert that the retrieved port is of the given type
Returns
-------
port : ``rill.engine.outputport.OutputPort`` or
``rill.engine.outputport.OutputArray`` or
``rill.engine.inputport.InputPort`` or
``rill.engine.inputport.InputArray``
"""
reg = re.compile(PORT_NAME_REG)
m = reg.match(port_name)
if not m:
raise FlowError("Invalid port name: " + port_name)
port_name, index = m.groups()
if index is not None:
index = int(index)
try:
port = self.ports[port_name]
except KeyError as err:
raise FlowError(str(err))
if kind is not None and port.kind != kind:
raise FlowError("Expected {} port: got {}".format(
kind, type(port)))
if index is not None:
if not port.is_array():
raise FlowError("Element {} specified for non-array "
"port {}".format(index, port))
port = port.get_element(index, create=True)
return port
def run_graph(graph, initializations=None, capture_results=False):
"""
Run a graph.
Parameters
----------
graph : ``rill.engine.network.Graph``
initializations : Optional[Dict[str, Any]]
map of exported inport names to initial content
capture_results : Union[bool, List[str]]
list of exported outport names whose results should be collected.
True for all, False for none.
Returns
-------
Dict[str, Any]
map network outport names to captured values
"""
from rill.components.basic import Capture
if capture_results is True:
outports = graph.outports.keys()
if not outports:
raise FlowError("Cannot capture results: graph has no exported "
"outports")
elif capture_results is False:
outports = []
else:
outports = capture_results
initializations = initializations or {}
if outports or initializations:
# use a wrapper so we don't modify the passed graph
# we could also copy the graph, but seeing as we're working with
# inports and outports, it might be best to operate on the graph as a
# SubGraph to ensure consistent behavior
wrapper = Graph()
apply = wrapper.add_graph('Apply', graph)
for (port_name, content) in initializations.items():
wrapper.initialize(content, apply.port(port_name))
captures = {}
for port_name in outports:
capture_name = 'Capture_{}'.format(port_name)
capture = wrapper.add_component(capture_name, Capture)
wrapper.connect(apply.port(port_name), capture.port('IN'))
captures[port_name] = capture
graph = wrapper
Network(graph).go()
# FIXME: re-raise errors?
if outports:
return {name: capture.value for (name, capture) in captures.items()}
def get_graph():
try:
if command in {'clear', 'addgraph'}:
return payload['id']
else:
return payload['graph']
except KeyError:
raise FlowError('No graph specified')
def _prep_args(self, ports):
# unlike BasePortCollection, we don't flatten arrays
ports = list(flatten_collections(ports))
# Enforce unique names between input and output ports. This ensures
# that _FunctionComponent functions can receive a named argument per
# port
names = [p.name for p in ports]
dupes = [n for n, count in Counter(names).items() if count > 1]
if dupes:
raise FlowError("{}: Duplicate port names: {}".format(
self.component, ', '.join(dupes)))
return ports
def rename_group(self, from_name, to_name):
"""
Rename group
"""
try:
group = self.groups[from_name]
except KeyError:
raise FlowError('group {} not found'.format(from_name))
self.groups[to_name] = group
del self.groups[from_name]
self.rename_group.event.emit(from_name, to_name)
def add_component(self, *args, **initializations):
"""
Instantiate a component and add it to the network.
Parameters
----------
name : str
name of component
comp_type : Type[``rill.engine.component.Component``]
component class to instantiate
Returns
-------
``rill.engine.component.Component``
"""
if len(args) == 1:
arg = args[0]
if isinstance(arg, Component):
comp = arg
name = comp.name
elif isclass(arg) and issubclass(arg, Component):
comp_type = arg
name = comp_type.type_name or comp_type.__name__
name = self._get_unique_name(name)
comp = comp_type(name)
else:
raise ValueError()
elif len(args) == 2:
name, comp_type = args
if not isclass(comp_type) or not issubclass(comp_type, Component):
raise TypeError("comp_type must be a sub-class of Component")
comp = comp_type(name)
else:
raise ValueError()
if name in self._components:
raise FlowError(
"Component {} already exists in network".format(name))
self.put_component(name, comp)
for name, value in initializations.items():
receiver = comp.port(name, kind='in')
if value is None and not receiver.required:
continue
if isinstance(value,
(OutputPort, OutputArray, InputPort, InputArray)):
self.connect(value, receiver)
else:
self.initialize(value, receiver)
return comp
def _new_graph(self, graph_id, description=None, metadata=None,
overwrite=True):
"""
Create a new graph.
"""
if not overwrite and self._graphs.get(graph_id, None):
raise FlowError('Graph already exists')
self.logger.debug('Graph {}: Initializing'.format(graph_id))
self.add_graph(graph_id, Graph(
name=graph_id,
description=description,
metadata=metadata
))
def validate(self):
"""
Validate the graph.
"""
errors = []
for component in self._components.values():
for port in component.ports:
try:
port.validate()
except FlowError as e:
errors.append(str(e))
if errors:
for error in errors:
logger.error(error)
raise FlowError("Errors opening ports")
def change_group(self, name, nodes=None, metadata=None):
"""
Change group
"""
try:
group = self.groups[name]
except KeyError:
raise FlowError('group {} not found'.format(name))
if nodes:
group['nodes'] = nodes
if metadata:
merge_metadata(metadata, group['metadata'])
self.change_group.event.emit(name, nodes, metadata)
def initiate(self):
"""
Go through components opening ports, and activating those which are
self-starting (have no input connections)
"""
self.reset()
self._build_runners()
self._open_ports()
self_starters = [r for r in self.runners if r.self_starting]
if not self_starters:
raise FlowError("No self-starters found")
for runner in self_starters:
runner.activate()
def get_graph(self, graph_id):
"""
Parameters
----------
graph_id : str
unique identifier for the graph to create or get
Returns
-------
graph : ``rill.engine.network.Graph``
the graph object.
"""
try:
return self._graphs[graph_id]
except KeyError:
raise FlowError('Requested graph not found: {}'.format(graph_id))
def handle(self, command, payload):
"""
Deliver a FBP graph message to dispatchers
Parameters
----------
command : str
payload : dict
"""
# FIXME: add (optional?) jsonschema validation of payload
try:
method_name = self.GRAPH_COMMAND_TO_METHOD[command]
except KeyError:
raise FlowError("Unknown command '%s' for protocol '%s'" %
(command, 'graph'))
for dispatcher in self.dispatchers:
method = getattr(dispatcher, method_name)
method(payload)
def component(self, name):
"""
Parameters
----------
name : str
name of component
Returns
-------
``rill.engine.component.Component``
Raises
------
``rill.engine.exceptions.FlowError`` : if no component found
"""
comp = self.get_component(name)
if comp is None:
raise FlowError("Reference to unknown component " + name)
return comp
def create_element(self, index=None):
"""
Create an element port within the array.
Parameters
----------
index : Optional[int]
index of element. If None, the next available index is used
Returns
-------
Union[``rill.engine.inputport.InputPort``, ``rill.engine.outputport.OutputPort``]
"""
if self.fixed_size is not None:
raise FlowError(
"New elements cannot be added to array ports with fixed size "
"after instantiation")
if index is None:
index = self.next_available_index()
comp = self._create_element(index)
self._elements[index] = comp
return comp
def validate_packet_contents(self, packet_content):
"""
Validate packet data.
Parameters
----------
packet_content : Any
Returns
-------
Any
original content, or content conformed based on the ``TypeHandler``
"""
if self.type is not None:
try:
conformed = self.type.validate(packet_content)
except PacketValidationError as err:
# catch and re-raise to provide port name in error message
raise FlowError("{} found invalid type: {}".format(self, err))
else:
return conformed if conformed is not None else packet_content
return packet_content
def get_component_port(self, arg, index=None, kind=None):
"""
Get a port on a component.
Parameters
----------
arg : Union[``rill.engine.outputport.OutputPort``, ``rill.engine.inputport.InputPort``, str]
index : Optional[int]
index of element, if port is an array. If None, the next available
index is used
kind : str
{'in', 'out'}
Returns
-------
Union[``rill.engine.outputport.OutputPort``, ``rill.engine.inputport.InputPort``]
"""
if isinstance(arg, (OutputPort, OutputArray, InputPort, InputArray)):
port = arg
if kind is not None and port.kind != kind:
raise FlowError("Expected {}port: got {}".format(
kind, type(port)))
else:
if isinstance(arg, (tuple, list)):
comp_name, port_name = arg
elif isinstance(arg, basestring):
split = arg.split('.')
comp_name = '.'.join(split[:-1])
port_name = split[-1]
else:
raise TypeError(arg)
comp = self.component(comp_name)
port = comp.port(port_name, kind=kind)
if port.is_array() and index is not False:
port = port.get_element(index, create=True)
return port
def execute(self):
inport = self.internal_port()
if inport is None or self.ports.OUT.is_closed():
return
self.logger.debug("Accessing input port: {}".format(inport))
old_receiver = inport.component
if inport.is_initialized():
raise FlowError("SubinSS cannot support IIP - use Subin")
inport.component = self
level = 0
for p in inport:
p.set_owner(self)
if p.get_type() == Packet.Type.OPEN:
if level > 0:
self.ports.OUT.send(p)
else:
self.drop(p)
self.logger.debug("open bracket detected")
level += 1
elif p.get_type() == Packet.Type.CLOSE:
if level > 1:
# pass on nested brackets
self.ports.OUT.send(p)
level -= 1
else:
self.drop(p)
self.logger.debug("close bracket detected")
break
else:
self.ports.OUT.send(p)
self.logger.debug("Releasing input port: {}".format(inport))
# inport.set_receiver(old_receiver)
inport.component = old_receiver
def add_component(self, *args, **initializations):
"""
Instantiate a component and add it to the network.
Parameters
----------
name : str
name of component
comp_type : Type[``rill.engine.component.Component``]
component class to instantiate
Returns
-------
``rill.engine.component.Component``
"""
if len(args) == 1:
arg = args[0]
if isinstance(arg, Component):
comp = arg
name = comp.name
elif isclass(arg) and issubclass(arg, Component):
comp_type = arg
name = comp_type.type_name or comp_type.__name__
name = self._get_unique_name(name)
comp = comp_type(name)
else:
raise ValueError()
elif len(args) == 2:
name, comp_type = args
if not isclass(comp_type) or not issubclass(comp_type, Component):
raise TypeError("comp_type must be a sub-class of Component")
comp = comp_type(name)
else:
raise ValueError()
if name in self._components:
raise FlowError(
"Component {} already exists in network".format(name))
self.put_component(name, comp)
# FIXME: would this make more sense as part of creating a new port?
# add port defaults to the initializations dict if they're not provided
for name, inport in comp.inport_definitions.iteritems():
if name not in initializations and inport.default != NOT_SET:
initializations[name] = inport.default
for name, value in initializations.items():
receiver = comp.port(name, kind='in')
if value is None and not receiver.required:
continue
if isinstance(value,
(OutputPort, OutputArray, InputPort, InputArray)):
self.connect(value, receiver)
else:
self.initialize(value, receiver)
# forward events from component to graph
comp.port_opened.event.add_listener(self.port_opened)
comp.port_closed.event.add_listener(self.port_closed)
return comp
