def initialize(self, static_value): """ Initialize a port to a static value. Parameters ---------- static_value : Any """ if self.is_null(): raise FlowError("Cannot initialize null port: {}".format(self)) if self.is_connected(): if not self.is_initialized(): raise FlowError( "Port cannot have both an initial packet and a " "connection: {}".format(self)) else: raise FlowError("Port is already initialized: {}".format(self)) # a Stream instance indicates that each item in the stream should be # yielded as a separate packet. This is used to differentiate between # a port which may expect a single packet to contain a list of data. if isinstance(static_value, Stream): if self.auto_receive: raise FlowError("Static port initialized with a stream of " "data: {}".format(self)) content = [self.validate_packet_contents(p) for p in static_value] else: content = [self.validate_packet_contents(static_value)] self._connection = InitializationConnection(content, self)
def recv_message(self, msg): """ Handle a FBP graph message Parameters ---------- msg: rill.runtime.plumbing.Message """ if msg.protocol == 'graph' and msg.command == 'clear': payload = msg.payload.copy() graph_id = payload.pop('id') self._new_graph(graph_id, **payload) elif msg.protocol == 'graph' and msg.command == 'addgraph': payload = msg.payload.copy() graph_id = payload.pop('id') self._new_graph(graph_id, overwrite=False, **payload) else: # automatic handling try: method_name = self.COMMAND_TO_METHOD[msg.protocol][msg.command] except KeyError: raise FlowError("Unknown command '%s' for protocol '%s'" % (msg.command, msg.protocol)) method = getattr(self, method_name) method(**_toargs(msg.payload))
def resume(self): """ Resume a graph that has been suspended. """ self_starters = [] for runner in self.runners: for port in runner.component.inports: if port.is_connected() and not port.is_initialized() and \ port._connection._queue: logger.info("Existing data in connection buffer: {}", args=[port]) if runner.status in (StatusValues.TERMINATED, StatusValues.ERROR): runner.kill() continue elif runner.self_starting or \ runner.status in (StatusValues.SUSP_RECV, StatusValues.SUSP_SEND, StatusValues.DORMANT, StatusValues.ACTIVE): self_starters.append(runner) if not self_starters: raise FlowError("No self-starters found") for runner in self_starters: runner.activate()
def connect(self, inport, outport, capacity, topics=['all']): """ Connect an ``InputPort`` to an ``OutputPort``. Parameters ---------- inport : ``rill.engine.inputport.InputPort`` outport : ``rill.engine.outputport.OutputPort`` capacity : int size of the buffer """ if self.outports: if capacity != self.capacity(): # a previous connect specified a destination port with the same # name and index. raise FlowError( "{}: Connection capacity does not agree with previous " "specification".format(self)) else: self._queue = deque(maxlen=capacity) self.topics=topics self.inport = inport self.outports.add(outport) outport._connections.append(self) for topic in topics: if not topic in outport._topic_map.keys(): outport._topic_map[topic]=[] outport._topic_map[topic].append(self)
def validate(self): self._check_port_types() if not self.ports() and self.required: raise FlowError("Required {} {} has no members".format( self.port_class.__name__, self)) if self.fixed_size is not None and self.required: missing = [] for port in self.ports(): if not port.is_connected(): missing.append(str(port)) if missing: raise FlowError( "Required {} {} has missing elements: {}".format( self.port_class.__name__, self, ', '.join(missing)))
def validate(self): """ Runs prior to opening a port to validate that the port can be opened. """ if not self.is_connected() and self.required: raise FlowError( "{} port is required, but not connected".format(self))
def uninitialize(self): """ Remove static value initialization from the port. Returns ------- ``InitializationConnection`` The removed initialization connection """ if self.is_null(): raise FlowError("Cannot uninitialize null port: {}".format(self)) if not self.is_initialized(): raise FlowError("Port is not initialized: {}".format(self)) conn = self._connection self._connection = None return conn
def port(self, port_name, kind=None): """ Get a port on the component. Handles names with array indices (e.g. 'IN[0]') Parameters ---------- port_name : str kind : {'in', 'out'} or None assert that the retrieved port is of the given type Returns ------- port : ``rill.engine.outputport.OutputPort`` or ``rill.engine.outputport.OutputArray`` or ``rill.engine.inputport.InputPort`` or ``rill.engine.inputport.InputArray`` """ reg = re.compile(PORT_NAME_REG) m = reg.match(port_name) if not m: raise FlowError("Invalid port name: " + port_name) port_name, index = m.groups() if index is not None: index = int(index) try: port = self.ports[port_name] except KeyError as err: raise FlowError(str(err)) if kind is not None and port.kind != kind: raise FlowError("Expected {} port: got {}".format( kind, type(port))) if index is not None: if not port.is_array(): raise FlowError("Element {} specified for non-array " "port {}".format(index, port)) port = port.get_element(index, create=True) return port
def run_graph(graph, initializations=None, capture_results=False): """ Run a graph. Parameters ---------- graph : ``rill.engine.network.Graph`` initializations : Optional[Dict[str, Any]] map of exported inport names to initial content capture_results : Union[bool, List[str]] list of exported outport names whose results should be collected. True for all, False for none. Returns ------- Dict[str, Any] map network outport names to captured values """ from rill.components.basic import Capture if capture_results is True: outports = graph.outports.keys() if not outports: raise FlowError("Cannot capture results: graph has no exported " "outports") elif capture_results is False: outports = [] else: outports = capture_results initializations = initializations or {} if outports or initializations: # use a wrapper so we don't modify the passed graph # we could also copy the graph, but seeing as we're working with # inports and outports, it might be best to operate on the graph as a # SubGraph to ensure consistent behavior wrapper = Graph() apply = wrapper.add_graph('Apply', graph) for (port_name, content) in initializations.items(): wrapper.initialize(content, apply.port(port_name)) captures = {} for port_name in outports: capture_name = 'Capture_{}'.format(port_name) capture = wrapper.add_component(capture_name, Capture) wrapper.connect(apply.port(port_name), capture.port('IN')) captures[port_name] = capture graph = wrapper Network(graph).go() # FIXME: re-raise errors? if outports: return {name: capture.value for (name, capture) in captures.items()}
def get_graph(): try: if command in {'clear', 'addgraph'}: return payload['id'] else: return payload['graph'] except KeyError: raise FlowError('No graph specified')
def _prep_args(self, ports): # unlike BasePortCollection, we don't flatten arrays ports = list(flatten_collections(ports)) # Enforce unique names between input and output ports. This ensures # that _FunctionComponent functions can receive a named argument per # port names = [p.name for p in ports] dupes = [n for n, count in Counter(names).items() if count > 1] if dupes: raise FlowError("{}: Duplicate port names: {}".format( self.component, ', '.join(dupes))) return ports
def rename_group(self, from_name, to_name): """ Rename group """ try: group = self.groups[from_name] except KeyError: raise FlowError('group {} not found'.format(from_name)) self.groups[to_name] = group del self.groups[from_name] self.rename_group.event.emit(from_name, to_name)
def add_component(self, *args, **initializations): """ Instantiate a component and add it to the network. Parameters ---------- name : str name of component comp_type : Type[``rill.engine.component.Component``] component class to instantiate Returns ------- ``rill.engine.component.Component`` """ if len(args) == 1: arg = args[0] if isinstance(arg, Component): comp = arg name = comp.name elif isclass(arg) and issubclass(arg, Component): comp_type = arg name = comp_type.type_name or comp_type.__name__ name = self._get_unique_name(name) comp = comp_type(name) else: raise ValueError() elif len(args) == 2: name, comp_type = args if not isclass(comp_type) or not issubclass(comp_type, Component): raise TypeError("comp_type must be a sub-class of Component") comp = comp_type(name) else: raise ValueError() if name in self._components: raise FlowError( "Component {} already exists in network".format(name)) self.put_component(name, comp) for name, value in initializations.items(): receiver = comp.port(name, kind='in') if value is None and not receiver.required: continue if isinstance(value, (OutputPort, OutputArray, InputPort, InputArray)): self.connect(value, receiver) else: self.initialize(value, receiver) return comp
def _new_graph(self, graph_id, description=None, metadata=None, overwrite=True): """ Create a new graph. """ if not overwrite and self._graphs.get(graph_id, None): raise FlowError('Graph already exists') self.logger.debug('Graph {}: Initializing'.format(graph_id)) self.add_graph(graph_id, Graph( name=graph_id, description=description, metadata=metadata ))
def validate(self): """ Validate the graph. """ errors = [] for component in self._components.values(): for port in component.ports: try: port.validate() except FlowError as e: errors.append(str(e)) if errors: for error in errors: logger.error(error) raise FlowError("Errors opening ports")
def change_group(self, name, nodes=None, metadata=None): """ Change group """ try: group = self.groups[name] except KeyError: raise FlowError('group {} not found'.format(name)) if nodes: group['nodes'] = nodes if metadata: merge_metadata(metadata, group['metadata']) self.change_group.event.emit(name, nodes, metadata)
def initiate(self): """ Go through components opening ports, and activating those which are self-starting (have no input connections) """ self.reset() self._build_runners() self._open_ports() self_starters = [r for r in self.runners if r.self_starting] if not self_starters: raise FlowError("No self-starters found") for runner in self_starters: runner.activate()
def get_graph(self, graph_id): """ Parameters ---------- graph_id : str unique identifier for the graph to create or get Returns ------- graph : ``rill.engine.network.Graph`` the graph object. """ try: return self._graphs[graph_id] except KeyError: raise FlowError('Requested graph not found: {}'.format(graph_id))
def handle(self, command, payload): """ Deliver a FBP graph message to dispatchers Parameters ---------- command : str payload : dict """ # FIXME: add (optional?) jsonschema validation of payload try: method_name = self.GRAPH_COMMAND_TO_METHOD[command] except KeyError: raise FlowError("Unknown command '%s' for protocol '%s'" % (command, 'graph')) for dispatcher in self.dispatchers: method = getattr(dispatcher, method_name) method(payload)
def component(self, name): """ Parameters ---------- name : str name of component Returns ------- ``rill.engine.component.Component`` Raises ------ ``rill.engine.exceptions.FlowError`` : if no component found """ comp = self.get_component(name) if comp is None: raise FlowError("Reference to unknown component " + name) return comp
def create_element(self, index=None): """ Create an element port within the array. Parameters ---------- index : Optional[int] index of element. If None, the next available index is used Returns ------- Union[``rill.engine.inputport.InputPort``, ``rill.engine.outputport.OutputPort``] """ if self.fixed_size is not None: raise FlowError( "New elements cannot be added to array ports with fixed size " "after instantiation") if index is None: index = self.next_available_index() comp = self._create_element(index) self._elements[index] = comp return comp
def validate_packet_contents(self, packet_content): """ Validate packet data. Parameters ---------- packet_content : Any Returns ------- Any original content, or content conformed based on the ``TypeHandler`` """ if self.type is not None: try: conformed = self.type.validate(packet_content) except PacketValidationError as err: # catch and re-raise to provide port name in error message raise FlowError("{} found invalid type: {}".format(self, err)) else: return conformed if conformed is not None else packet_content return packet_content
def get_component_port(self, arg, index=None, kind=None): """ Get a port on a component. Parameters ---------- arg : Union[``rill.engine.outputport.OutputPort``, ``rill.engine.inputport.InputPort``, str] index : Optional[int] index of element, if port is an array. If None, the next available index is used kind : str {'in', 'out'} Returns ------- Union[``rill.engine.outputport.OutputPort``, ``rill.engine.inputport.InputPort``] """ if isinstance(arg, (OutputPort, OutputArray, InputPort, InputArray)): port = arg if kind is not None and port.kind != kind: raise FlowError("Expected {}port: got {}".format( kind, type(port))) else: if isinstance(arg, (tuple, list)): comp_name, port_name = arg elif isinstance(arg, basestring): split = arg.split('.') comp_name = '.'.join(split[:-1]) port_name = split[-1] else: raise TypeError(arg) comp = self.component(comp_name) port = comp.port(port_name, kind=kind) if port.is_array() and index is not False: port = port.get_element(index, create=True) return port
def execute(self): inport = self.internal_port() if inport is None or self.ports.OUT.is_closed(): return self.logger.debug("Accessing input port: {}".format(inport)) old_receiver = inport.component if inport.is_initialized(): raise FlowError("SubinSS cannot support IIP - use Subin") inport.component = self level = 0 for p in inport: p.set_owner(self) if p.get_type() == Packet.Type.OPEN: if level > 0: self.ports.OUT.send(p) else: self.drop(p) self.logger.debug("open bracket detected") level += 1 elif p.get_type() == Packet.Type.CLOSE: if level > 1: # pass on nested brackets self.ports.OUT.send(p) level -= 1 else: self.drop(p) self.logger.debug("close bracket detected") break else: self.ports.OUT.send(p) self.logger.debug("Releasing input port: {}".format(inport)) # inport.set_receiver(old_receiver) inport.component = old_receiver
def add_component(self, *args, **initializations): """ Instantiate a component and add it to the network. Parameters ---------- name : str name of component comp_type : Type[``rill.engine.component.Component``] component class to instantiate Returns ------- ``rill.engine.component.Component`` """ if len(args) == 1: arg = args[0] if isinstance(arg, Component): comp = arg name = comp.name elif isclass(arg) and issubclass(arg, Component): comp_type = arg name = comp_type.type_name or comp_type.__name__ name = self._get_unique_name(name) comp = comp_type(name) else: raise ValueError() elif len(args) == 2: name, comp_type = args if not isclass(comp_type) or not issubclass(comp_type, Component): raise TypeError("comp_type must be a sub-class of Component") comp = comp_type(name) else: raise ValueError() if name in self._components: raise FlowError( "Component {} already exists in network".format(name)) self.put_component(name, comp) # FIXME: would this make more sense as part of creating a new port? # add port defaults to the initializations dict if they're not provided for name, inport in comp.inport_definitions.iteritems(): if name not in initializations and inport.default != NOT_SET: initializations[name] = inport.default for name, value in initializations.items(): receiver = comp.port(name, kind='in') if value is None and not receiver.required: continue if isinstance(value, (OutputPort, OutputArray, InputPort, InputArray)): self.connect(value, receiver) else: self.initialize(value, receiver) # forward events from component to graph comp.port_opened.event.add_listener(self.port_opened) comp.port_closed.event.add_listener(self.port_closed) return comp