def make_pipeline_nested(pipeline_type):
"""Make a nested pipeline."""
pipeline = pipeline_type(make_pipeline_singular(pipeline_type),
make_pipeline_events(pipeline_type), EventLink(),
AutomaticPipe(EventLink(), EventLink()),
make_pipeline_events(pipeline_type))
return pipeline
def make_pipeline_delayed(pipeline_type):
"""Make a short pipeline."""
pipeline = pipeline_type(StreamLink(), ChunkedStreamLink(),
DelayedEventLink(), EventLink(), EventLink())
assert isinstance(pipeline.bottom, StreamLink)
assert isinstance(pipeline.top, EventLink)
return pipeline
def make_pipeline_events(pipeline_type):
"""Make a events-only pipeline."""
pipeline = pipeline_type(EventLink(), EventLink(), EventLink(),
EventLink())
assert isinstance(pipeline.bottom, EventLink)
assert isinstance(pipeline.top, EventLink)
return pipeline
def test_manual_pipe_composition():
"""Exercise ManualPipe nested composition."""
print('Testing Nesting of Piped Event Links with Manual Synchronization:')
chunked_stream_link = ChunkedStreamLink()
upper_event_link = EventLink()
outer_pipe = ManualPipe(ManualPipe(chunked_stream_link, EventLink()),
ManualPipe(EventLink(), upper_event_link))
# Read/write on links
write_bottom_chunked_buffers(chunked_stream_link)
outer_pipe.sync()
assert_bottom_events(upper_event_link)
write_top_events(upper_event_link)
outer_pipe.sync()
result = chunked_stream_link.to_write()
print('Chunked Stream Link wrote to stream: {}'.format(result))
assert result == HIGHER_CHUNKED_STREAM
# Read/write on pipe
write_bottom_chunked_buffers(outer_pipe)
outer_pipe.sync()
assert_bottom_events(outer_pipe)
write_top_events(outer_pipe)
outer_pipe.sync()
result = outer_pipe.to_write()
print('Chunked Stream Link wrote to stream: {}'.format(result))
assert result == HIGHER_CHUNKED_STREAM
def make_pipeline_long(pipeline_type):
"""Make a long pipeline."""
pipeline = pipeline_type(StreamLink(), StreamLink(), StreamLink(),
ChunkedStreamLink(), EventLink(), EventLink(),
EventLink())
assert isinstance(pipeline.bottom, StreamLink)
assert isinstance(pipeline.top, EventLink)
return pipeline
def make_pipeline_loopback(pipeline_factory):
"""Make a long pipeline with a loopback at the top."""
manual_pipeline = pipeline_factory(StreamLink(), StreamLink(),
StreamLink(), ChunkedStreamLink(),
EventLink(), EventLink(), EventLink(),
TopLoopbackLink())
assert isinstance(manual_pipeline.bottom, StreamLink)
assert isinstance(manual_pipeline.top, TopLoopbackLink)
return manual_pipeline
def __init__(self,
name=None,
chunk_separator=b'\0',
begin_chunk_separator=True):
"""Initialize processors."""
self._event_link = EventLink(
sender_processor=self.sender_processor,
sender_processor_args=(chunk_separator, begin_chunk_separator),
receiver_event_passthrough=True)
self._event_link.after_receive = self.__after_receive
self._stream_link = StreamLink(
reader_processor=self.reader_processor,
reader_processor_args=(chunk_separator, ))
self._stream_link.after_write = self.__after_write
self.name = name
def test_event_link():
"""Exercise EventLink's interface."""
print('Testing Event Link:')
event_link = EventLink()
for event in LOWER_EVENTS:
event_link.to_receive(event)
assert event_link.has_receive()
for (i, event) in enumerate(event_link.receive_all()):
print('Event Link received from queue: {}'.format(event))
assert event.data == LOWER_EVENTS[i]
for event in HIGHER_EVENTS:
event_link.send(event)
assert event_link.has_to_send()
for (i, event) in enumerate(event_link.to_send_all()):
print('Event Link sent to queue: {}'.format(event))
assert event.data == HIGHER_EVENTS[i]
def test_manual_pipe():
"""Exercise ManualPipe's interface."""
print('Testing Piped Event Links with Manual Synchronization:')
chunked_stream_link = ChunkedStreamLink()
event_link = EventLink()
pipe = ManualPipe(chunked_stream_link, event_link)
# Read/write on links
write_bottom_chunked_buffers(chunked_stream_link)
pipe.sync()
assert_bottom_events(event_link)
write_top_events(event_link)
pipe.sync()
result = chunked_stream_link.to_write()
print('Chunked Stream Link wrote to stream: {}'.format(result))
assert result == HIGHER_CHUNKED_STREAM
# Read/write on pipe
write_bottom_chunked_buffers(pipe)
pipe.sync()
assert_bottom_events(pipe)
write_top_events(pipe)
pipe.sync()
result = pipe.to_write()
print('Chunked Stream Link wrote to stream: {}'.format(result))
assert result == HIGHER_CHUNKED_STREAM
def make_pipeline_short(pipeline_type):
"""Make a short pipeline."""
pipeline = pipeline_type(
ChunkedStreamLink(),
EventLink(),
)
assert isinstance(pipeline.bottom, ChunkedStreamLink)
assert isinstance(pipeline.top, EventLink)
return pipeline
class ChunkedStreamLink(StreamLinkBelow, EventLinkAbove, DataEventLink):
"""Processor which sends and receives delimited chunks of data over a stream.
Adapts byte buffer-based stream processors to event queue processors by
discretizing the stream into chunks delimited by a chunk separator in the stream.
When begin_chunk_separator is enabled, each chunk will be delimited by the
chunk separator at both the start and end of the chunk; otherwise, it will
only be delimited by a chunk separator at the end of the chunk.
Omits empty chunks, which will be neither sent nor received.
Interface:
Above: sends and receives bytestrings of chunks.
Below: to_send and to_receive delimited bytestrings (not following chunk boundaries).
"""
def __init__(self,
name=None,
chunk_separator=b'\0',
begin_chunk_separator=True):
"""Initialize processors."""
self._event_link = EventLink(
sender_processor=self.sender_processor,
sender_processor_args=(chunk_separator, begin_chunk_separator),
receiver_event_passthrough=True)
self._event_link.after_receive = self.__after_receive
self._stream_link = StreamLink(
reader_processor=self.reader_processor,
reader_processor_args=(chunk_separator, ))
self._stream_link.after_write = self.__after_write
self.name = name
def __repr__(self):
"""Return a string representation of the link."""
if self.name is not None:
return '⇌~ {}({}) □⇌'.format(self.__class__.__qualname__,
self.name)
else:
return '⇌~ {} □⇌'.format(self.__class__.__qualname__)
# Implement EventLinkAbove
def receive(self):
"""Implement EventLinkAbove.receive."""
return self._event_link.receive()
def has_receive(self):
"""Implement EventLinkAbove.has_receive."""
return self._event_link.has_receive()
def send(self, chunk):
"""Implement EventLinkAbove.send."""
self._event_link.send(chunk)
# Implement StreamLinkBelow
def to_read(self, bytes_data):
"""Implement StreamLinkBelow.to_read."""
self._stream_link.to_read(bytes_data)
def to_write(self):
"""Implement StreamLinkBelow.to_write."""
return self._stream_link.to_write()
# Utilities for writing receive and send processors
def __after_receive(self, event):
yield from self.after_receive(event)
def __after_write(self, event):
yield from self.after_write(event)
def after_receive(self, event):
"""Enqueue the event for the layer above for consumption by that layer.
This gets overridden to change the receiver's behavior to send the event.
Note that this gets monkey-patched by other links and link utilities
which combine links, such as ChunkedStreamLink and AutomaticPipe!
"""
# print('Event link after_receive: {}'.format(event))
yield from send(event)
def after_write(self, buffer):
"""Enqueue the buffer for the layer below for consumption by that layer.
This gets overridden to change the receiver's behavior to send the event.
Note that this gets monkey-patched by other links and link utilities
which combine links, such as ChunkedStreamLink and AutomaticPipe!
"""
# print('Stream link after_write: {}'.format(buffer))
yield from write(buffer)
# Receive and send processors
@stream_processor
def reader_processor(self, chunk_separator):
"""Stream reader processor."""
while True:
buffer = yield from read_until(chunk_separator)
if len(buffer) == 0:
continue
# print(hex_bytes(buffer))
data_event = self.make_link_data(buffer, 'up', None)
self._event_link.to_receive(data_event)
@event_processor
def sender_processor(self, chunk_separator, begin_chunk_separator):
"""Event sender processor."""
while True:
event = yield from receive()
data_event = self.get_link_data(event, 'down')
event = data_event.data
if len(event) == 0:
continue
stream_contents = event + chunk_separator
if begin_chunk_separator:
stream_contents = chunk_separator + stream_contents
self._stream_link.write(stream_contents)
def __init__(self, name=None):
"""Initialize members."""
self.name = name
self._event_link = EventLink(
receiver_processor=self.receiver_processor)
self._stream_link = StreamLink(reader_processor=self.reader_processor)
class TopLoopbackLink(GenericLinkBelow, DataEventLink):
"""A data sink to echo events and stream buffers received from below to send back down.
Note that the events received by to_receive are only echoed back on to_send,
and buffers read by to_read are only echoed back on to_write.
"""
def __init__(self, name=None):
"""Initialize members."""
self.name = name
self._event_link = EventLink(
receiver_processor=self.receiver_processor)
self._stream_link = StreamLink(reader_processor=self.reader_processor)
def __repr__(self):
"""Return a string representation of the link."""
if self.name is not None:
return '⇌* {}({})'.format(self.__class__.__qualname__, self.name)
else:
return '⇌* {}'.format(self.__class__.__qualname__)
# Implement EventLinkBelow
def to_receive(self, event):
"""Implement EventLinkBelow."""
self._event_link.to_receive(event)
def to_send(self):
"""Implement EventLinkBelow."""
return self._event_link.to_send()
def has_to_send(self):
"""Implement EventLinkBelow."""
return self._event_link.has_to_send()
# Implement StreamLinkBelow
def to_read(self, bytes_data):
"""Implement StreamLinkBelow."""
self._stream_link.to_read(bytes_data)
def to_write(self):
"""Implement StreamLinkBelow."""
return self._stream_link.to_write()
# Read and write processors
def after_send(self, event):
"""Do nothing."""
yield from wait()
def after_write(self, buffer):
"""Do nothing."""
yield from wait()
@event_processor
def receiver_processor(self):
"""Event receiver echo processor."""
while True:
event = yield from receive()
if isinstance(event, LinkException):
print('Ignoring exception: {}'.format(event))
continue
data_event = self.get_link_data(event, 'loopback')
# print('Echoing: {}'.format(data_event))
data_event = self.make_link_data(data_event.data, 'loopback',
event)
self._event_link._sender.send(data_event)
yield from self.after_send(
event) # for correctness in automatic pipeline
@stream_processor
def reader_processor(self):
"""Stream reader echo processor."""
while True:
buffer = yield from read()
# print('Echoing: {}'.format(buffer))
self._stream_link._writer.send(buffer)
yield from self.after_write(
buffer) # for correctness in automatic pipeline
class BottomLoopbackLink(GenericLinkAbove, DataEventLink):
"""A data sink to echo events and stream buffers sent from above to receive back up.
Note that the events sent by send are only echoed back on receive,
and buffers read by write are only echoed back on read.
"""
def __init__(self, name=None):
"""Initialize members."""
self.name = name
self._event_link = EventLink(sender_processor=self.sender_processor)
self._stream_link = StreamLink(writer_processor=self.writer_processor)
def __repr__(self):
"""Return a string representation of the link."""
if self.name is not None:
return '{}({}) *⇌'.format(self.__class__.__qualname__, self.name)
else:
return '{} *⇌'.format(self.__class__.__qualname__)
# Implement EventLinkAbove
def receive(self):
"""Implement EventLinkAbove.receive."""
return self._event_link.receive()
def has_receive(self):
"""Implement EventLinkAbove.has_receive."""
return self._event_link.has_receive()
def send(self, event):
"""Implement EventLinkAbove.send."""
self._event_link.send(event)
# Implement StreamLinkAbove
def read(self):
"""Implement StreamLinkAbove.read."""
return self._stream_link.read()
def write(self, bytes_data):
"""Implement StreamLinkAbove.write."""
self._stream_link.write(bytes_data)
# Read and write processors
def after_receive(self, event):
"""Do nothing."""
yield from wait()
def after_read(self, buffer):
"""Do nothing."""
yield from wait()
@event_processor
def sender_processor(self):
"""Event sender echo processor."""
while True:
event = yield from receive()
data_event = self.get_link_data(event, 'loopback')
data_event = self.make_link_data(data_event.data, 'loopback',
event)
self._event_link._receiver.send(data_event)
yield from self.after_receive(
data_event) # for correctness in automatic pipeline
@stream_processor
def writer_processor(self):
"""Stream writer echo processor."""
while True:
buffer = yield from read()
self._stream_link._reader.send(buffer)
yield from self.after_read(
buffer) # needed for correctness in automatic pipeline
