def __init__(self,
_py_read_stream: PyReadStream = None,
_name: Union[str, None] = None,
_id: Union[str, None] = None):
logger.debug(
"Initializing ReadStream with the name: {}, and ID: {}.".format(
_name, _id))
self._py_read_stream = PyReadStream(
) if _py_read_stream is None else _py_read_stream
self._name = _name
self._id = _id
def __init__(self, _py_read_stream: PyReadStream):
logger.debug(
"Initializing ReadStream with the name: {}, and ID: {}.".format(
_py_read_stream.name(), _py_read_stream.id))
self._py_read_stream = _py_read_stream
def __init__(self, _py_read_stream=None):
self._py_read_stream = PyReadStream(
) if _py_read_stream is None else _py_read_stream
class ReadStream(object):
"""Reads data and invokes callbacks when messages are received.
Handles deserialization of messages, and wraps an `internal.PyReadStream`.
Currently, no callbacks are invoked while `Operator.run` is executing.
`_py_read_stream` is set during erdos.run(), and should never be set
manually.
"""
def __init__(self, _py_read_stream=None):
self._py_read_stream = PyReadStream(
) if _py_read_stream is None else _py_read_stream
def is_closed(self):
"""Whether a top watermark message has been received."""
return self._py_read_stream.is_closed()
def read(self):
"""Blocks until a message is read from the stream."""
return _parse_message(self._py_read_stream.read())
def try_read(self):
"""Tries to read a mesage from the stream.
Returns None if no messages are available at the moment.
"""
internal_msg = self._py_read_stream.try_read()
if internal_msg is None:
return None
return _parse_message(internal_msg)
def add_callback(self, callback, write_streams=None):
"""Adds a callback to the stream.
Args:
callback (Message, list of WriteStream -> None): callback that
takes a message.
write_streams (list of WriteStream): write streams passed to the
callback.
"""
if write_streams is None:
write_streams = []
def internal_callback(serialized):
msg = pickle.loads(serialized)
callback(msg, *write_streams)
self._py_read_stream.add_callback(internal_callback)
def add_watermark_callback(self, callback, write_streams=None):
"""Adds a watermark callback to the stream.
Args:
callback (Message, list of WriteStream -> None): callback that
takes a message.
write_streams (list of WriteStream): write streams passed to the
callback.
"""
if write_streams is None:
write_streams = []
def internal_watermark_callback(coordinates, is_top):
timestamp = Timestamp(coordinates=coordinates, is_top=is_top)
callback(timestamp, *write_streams)
self._py_read_stream.add_watermark_callback(
internal_watermark_callback)
class ReadStream(object):
""" A :py:class:`ReadStream` allows an :py:class:`Operator` to read and
do work on data sent by other operators on a corresponding
:py:class:`WriteStream`.
An :py:class:`Operator` can interface with a :py:class:`ReadStream` by
registering callbacks on the data or watermark received on the stream by
invoking the :py:func:`add_callback` or :py:func:`add_watermark_callback`
method respectively.
An :py:class:`Operator` that takes control of its execution using the
:py:func:`Operator.run` method can retrieve the messages on a
:py:class:`ReadStream` using the :py:func:`ReadStream.read` or
:py:func:`ReadStream.try_read` methods.
Note:
No callbacks are invoked if an operator takes control of the execution
in :py:func:`Operator.run`.
"""
def __init__(self,
_py_read_stream: PyReadStream = None,
_name: Union[str, None] = None,
_id: Union[str, None] = None):
logger.debug(
"Initializing ReadStream with the name: {}, and ID: {}.".format(
_name, _id))
self._py_read_stream = PyReadStream(
) if _py_read_stream is None else _py_read_stream
self._name = _name
self._id = _id
@property
def name(self) -> Union[str, None]:
""" The name of the stream. `None` if no name was given. """
return self._name
def is_closed(self) -> bool:
"""Whether a top watermark message has been received."""
return self._py_read_stream.is_closed()
def read(self) -> Message:
"""Blocks until a message is read from the stream."""
return _parse_message(self._py_read_stream.read())
def try_read(self) -> Union[Message, None]:
"""Tries to read a mesage from the stream.
Returns None if no messages are available at the moment.
"""
internal_msg = self._py_read_stream.try_read()
if internal_msg is None:
return None
return _parse_message(internal_msg)
def add_callback(self, callback: Callable, write_streams=None):
"""Adds a callback to the stream.
Args:
callback: A callback that takes a message and a sequence of
:py:class:`WriteStream` s.
write_streams: Write streams passed to the callback.
"""
if write_streams is None:
write_streams = []
cb_name = callback.__name__ if "__name__" in dir(callback) else "None"
logger.debug("Adding callback {name} to the input stream {_input}, "
"and passing the output streams: {_output}".format(
name=cb_name,
_input=self._name,
_output=list(map(attrgetter("_name"),
write_streams))))
def internal_callback(serialized):
msg = pickle.loads(serialized)
callback(msg, *write_streams)
self._py_read_stream.add_callback(internal_callback)
def add_watermark_callback(self, callback: Callable, write_streams=None):
"""Adds a watermark callback to the stream.
Args:
callback: A callback that takes a message and a sequence of
:py:class:`WriteStream` s.
write_streams: Write streams passed to the callback.
"""
if write_streams is None:
write_streams = []
cb_name = callback.__name__ if "__name__" in dir(callback) else "None"
logger.debug(
"Adding watermark callback {name} to the input stream "
"{_input}, and passing the output streams: {_output}".format(
name=cb_name,
_input=self._name,
_output=list(map(attrgetter("_name"), write_streams))))
def internal_watermark_callback(coordinates, is_top):
timestamp = Timestamp(coordinates=coordinates, is_top=is_top)
callback(timestamp, *write_streams)
self._py_read_stream.add_watermark_callback(
internal_watermark_callback)