def _next_message(self) -> Union[StreamEvent, BinaryMessage]:
line = ""
while not line.strip():
line = self._stream.readline()
if line == "":
return PermanentEndOfStream("End of pipe reached")
try:
deserialized_object: Dict[str, Any] = json.loads(line)
except ValueError as e:
raise EsqueIOHandlerReadException(
"Error parsing JSON object from input. "
f"Make sure json objects are single-line and not pretty printed. Original Error: {e}"
)
key_encoding = deserialized_object.get("keyenc",
self.config.key_encoding)
value_encoding = deserialized_object.get("valueenc",
self.config.value_encoding)
return BinaryMessage(
key=extract(deserialized_object.get("key"), key_encoding),
value=extract(deserialized_object.get("value"), value_encoding),
offset=deserialized_object.get("offset", -1),
partition=deserialized_object.get("partition", -1),
timestamp=datetime.datetime.fromtimestamp(
deserialized_object.get("timestamp", 0),
tz=datetime.timezone.utc),
headers=[
MessageHeader(h["key"], h.get("value"))
for h in deserialized_object.get("headers", [])
],
)
def create_tombstone_message(ping_id) -> BinaryMessage:
create_time = datetime.datetime.fromtimestamp(round(time.time(), 3))
return BinaryMessage(key=ping_id,
value=None,
partition=-1,
offset=-1,
timestamp=create_time,
headers=[])
def mk_binary_message(partition: int, offset: int, ts: int) -> BinaryMessage:
return BinaryMessage(
key=f"k_p{partition}_o{offset}".encode("utf-8"),
value=f"v_p{partition}_o{offset}".encode("utf-8"),
partition=partition,
offset=offset,
timestamp=datetime.datetime.fromtimestamp(ts,
tz=datetime.timezone.utc),
headers=[],
)
def _confluent_to_binary_message(
self, consumed_message: Message) -> BinaryMessage:
binary_message = BinaryMessage(
key=consumed_message.key(),
value=consumed_message.value(),
partition=consumed_message.partition(),
offset=consumed_message.offset(),
timestamp=self._confluent_to_io_timestamp(consumed_message),
headers=self._confluent_to_io_headers(consumed_message.headers()),
)
return binary_message
def serialize(
self,
message: Union[Message,
StreamEvent]) -> Union[BinaryMessage, StreamEvent]:
if isinstance(message, StreamEvent):
return message
key_data = self._key_serializer.serialize(message.key)
value_data = self._value_serializer.serialize(message.value)
return BinaryMessage(
key=key_data,
value=value_data,
offset=message.offset,
partition=message.partition,
timestamp=message.timestamp,
headers=message.headers.copy(),
)
def binary_messages() -> List[BinaryMessage]:
return [
BinaryMessage(
key=b"foo1",
value=b"bar1",
partition=0,
offset=0,
timestamp=datetime.datetime(year=2021,
month=1,
day=1,
hour=0,
minute=0,
tzinfo=datetime.timezone.utc),
headers=[MessageHeader("a", "b")],
),
BinaryMessage(
key=b"foo2",
value=b"bar2",
partition=0,
offset=1,
timestamp=datetime.datetime(year=2021,
month=1,
day=1,
hour=0,
minute=1,
tzinfo=datetime.timezone.utc),
headers=[MessageHeader("c", None)],
),
BinaryMessage(
key=b"foo3",
value=b"bar3",
partition=1,
offset=0,
timestamp=datetime.datetime(year=2021,
month=1,
day=1,
hour=0,
minute=2,
tzinfo=datetime.timezone.utc),
headers=[],
),
BinaryMessage(
key=b"foo4",
value=b"bar4",
partition=1,
offset=1,
timestamp=datetime.datetime(year=2021,
month=1,
day=1,
hour=0,
minute=3,
tzinfo=datetime.timezone.utc),
headers=[],
),
BinaryMessage(
key=b"foo5",
value=b"bar5",
partition=1,
offset=2,
timestamp=datetime.datetime(year=2021,
month=1,
day=1,
hour=0,
minute=4,
tzinfo=datetime.timezone.utc),
headers=[],
),
BinaryMessage(
key=b"foo6",
value=b"bar6",
partition=1,
offset=3,
timestamp=datetime.datetime(year=2021,
month=1,
day=1,
hour=0,
minute=5,
tzinfo=datetime.timezone.utc),
headers=[],
),
]