def test_single_reader(self):
"""Tests that we expect to see all the correctly emitted TestStreamPayloads.
"""
in_memory_reader = InMemoryReader()
in_memory_reader.add_element(
element=0,
event_time=0,
processing_time=0)
in_memory_reader.add_element(
element=1,
event_time=1,
processing_time=1)
in_memory_reader.add_element(
element=2,
event_time=2,
processing_time=2)
cache = StreamingCache([in_memory_reader])
reader = cache.reader()
coder = coders.FastPrimitivesCoder()
events = all_events(reader)
expected = [
TestStreamPayload.Event(
element_event=TestStreamPayload.Event.AddElements(
elements=[TestStreamPayload.TimestampedElement(
encoded_element=coder.encode(0),
timestamp=0)])),
TestStreamPayload.Event(
processing_time_event=TestStreamPayload.Event.AdvanceProcessingTime(
advance_duration=1 * 10**6)),
TestStreamPayload.Event(
element_event=TestStreamPayload.Event.AddElements(
elements=[TestStreamPayload.TimestampedElement(
encoded_element=coder.encode(1),
timestamp=1 * 10**6)])),
TestStreamPayload.Event(
processing_time_event=TestStreamPayload.Event.AdvanceProcessingTime(
advance_duration=1 * 10**6)),
TestStreamPayload.Event(
element_event=TestStreamPayload.Event.AddElements(
elements=[TestStreamPayload.TimestampedElement(
encoded_element=coder.encode(2),
timestamp=2 * 10**6)])),
]
self.assertSequenceEqual(events, expected)
def test_multiple_readers(self):
"""Tests that the service advances the clock with multiple outputs."""
letters = InMemoryReader('letters')
letters.advance_watermark(0, 1)
letters.add_element(
element='a',
event_time=0,
processing_time=1)
letters.advance_watermark(10, 11)
letters.add_element(
element='b',
event_time=10,
processing_time=11)
numbers = InMemoryReader('numbers')
numbers.add_element(
element=1,
event_time=0,
processing_time=2)
numbers.add_element(
element=2,
event_time=0,
processing_time=3)
numbers.add_element(
element=2,
event_time=0,
processing_time=4)
late = InMemoryReader('late')
late.add_element(
element='late',
event_time=0,
processing_time=101)
cache = StreamingCache([letters, numbers, late])
reader = cache.reader()
coder = coders.FastPrimitivesCoder()
events = all_events(reader)
expected = [
# Advances clock from 0 to 1
TestStreamPayload.Event(
processing_time_event=TestStreamPayload.Event.AdvanceProcessingTime(
advance_duration=1 * 10**6)),
TestStreamPayload.Event(
watermark_event=TestStreamPayload.Event.AdvanceWatermark(
new_watermark=0,
tag='letters')),
TestStreamPayload.Event(
element_event=TestStreamPayload.Event.AddElements(
elements=[TestStreamPayload.TimestampedElement(
encoded_element=coder.encode('a'),
timestamp=0)],
tag='letters')),
# Advances clock from 1 to 2
TestStreamPayload.Event(
processing_time_event=TestStreamPayload.Event.AdvanceProcessingTime(
advance_duration=1 * 10**6)),
TestStreamPayload.Event(
element_event=TestStreamPayload.Event.AddElements(
elements=[TestStreamPayload.TimestampedElement(
encoded_element=coder.encode(1),
timestamp=0)],
tag='numbers')),
# Advances clock from 2 to 3
TestStreamPayload.Event(
processing_time_event=TestStreamPayload.Event.AdvanceProcessingTime(
advance_duration=1 * 10**6)),
TestStreamPayload.Event(
element_event=TestStreamPayload.Event.AddElements(
elements=[TestStreamPayload.TimestampedElement(
encoded_element=coder.encode(2),
timestamp=0)],
tag='numbers')),
# Advances clock from 3 to 4
TestStreamPayload.Event(
processing_time_event=TestStreamPayload.Event.AdvanceProcessingTime(
advance_duration=1 * 10**6)),
TestStreamPayload.Event(
element_event=TestStreamPayload.Event.AddElements(
elements=[TestStreamPayload.TimestampedElement(
encoded_element=coder.encode(2),
timestamp=0)],
tag='numbers')),
# Advances clock from 4 to 11
TestStreamPayload.Event(
processing_time_event=TestStreamPayload.Event.AdvanceProcessingTime(
advance_duration=7 * 10**6)),
TestStreamPayload.Event(
watermark_event=TestStreamPayload.Event.AdvanceWatermark(
new_watermark=10 * 10**6,
tag='letters')),
TestStreamPayload.Event(
element_event=TestStreamPayload.Event.AddElements(
elements=[TestStreamPayload.TimestampedElement(
encoded_element=coder.encode('b'),
timestamp=10 * 10**6)],
tag='letters')),
# Advances clock from 11 to 101
TestStreamPayload.Event(
processing_time_event=TestStreamPayload.Event.AdvanceProcessingTime(
advance_duration=90 * 10**6)),
TestStreamPayload.Event(
element_event=TestStreamPayload.Event.AddElements(
elements=[TestStreamPayload.TimestampedElement(
encoded_element=coder.encode('late'),
timestamp=0)],
tag='late')),
]
self.assertSequenceEqual(events, expected)