def finish_bundle(self):
data = self._read_from_pubsub(self.source.timestamp_attribute)
if data:
output_pcollection = list(self._outputs)[0]
bundle = self._evaluation_context.create_bundle(output_pcollection)
# TODO(ccy): Respect the PubSub source's id_label field.
for timestamp, message in data:
if self.source.with_attributes:
element = message
else:
element = message.data
bundle.output(
GlobalWindows.windowed_value(element, timestamp=timestamp))
bundles = [bundle]
else:
bundles = []
if self._applied_ptransform.inputs:
input_pvalue = self._applied_ptransform.inputs[0]
else:
input_pvalue = pvalue.PBegin(self._applied_ptransform.transform.pipeline)
unprocessed_bundle = self._evaluation_context.create_bundle(
input_pvalue)
# TODO(udim): Correct value for watermark hold.
return TransformResult(self, bundles, [unprocessed_bundle], None,
{None: Timestamp.of(time.time())})
def __init__(self, start, end):
if start is not None or end is not None:
self._start_object = Timestamp.of(start)
self._end_object = Timestamp.of(end)
try:
self._start_micros = self._start_object.micros
except OverflowError:
self._start_micros = (
MIN_TIMESTAMP.micros if self._start_object.micros < 0
else MAX_TIMESTAMP.micros)
try:
self._end_micros = self._end_object.micros
except OverflowError:
self._end_micros = (
MIN_TIMESTAMP.micros if self._end_object.micros < 0
else MAX_TIMESTAMP.micros)
else:
# Micros must be populated elsewhere.
self._start_object = self._end_object = None
def __init__(self, value, timestamp, windows):
# For performance reasons, only timestamp_micros is stored by default
# (as a C int). The Timestamp object is created on demand below.
self.value = value
if isinstance(timestamp, int):
self.timestamp_micros = timestamp * 1000000
else:
self.timestamp_object = (timestamp if isinstance(timestamp, Timestamp)
else Timestamp.of(timestamp))
self.timestamp_micros = self.timestamp_object.micros
self.windows = windows
def finish_bundle(self):
data = self._read_from_pubsub()
if data:
output_pcollection = list(self._outputs)[0]
bundle = self._evaluation_context.create_bundle(output_pcollection)
# TODO(ccy): we currently do not use the PubSub message timestamp or
# respect the PubSub source's id_label field.
now = Timestamp.of(time.time())
for message_data in data:
bundle.output(GlobalWindows.windowed_value(message_data, timestamp=now))
bundles = [bundle]
else:
bundles = []
if self._applied_ptransform.inputs:
input_pvalue = self._applied_ptransform.inputs[0]
else:
input_pvalue = pvalue.PBegin(self._applied_ptransform.transform.pipeline)
unprocessed_bundle = self._evaluation_context.create_bundle(
input_pvalue)
return TransformResult(
self._applied_ptransform, bundles,
[unprocessed_bundle], None, {None: Timestamp.of(time.time())})
def __init__(self, value, timestamp): self.value = value self.timestamp = Timestamp.of(timestamp)
def __init__(self, end): self.end = Timestamp.of(end)
def __init__(self, start, end): super(IntervalWindow, self).__init__(end) self.start = Timestamp.of(start)
def __init__(self, new_watermark, tag=None): self.new_watermark = Timestamp.of(new_watermark) self.tag = tag
def __init__(self, timestamp, element=None, window=None): self.timestamp = Timestamp.of(timestamp) self.element = element self.window = window
def __init__(self, value, timestamp):
# type: (Any, TimestampTypes) -> None
self.value = value
self.timestamp = Timestamp.of(timestamp)
def test_timestamps(self):
wv = windowed_value.WindowedValue(None, 3, ())
self.assertEqual(wv.timestamp, Timestamp.of(3))
self.assertTrue(wv.timestamp is wv.timestamp)
self.assertEqual(windowed_value.WindowedValue(None, -2.5, ()).timestamp,
Timestamp.of(-2.5))
def test_of(self):
interval = Timestamp(123)
self.assertEqual(id(interval), id(Timestamp.of(interval)))
self.assertEqual(interval, Timestamp.of(123.0))
with self.assertRaises(TypeError):
Timestamp.of(Duration(10))
def test_of(self):
interval = Timestamp(123)
self.assertEqual(id(interval), id(Timestamp.of(interval)))
self.assertEqual(interval, Timestamp.of(123.0))
with self.assertRaises(TypeError):
Timestamp.of(Duration(10))
def __init__(self, value, timestamp):
self.value = value
self.timestamp = Timestamp.of(timestamp)
def __init__(self, size, period, offset=0):
if size <= 0:
raise ValueError('The size parameter must be strictly positive.')
self.size = Duration.of(size)
self.period = Duration.of(period)
self.offset = Timestamp.of(offset) % period
def __init__(self, start, end):
super(IntervalWindow, self).__init__(end)
self.start = Timestamp.of(start)
def __init__(self, end):
self.end = Timestamp.of(end)
def __init__(self, timestamp, element=None):
self.timestamp = Timestamp.of(timestamp)
self.element = element
def __init__(self, size, period, offset=0):
if size <= 0:
raise ValueError('The size parameter must be strictly positive.')
self.size = Duration.of(size)
self.period = Duration.of(period)
self.offset = Timestamp.of(offset) % period
def __init__(self, processing_time, watermark): self._processing_time = Timestamp.of(processing_time) self._watermark = Timestamp.of(watermark)
def advance_watermark(self, watermark_secs):
record = TestStreamFileRecord(recorded_event=TestStreamPayload.Event(
watermark_event=TestStreamPayload.Event.AdvanceWatermark(
new_watermark=Timestamp.of(watermark_secs).micros)))
self._records.append(record)
return self
def __init__(self, end):
# type: (TimestampTypes) -> None
self._end = Timestamp.of(end)
