def finish_bundle(self): data = self._read_from_pubsub(self.source.timestamp_attribute) if data: output_pcollection = list(self._outputs)[0] bundle = self._evaluation_context.create_bundle(output_pcollection) # TODO(ccy): Respect the PubSub source's id_label field. for timestamp, message in data: if self.source.with_attributes: element = message else: element = message.data bundle.output( GlobalWindows.windowed_value(element, timestamp=timestamp)) bundles = [bundle] else: bundles = [] if self._applied_ptransform.inputs: input_pvalue = self._applied_ptransform.inputs[0] else: input_pvalue = pvalue.PBegin(self._applied_ptransform.transform.pipeline) unprocessed_bundle = self._evaluation_context.create_bundle( input_pvalue) # TODO(udim): Correct value for watermark hold. return TransformResult(self, bundles, [unprocessed_bundle], None, {None: Timestamp.of(time.time())})
def __init__(self, start, end): if start is not None or end is not None: self._start_object = Timestamp.of(start) self._end_object = Timestamp.of(end) try: self._start_micros = self._start_object.micros except OverflowError: self._start_micros = ( MIN_TIMESTAMP.micros if self._start_object.micros < 0 else MAX_TIMESTAMP.micros) try: self._end_micros = self._end_object.micros except OverflowError: self._end_micros = ( MIN_TIMESTAMP.micros if self._end_object.micros < 0 else MAX_TIMESTAMP.micros) else: # Micros must be populated elsewhere. self._start_object = self._end_object = None
def __init__(self, value, timestamp, windows): # For performance reasons, only timestamp_micros is stored by default # (as a C int). The Timestamp object is created on demand below. self.value = value if isinstance(timestamp, int): self.timestamp_micros = timestamp * 1000000 else: self.timestamp_object = (timestamp if isinstance(timestamp, Timestamp) else Timestamp.of(timestamp)) self.timestamp_micros = self.timestamp_object.micros self.windows = windows
def finish_bundle(self): data = self._read_from_pubsub() if data: output_pcollection = list(self._outputs)[0] bundle = self._evaluation_context.create_bundle(output_pcollection) # TODO(ccy): we currently do not use the PubSub message timestamp or # respect the PubSub source's id_label field. now = Timestamp.of(time.time()) for message_data in data: bundle.output(GlobalWindows.windowed_value(message_data, timestamp=now)) bundles = [bundle] else: bundles = [] if self._applied_ptransform.inputs: input_pvalue = self._applied_ptransform.inputs[0] else: input_pvalue = pvalue.PBegin(self._applied_ptransform.transform.pipeline) unprocessed_bundle = self._evaluation_context.create_bundle( input_pvalue) return TransformResult( self._applied_ptransform, bundles, [unprocessed_bundle], None, {None: Timestamp.of(time.time())})
def __init__(self, value, timestamp): self.value = value self.timestamp = Timestamp.of(timestamp)
def __init__(self, end): self.end = Timestamp.of(end)
def __init__(self, start, end): super(IntervalWindow, self).__init__(end) self.start = Timestamp.of(start)
def __init__(self, new_watermark, tag=None): self.new_watermark = Timestamp.of(new_watermark) self.tag = tag
def __init__(self, timestamp, element=None, window=None): self.timestamp = Timestamp.of(timestamp) self.element = element self.window = window
def __init__(self, value, timestamp): # type: (Any, TimestampTypes) -> None self.value = value self.timestamp = Timestamp.of(timestamp)
def test_timestamps(self): wv = windowed_value.WindowedValue(None, 3, ()) self.assertEqual(wv.timestamp, Timestamp.of(3)) self.assertTrue(wv.timestamp is wv.timestamp) self.assertEqual(windowed_value.WindowedValue(None, -2.5, ()).timestamp, Timestamp.of(-2.5))
def test_of(self): interval = Timestamp(123) self.assertEqual(id(interval), id(Timestamp.of(interval))) self.assertEqual(interval, Timestamp.of(123.0)) with self.assertRaises(TypeError): Timestamp.of(Duration(10))
def test_of(self): interval = Timestamp(123) self.assertEqual(id(interval), id(Timestamp.of(interval))) self.assertEqual(interval, Timestamp.of(123.0)) with self.assertRaises(TypeError): Timestamp.of(Duration(10))
def __init__(self, value, timestamp): self.value = value self.timestamp = Timestamp.of(timestamp)
def __init__(self, size, period, offset=0): if size <= 0: raise ValueError('The size parameter must be strictly positive.') self.size = Duration.of(size) self.period = Duration.of(period) self.offset = Timestamp.of(offset) % period
def __init__(self, start, end): super(IntervalWindow, self).__init__(end) self.start = Timestamp.of(start)
def __init__(self, end): self.end = Timestamp.of(end)
def __init__(self, timestamp, element=None): self.timestamp = Timestamp.of(timestamp) self.element = element
def __init__(self, size, period, offset=0): if size <= 0: raise ValueError('The size parameter must be strictly positive.') self.size = Duration.of(size) self.period = Duration.of(period) self.offset = Timestamp.of(offset) % period
def __init__(self, processing_time, watermark): self._processing_time = Timestamp.of(processing_time) self._watermark = Timestamp.of(watermark)
def advance_watermark(self, watermark_secs): record = TestStreamFileRecord(recorded_event=TestStreamPayload.Event( watermark_event=TestStreamPayload.Event.AdvanceWatermark( new_watermark=Timestamp.of(watermark_secs).micros))) self._records.append(record) return self
def __init__(self, end): # type: (TimestampTypes) -> None self._end = Timestamp.of(end)