def process(self, o): logging.debug('Processing [%s] in %s', o, self) assert isinstance(o, WindowedValue) keyed_work = o.value driver = trigger.create_trigger_driver(self.windowing) state = self.spec.context.state for out_window, values in driver.process_elements(keyed_work.elements(), state): self.output(window.WindowedValue((keyed_work.key, values), out_window.end, [out_window]))
def process(self, context): k, vs = context.element # pylint: disable=g-import-not-at-top from google.cloud.dataflow.transforms.trigger import InMemoryUnmergedState from google.cloud.dataflow.transforms.trigger import create_trigger_driver # pylint: enable=g-import-not-at-top driver = create_trigger_driver(self.windowing, True) state = InMemoryUnmergedState() # TODO(robertwb): Conditionally process in smaller chunks. for wvalue in driver.process_elements(state, vs, MIN_TIMESTAMP): yield wvalue.with_value((k, wvalue.value)) while state.timers: fired = state.get_and_clear_timers() for timer_window, (name, time_domain, fire_time) in fired: for wvalue in driver.process_timer( timer_window, name, time_domain, fire_time, state): yield wvalue.with_value((k, wvalue.value))
def process(self, context): k, vs = context.element # pylint: disable=g-import-not-at-top from google.cloud.dataflow.transforms.trigger import InMemoryUnmergedState from google.cloud.dataflow.transforms.trigger import create_trigger_driver # pylint: enable=g-import-not-at-top driver = create_trigger_driver(self.windowing, True) state = InMemoryUnmergedState() # TODO(robertwb): Conditionally process in smaller chunks. for out_window, values in driver.process_elements(vs, state): yield window.WindowedValue((k, values), out_window.end, [out_window]) while state.timers: for timer_window, (tag, timestamp) in state.get_and_clear_timers(): for out_window, values in driver.process_timer(timer_window, timestamp, tag, state): yield window.WindowedValue( (k, values), out_window.end, [out_window])
def process(self, o): """Process a given value.""" logging.debug('Processing [%s] in %s', o, self) assert isinstance(o, WindowedValue) k, vs = o.value driver = trigger.create_trigger_driver(self.windowing, True) state = InMemoryUnmergedState() # TODO(robertwb): Process in smaller chunks. for out_window, values in driver.process_elements(vs, state): self.output( window.WindowedValue((k, values), out_window.end, [out_window])) while state.timers: for timer_window, (tag, timestamp) in state.get_and_clear_timers(): for out_window, values in driver.process_timer(timer_window, timestamp, tag, state): self.output(window.WindowedValue( (k, values), out_window.end, [out_window]))
def process(self, o): if self.debug_logging_enabled: logging.debug('Processing [%s] in %s', o, self) assert isinstance(o, WindowedValue) keyed_work = o.value driver = trigger.create_trigger_driver(self.windowing) state = self.spec.context.state output_watermark = self.spec.context.output_data_watermark key = keyed_work.key for wvalue in driver.process_elements( state, keyed_work.elements(), output_watermark): self.output(wvalue.with_value((key, wvalue.value))) for timer in keyed_work.timers(): timer_window = int(timer.namespace) for wvalue in driver.process_timer( timer_window, timer.name, timer.time_domain, timer.timestamp, state): self.output(wvalue.with_value((key, wvalue.value)))
def process(self, o): """Process a given value.""" if self.debug_logging_enabled: logging.debug('Processing [%s] in %s', o, self) assert isinstance(o, WindowedValue) k, vs = o.value driver = trigger.create_trigger_driver( self.windowing, is_batch=True, phased_combine_fn=self.phased_combine_fn) state = InMemoryUnmergedState() # TODO(robertwb): Process in smaller chunks. for wvalue in driver.process_elements(state, vs, MIN_TIMESTAMP): self.output(wvalue.with_value((k, wvalue.value))) while state.timers: timers = state.get_and_clear_timers() for timer_window, (name, time_domain, timestamp) in timers: for wvalue in driver.process_timer( timer_window, name, time_domain, timestamp, state): self.output(wvalue.with_value((k, wvalue.value)))