def process(self, o):
logging.debug('Processing [%s] in %s', o, self)
assert isinstance(o, WindowedValue)
keyed_work = o.value
driver = trigger.create_trigger_driver(self.windowing)
state = self.spec.context.state
for out_window, values in driver.process_elements(keyed_work.elements(),
state):
self.output(window.WindowedValue((keyed_work.key, values),
out_window.end, [out_window]))
def process(self, context):
k, vs = context.element
# pylint: disable=g-import-not-at-top
from google.cloud.dataflow.transforms.trigger import InMemoryUnmergedState
from google.cloud.dataflow.transforms.trigger import create_trigger_driver
# pylint: enable=g-import-not-at-top
driver = create_trigger_driver(self.windowing, True)
state = InMemoryUnmergedState()
# TODO(robertwb): Conditionally process in smaller chunks.
for wvalue in driver.process_elements(state, vs, MIN_TIMESTAMP):
yield wvalue.with_value((k, wvalue.value))
while state.timers:
fired = state.get_and_clear_timers()
for timer_window, (name, time_domain, fire_time) in fired:
for wvalue in driver.process_timer(
timer_window, name, time_domain, fire_time, state):
yield wvalue.with_value((k, wvalue.value))
def process(self, context):
k, vs = context.element
# pylint: disable=g-import-not-at-top
from google.cloud.dataflow.transforms.trigger import InMemoryUnmergedState
from google.cloud.dataflow.transforms.trigger import create_trigger_driver
# pylint: enable=g-import-not-at-top
driver = create_trigger_driver(self.windowing, True)
state = InMemoryUnmergedState()
# TODO(robertwb): Conditionally process in smaller chunks.
for wvalue in driver.process_elements(state, vs, MIN_TIMESTAMP):
yield wvalue.with_value((k, wvalue.value))
while state.timers:
fired = state.get_and_clear_timers()
for timer_window, (name, time_domain, fire_time) in fired:
for wvalue in driver.process_timer(
timer_window, name, time_domain, fire_time, state):
yield wvalue.with_value((k, wvalue.value))
def process(self, context):
k, vs = context.element
# pylint: disable=g-import-not-at-top
from google.cloud.dataflow.transforms.trigger import InMemoryUnmergedState
from google.cloud.dataflow.transforms.trigger import create_trigger_driver
# pylint: enable=g-import-not-at-top
driver = create_trigger_driver(self.windowing, True)
state = InMemoryUnmergedState()
# TODO(robertwb): Conditionally process in smaller chunks.
for out_window, values in driver.process_elements(vs, state):
yield window.WindowedValue((k, values), out_window.end, [out_window])
while state.timers:
for timer_window, (tag, timestamp) in state.get_and_clear_timers():
for out_window, values in driver.process_timer(timer_window,
timestamp, tag, state):
yield window.WindowedValue(
(k, values), out_window.end, [out_window])
def process(self, o):
"""Process a given value."""
logging.debug('Processing [%s] in %s', o, self)
assert isinstance(o, WindowedValue)
k, vs = o.value
driver = trigger.create_trigger_driver(self.windowing, True)
state = InMemoryUnmergedState()
# TODO(robertwb): Process in smaller chunks.
for out_window, values in driver.process_elements(vs, state):
self.output(
window.WindowedValue((k, values), out_window.end, [out_window]))
while state.timers:
for timer_window, (tag, timestamp) in state.get_and_clear_timers():
for out_window, values in driver.process_timer(timer_window,
timestamp, tag, state):
self.output(window.WindowedValue(
(k, values), out_window.end, [out_window]))
def process(self, o):
if self.debug_logging_enabled:
logging.debug('Processing [%s] in %s', o, self)
assert isinstance(o, WindowedValue)
keyed_work = o.value
driver = trigger.create_trigger_driver(self.windowing)
state = self.spec.context.state
output_watermark = self.spec.context.output_data_watermark
key = keyed_work.key
for wvalue in driver.process_elements(
state, keyed_work.elements(), output_watermark):
self.output(wvalue.with_value((key, wvalue.value)))
for timer in keyed_work.timers():
timer_window = int(timer.namespace)
for wvalue in driver.process_timer(
timer_window, timer.name, timer.time_domain, timer.timestamp, state):
self.output(wvalue.with_value((key, wvalue.value)))
def process(self, o):
if self.debug_logging_enabled:
logging.debug('Processing [%s] in %s', o, self)
assert isinstance(o, WindowedValue)
keyed_work = o.value
driver = trigger.create_trigger_driver(self.windowing)
state = self.spec.context.state
output_watermark = self.spec.context.output_data_watermark
key = keyed_work.key
for wvalue in driver.process_elements(
state, keyed_work.elements(), output_watermark):
self.output(wvalue.with_value((key, wvalue.value)))
for timer in keyed_work.timers():
timer_window = int(timer.namespace)
for wvalue in driver.process_timer(
timer_window, timer.name, timer.time_domain, timer.timestamp, state):
self.output(wvalue.with_value((key, wvalue.value)))
def process(self, o):
"""Process a given value."""
if self.debug_logging_enabled:
logging.debug('Processing [%s] in %s', o, self)
assert isinstance(o, WindowedValue)
k, vs = o.value
driver = trigger.create_trigger_driver(
self.windowing, is_batch=True, phased_combine_fn=self.phased_combine_fn)
state = InMemoryUnmergedState()
# TODO(robertwb): Process in smaller chunks.
for wvalue in driver.process_elements(state, vs, MIN_TIMESTAMP):
self.output(wvalue.with_value((k, wvalue.value)))
while state.timers:
timers = state.get_and_clear_timers()
for timer_window, (name, time_domain, timestamp) in timers:
for wvalue in driver.process_timer(
timer_window, name, time_domain, timestamp, state):
self.output(wvalue.with_value((k, wvalue.value)))
def process(self, o):
"""Process a given value."""
if self.debug_logging_enabled:
logging.debug('Processing [%s] in %s', o, self)
assert isinstance(o, WindowedValue)
k, vs = o.value
driver = trigger.create_trigger_driver(
self.windowing, is_batch=True, phased_combine_fn=self.phased_combine_fn)
state = InMemoryUnmergedState()
# TODO(robertwb): Process in smaller chunks.
for wvalue in driver.process_elements(state, vs, MIN_TIMESTAMP):
self.output(wvalue.with_value((k, wvalue.value)))
while state.timers:
timers = state.get_and_clear_timers()
for timer_window, (name, time_domain, timestamp) in timers:
for wvalue in driver.process_timer(
timer_window, name, time_domain, timestamp, state):
self.output(wvalue.with_value((k, wvalue.value)))