def open(self, runtime_context: RuntimeContext):
self.num_partitions = int(
runtime_context.get_job_parameter("NUM_PARTITIONS", "-1"))
if self.num_partitions <= 0:
raise ValueError(
"The partition number should be a positive value, got %s"
% self.num_partitions)
def open(self, runtime_context: RuntimeContext,
internal_timer_service: InternalTimerService):
self.window_function.open(runtime_context)
self.num_late_records_dropped = runtime_context.get_metrics_group(
).counter(self.LATE_ELEMENTS_DROPPED_METRIC_NAME)
self.internal_timer_service = internal_timer_service
self.trigger_context = Context(runtime_context, internal_timer_service,
self.trigger)
self.process_context = WindowContext(self.window_assigner,
runtime_context,
self.window_function,
self.internal_timer_service)
self.window_assigner_context = WindowAssignerContext(
self.internal_timer_service, runtime_context)
# create (or restore) the state that hold the actual window contents
# NOTE - the state may be null in the case of the overriding evicting window operator
if self.window_state_descriptor is not None:
self.window_state = get_or_create_keyed_state(
runtime_context, self.window_state_descriptor)
if isinstance(self.window_assigner, MergingWindowAssigner):
if isinstance(self.window_state, InternalMergingState):
self.window_merging_state = self.window_state
window_coder = self.keyed_state_backend.namespace_coder
self.merging_sets_state = self.keyed_state_backend.get_map_state(
"merging-window-set", window_coder, window_coder)
self.merge_function = WindowMergeFunction(self)
def open(self, runtime_context: RuntimeContext):
state_descriptor = ValueStateDescriptor("state", Types.FLOAT())
state_ttl_config = StateTtlConfig \
.new_builder(Time.seconds(1)) \
.set_update_type(StateTtlConfig.UpdateType.OnReadAndWrite) \
.disable_cleanup_in_background() \
.build()
state_descriptor.enable_time_to_live(state_ttl_config)
self.state = runtime_context.get_state(state_descriptor)
def open_func(self):
for user_defined_func in self.user_defined_funcs:
runtime_context = RuntimeContext(
self.spec.serialized_fn.runtime_context.task_name,
self.spec.serialized_fn.runtime_context.task_name_with_subtasks,
self.spec.serialized_fn.runtime_context.number_of_parallel_subtasks,
self.spec.serialized_fn.runtime_context.max_number_of_parallel_subtasks,
self.spec.serialized_fn.runtime_context.index_of_this_subtask,
self.spec.serialized_fn.runtime_context.attempt_number,
{p.key: p.value for p in self.spec.serialized_fn.runtime_context.job_parameters})
user_defined_func.open(runtime_context)
def open(self, runtime_context: RuntimeContext,
internal_timer_service: InternalTimerService):
self.window_function.open(runtime_context)
self.num_late_records_dropped = runtime_context.get_metrics_group(
).counter(self.LATE_ELEMENTS_DROPPED_METRIC_NAME)
self.internal_timer_service = internal_timer_service
self.trigger_context = Context(runtime_context, internal_timer_service,
self.trigger)
self.process_context = WindowContext(self.window_assigner,
runtime_context,
self.window_function,
self.internal_timer_service)
self.window_assigner_context = WindowAssignerContext(
self.internal_timer_service, runtime_context)
# create (or restore) the state that hold the actual window contents
# NOTE - the state may be null in the case of the overriding evicting window operator
if self.window_state_descriptor is not None:
self.window_state = get_or_create_keyed_state(
runtime_context, self.window_state_descriptor)
if isinstance(self.window_assigner, MergingWindowAssigner):
if isinstance(self.window_state, InternalMergingState):
self.window_merging_state = self.window_state
# TODO: the type info is just a placeholder currently.
# it should be the real type serializer after supporting the user-defined state type
# serializer
merging_sets_state_descriptor = ListStateDescriptor(
"merging-window-set", Types.PICKLED_BYTE_ARRAY())
self.merging_sets_state = get_or_create_keyed_state(
runtime_context, merging_sets_state_descriptor)
self.merge_function = WindowMergeFunction(self)
def open(self, runtime_context: RuntimeContext):
state_desc = MapStateDescriptor('map', Types.LONG(), Types.LONG())
self.state = runtime_context.get_map_state(state_desc)
def open(self, runtime_context: RuntimeContext):
state_desc = ValueStateDescriptor('cnt', Types.LONG())
self.cnt_state = runtime_context.get_state(state_desc)
