def inner(fn): sentry_init(default_integrations=False, integrations=[BeamIntegration()]) # Little hack to avoid having to run the whole pipeline. pardo = ParDo(fn) signature = pardo._signature output_processor = _OutputProcessor() return DoFnInvoker.create_invoker(signature, output_processor, DoFnContext("test"))
def __init__( self, sdf, args_for_invoker, kwargs_for_invoker): self.sdf = sdf self._element_tag = _ValueStateTag('element') self._restriction_tag = _ValueStateTag('restriction') self.watermark_hold_tag = _ValueStateTag('watermark_hold') self._process_element_invoker = None self.sdf_invoker = DoFnInvoker.create_invoker( DoFnSignature(self.sdf), context=DoFnContext('unused_context'), input_args=args_for_invoker, input_kwargs=kwargs_for_invoker) self._step_context = None
def __init__( self, sdf, args_for_invoker, kwargs_for_invoker): self.sdf = sdf self._element_tag = _ValueStateTag('element') self._restriction_tag = _ValueStateTag('restriction') self.watermark_hold_tag = _ValueStateTag('watermark_hold') self._process_element_invoker = None self.sdf_invoker = DoFnInvoker.create_invoker( DoFnSignature(self.sdf), context=DoFnContext('unused_context'), input_args=args_for_invoker, input_kwargs=kwargs_for_invoker) self._step_context = None
def expand(self, pcoll): sdf = self._ptransform.fn signature = DoFnSignature(sdf) invoker = DoFnInvoker.create_invoker(signature, process_invocation=False) element_coder = typecoders.registry.get_coder(pcoll.element_type) restriction_coder = invoker.invoke_restriction_coder() keyed_elements = (pcoll | 'pair' >> ParDo(PairWithRestrictionFn(sdf)) | 'split' >> ParDo(SplitRestrictionFn(sdf)) | 'explode' >> ParDo(ExplodeWindowsFn()) | 'random' >> ParDo(RandomUniqueKeyFn())) return keyed_elements | ProcessKeyedElements( sdf, element_coder, restriction_coder, pcoll.windowing, self._ptransform.args, self._ptransform.kwargs)
def expand(self, pcoll): sdf = self._ptransform.fn signature = DoFnSignature(sdf) invoker = DoFnInvoker.create_invoker(signature, process_invocation=False) element_coder = typecoders.registry.get_coder(pcoll.element_type) restriction_coder = invoker.invoke_restriction_coder() keyed_elements = (pcoll | 'pair' >> ParDo(PairWithRestrictionFn(sdf)) | 'split' >> ParDo(SplitRestrictionFn(sdf)) | 'explode' >> ParDo(ExplodeWindowsFn()) | 'random' >> ParDo(RandomUniqueKeyFn())) return keyed_elements | ProcessKeyedElements( sdf, element_coder, restriction_coder, pcoll.windowing, self._ptransform.args, self._ptransform.kwargs)
def __init__(self, sdf, args_for_invoker, kwargs_for_invoker): self.sdf = sdf self._element_tag = _ReadModifyWriteStateTag('element') self._restriction_tag = _ReadModifyWriteStateTag('restriction') self._watermark_state_tag = _ReadModifyWriteStateTag( 'watermark_estimator_state') self.watermark_hold_tag = _ReadModifyWriteStateTag('watermark_hold') self._process_element_invoker = None self._output_processor = _OutputProcessor() self.sdf_invoker = DoFnInvoker.create_invoker( DoFnSignature(self.sdf), context=DoFnContext('unused_context'), output_processor=self._output_processor, input_args=args_for_invoker, input_kwargs=kwargs_for_invoker) self._step_context = None
def start_bundle(self): signature = DoFnSignature(self._do_fn) self._invoker = DoFnInvoker.create_invoker(signature, process_invocation=False)
def start_bundle(self): signature = DoFnSignature(self._do_fn) self._invoker = DoFnInvoker.create_invoker( signature, output_processor=_NoneShallPassOutputProcessor(), process_invocation=False)
def start_bundle(self): signature = DoFnSignature(self._do_fn) self._invoker = DoFnInvoker.create_invoker( signature, process_invocation=False)
def start_bundle(self): self._invoker = DoFnInvoker.create_invoker( self._signature, output_processor=_NoneShallPassOutputHandler(), process_invocation=False)