def expand(self, pbegin): if not isinstance(pbegin, pvalue.PBegin): raise Exception("GenerateSequence must be a root transform") coder = VarIntCoder() coder_urn = ['beam:coder:varint:v1'] args = { 'start': ConfigValue(coder_urn=coder_urn, payload=coder.encode(self.start)) } if self.stop: args['stop'] = ConfigValue(coder_urn=coder_urn, payload=coder.encode(self.stop)) if self.elements_per_period: args['elements_per_period'] = ConfigValue( coder_urn=coder_urn, payload=coder.encode(self.elements_per_period)) if self.max_read_time: args['max_read_time'] = ConfigValue(coder_urn=coder_urn, payload=coder.encode( self.max_read_time)) payload = ExternalConfigurationPayload(configuration=args) return pbegin.apply( ExternalTransform(self._urn, payload.SerializeToString(), self.expansion_service))
def __init__(self, start, stop=None, elements_per_period=None, max_read_time=None, expansion_service=None): coder = VarIntCoder() coder_urn = 'beam:coder:varint:v1' args = { 'start': ConfigValue(coder_urn=coder_urn, payload=coder.encode(start)) } if stop: args['stop'] = ConfigValue(coder_urn=coder_urn, payload=coder.encode(stop)) if elements_per_period: args['elements_per_period'] = ConfigValue( coder_urn=coder_urn, payload=coder.encode(elements_per_period)) if max_read_time: args['max_read_time'] = ConfigValue( coder_urn=coder_urn, payload=coder.encode(max_read_time)) payload = ExternalConfigurationPayload(configuration=args) super(GenerateSequence, self).__init__('beam:external:java:generate_sequence:v1', payload.SerializeToString(), expansion_service)
def parse_string_payload(input_byte): payload = ExternalConfigurationPayload() payload.ParseFromString(input_byte) coder = StrUtf8Coder() return { k: coder.decode_nested(v.payload) for k, v in payload.configuration.items() }
def expand(self, pvalue): args = { 'producer_config': _encode_map(self.producer_config), 'topic': _encode_str(self.topic), 'key_serializer': _encode_str(self.key_serializer), 'value_serializer': _encode_str(self.value_serializer), } payload = ExternalConfigurationPayload(configuration=args) return pvalue.apply( ExternalTransform(self._urn, payload.SerializeToString(), self.expansion_service))
def expand(self, pbegin): if not isinstance(pbegin, pvalue.PBegin): raise Exception("ReadFromKafka must be a root transform") args = { 'consumer_config': _encode_map(self.consumer_config), 'topics': _encode_list(self.topics), 'key_deserializer': _encode_str(self.key_deserializer), 'value_deserializer': _encode_str(self.value_deserializer), } payload = ExternalConfigurationPayload(configuration=args) return pbegin.apply( ExternalTransform(self._urn, payload.SerializeToString(), self.expansion_service))
def to_runner_api_parameter(self, unused_context): _args_schema = named_fields_to_schema([ (f'arg{ix}', convert_to_typing_type(instance_to_type(value))) for (ix, value) in enumerate(self._args) ]) _kwargs_schema = named_fields_to_schema([ (key, convert_to_typing_type(instance_to_type(value))) for (key, value) in self._kwargs.items() ]) payload_schema = named_fields_to_schema({ 'constructor': str, 'args': _args_schema, 'kwargs': _kwargs_schema, }) return (PYTHON_FULLY_QUALIFIED_NAMED_TRANSFORM_URN, ExternalConfigurationPayload( schema=payload_schema, payload=coders.RowCoder(payload_schema).encode( Row(constructor=self._constructor, args=Row( **{ f'arg{ix}': arg for (ix, arg) in enumerate(self._args) }), kwargs=Row(**self._kwargs)), )))
def build(self): """ :return: ExternalConfigurationPayload """ args = self._encode_config(self._values, self._schema) return ExternalConfigurationPayload(configuration=args)
def build(self): row = self._get_named_tuple_instance() schema = named_tuple_to_schema(type(row)) return ExternalConfigurationPayload( schema=schema, payload=RowCoder(schema).encode(row))
def get_payload(cls): payload = ExternalConfigurationPayload() payload.ParseFromString(cls._payload) return payload
def get_payload(args): return ExternalConfigurationPayload(configuration=args)
def parse_string_payload(input_byte): payload = ExternalConfigurationPayload() payload.ParseFromString(input_byte) return RowCoder(payload.schema).decode(payload.payload)._asdict()