def expand(self, pbegin):
if not isinstance(pbegin, pvalue.PBegin):
raise Exception("GenerateSequence must be a root transform")
coder = VarIntCoder()
coder_urn = ['beam:coder:varint:v1']
args = {
'start':
ConfigValue(coder_urn=coder_urn, payload=coder.encode(self.start))
}
if self.stop:
args['stop'] = ConfigValue(coder_urn=coder_urn,
payload=coder.encode(self.stop))
if self.elements_per_period:
args['elements_per_period'] = ConfigValue(
coder_urn=coder_urn,
payload=coder.encode(self.elements_per_period))
if self.max_read_time:
args['max_read_time'] = ConfigValue(coder_urn=coder_urn,
payload=coder.encode(
self.max_read_time))
payload = ExternalConfigurationPayload(configuration=args)
return pbegin.apply(
ExternalTransform(self._urn, payload.SerializeToString(),
self.expansion_service))
def __init__(self,
start,
stop=None,
elements_per_period=None,
max_read_time=None,
expansion_service=None):
coder = VarIntCoder()
coder_urn = 'beam:coder:varint:v1'
args = {
'start': ConfigValue(coder_urn=coder_urn,
payload=coder.encode(start))
}
if stop:
args['stop'] = ConfigValue(coder_urn=coder_urn,
payload=coder.encode(stop))
if elements_per_period:
args['elements_per_period'] = ConfigValue(
coder_urn=coder_urn, payload=coder.encode(elements_per_period))
if max_read_time:
args['max_read_time'] = ConfigValue(
coder_urn=coder_urn, payload=coder.encode(max_read_time))
payload = ExternalConfigurationPayload(configuration=args)
super(GenerateSequence,
self).__init__('beam:external:java:generate_sequence:v1',
payload.SerializeToString(), expansion_service)
def parse_string_payload(input_byte):
payload = ExternalConfigurationPayload()
payload.ParseFromString(input_byte)
coder = StrUtf8Coder()
return {
k: coder.decode_nested(v.payload)
for k, v in payload.configuration.items()
}
def expand(self, pvalue):
args = {
'producer_config': _encode_map(self.producer_config),
'topic': _encode_str(self.topic),
'key_serializer': _encode_str(self.key_serializer),
'value_serializer': _encode_str(self.value_serializer),
}
payload = ExternalConfigurationPayload(configuration=args)
return pvalue.apply(
ExternalTransform(self._urn, payload.SerializeToString(),
self.expansion_service))
def expand(self, pbegin):
if not isinstance(pbegin, pvalue.PBegin):
raise Exception("ReadFromKafka must be a root transform")
args = {
'consumer_config': _encode_map(self.consumer_config),
'topics': _encode_list(self.topics),
'key_deserializer': _encode_str(self.key_deserializer),
'value_deserializer': _encode_str(self.value_deserializer),
}
payload = ExternalConfigurationPayload(configuration=args)
return pbegin.apply(
ExternalTransform(self._urn, payload.SerializeToString(),
self.expansion_service))
def to_runner_api_parameter(self, unused_context):
_args_schema = named_fields_to_schema([
(f'arg{ix}', convert_to_typing_type(instance_to_type(value)))
for (ix, value) in enumerate(self._args)
])
_kwargs_schema = named_fields_to_schema([
(key, convert_to_typing_type(instance_to_type(value)))
for (key, value) in self._kwargs.items()
])
payload_schema = named_fields_to_schema({
'constructor': str,
'args': _args_schema,
'kwargs': _kwargs_schema,
})
return (PYTHON_FULLY_QUALIFIED_NAMED_TRANSFORM_URN,
ExternalConfigurationPayload(
schema=payload_schema,
payload=coders.RowCoder(payload_schema).encode(
Row(constructor=self._constructor,
args=Row(
**{
f'arg{ix}': arg
for (ix, arg) in enumerate(self._args)
}),
kwargs=Row(**self._kwargs)), )))
def build(self):
"""
:return: ExternalConfigurationPayload
"""
args = self._encode_config(self._values, self._schema)
return ExternalConfigurationPayload(configuration=args)
def build(self):
row = self._get_named_tuple_instance()
schema = named_tuple_to_schema(type(row))
return ExternalConfigurationPayload(
schema=schema, payload=RowCoder(schema).encode(row))
def get_payload(cls):
payload = ExternalConfigurationPayload()
payload.ParseFromString(cls._payload)
return payload
def get_payload(args):
return ExternalConfigurationPayload(configuration=args)
def parse_string_payload(input_byte):
payload = ExternalConfigurationPayload()
payload.ParseFromString(input_byte)
return RowCoder(payload.schema).decode(payload.payload)._asdict()