def expand(self, pbegin):
if not isinstance(pbegin, pvalue.PBegin):
raise Exception("GenerateSequence must be a root transform")
coder = VarIntCoder()
coder_urn = ['beam:coder:varint:v1']
args = {
'start':
ConfigValue(coder_urn=coder_urn, payload=coder.encode(self.start))
}
if self.stop:
args['stop'] = ConfigValue(coder_urn=coder_urn,
payload=coder.encode(self.stop))
if self.elements_per_period:
args['elements_per_period'] = ConfigValue(
coder_urn=coder_urn,
payload=coder.encode(self.elements_per_period))
if self.max_read_time:
args['max_read_time'] = ConfigValue(coder_urn=coder_urn,
payload=coder.encode(
self.max_read_time))
payload = ExternalConfigurationPayload(configuration=args)
return pbegin.apply(
ExternalTransform(self._urn, payload.SerializeToString(),
self.expansion_service))
def __init__(self,
start,
stop=None,
elements_per_period=None,
max_read_time=None,
expansion_service=None):
coder = VarIntCoder()
coder_urn = 'beam:coder:varint:v1'
args = {
'start': ConfigValue(coder_urn=coder_urn,
payload=coder.encode(start))
}
if stop:
args['stop'] = ConfigValue(coder_urn=coder_urn,
payload=coder.encode(stop))
if elements_per_period:
args['elements_per_period'] = ConfigValue(
coder_urn=coder_urn, payload=coder.encode(elements_per_period))
if max_read_time:
args['max_read_time'] = ConfigValue(
coder_urn=coder_urn, payload=coder.encode(max_read_time))
payload = ExternalConfigurationPayload(configuration=args)
super(GenerateSequence,
self).__init__('beam:external:java:generate_sequence:v1',
payload.SerializeToString(), expansion_service)
def expand(self, pvalue):
args = {
'producer_config': _encode_map(self.producer_config),
'topic': _encode_str(self.topic),
'key_serializer': _encode_str(self.key_serializer),
'value_serializer': _encode_str(self.value_serializer),
}
payload = ExternalConfigurationPayload(configuration=args)
return pvalue.apply(
ExternalTransform(self._urn, payload.SerializeToString(),
self.expansion_service))
def expand(self, pbegin):
if not isinstance(pbegin, pvalue.PBegin):
raise Exception("ReadFromKafka must be a root transform")
args = {
'consumer_config': _encode_map(self.consumer_config),
'topics': _encode_list(self.topics),
'key_deserializer': _encode_str(self.key_deserializer),
'value_deserializer': _encode_str(self.value_deserializer),
}
payload = ExternalConfigurationPayload(configuration=args)
return pbegin.apply(
ExternalTransform(self._urn, payload.SerializeToString(),
self.expansion_service))