def expand(self, pbegin):
return (pbegin
| 'User data mapper' >> beam.Map(
self.user_data_mapper).with_output_types(List[bytes])
| ExternalTransform(self.URN,
NamedTupleBasedPayloadBuilder(self.params),
self.expansion_service))
def expand(self, pbegin):
return (
pbegin
| ExternalTransform(
self.URN,
NamedTupleBasedPayloadBuilder(self.params),
self.expansion_service,
)
| 'CSV to array mapper' >> beam.Map(lambda csv: csv.split(b','))
| 'CSV mapper' >> beam.Map(self.csv_mapper))
def expand(self, pbegin):
pcoll = pbegin.apply(
ExternalTransform(self.URN,
NamedTupleBasedPayloadBuilder(self.params),
self.expansion_service))
if self.params.with_attributes:
pcoll = pcoll | 'FromProto' >> Map(
pubsub.PubsubMessage._from_proto_str)
pcoll.element_type = pubsub.PubsubMessage
else:
pcoll.element_type = bytes
return pcoll
def expand(self, pvalue):
if self.with_attributes:
pcoll = pvalue | 'ToProto' >> Map(
pubsub.WriteToPubSub.to_proto_str)
else:
pcoll = pvalue | 'ToProto' >> Map(
lambda x: pubsub.PubsubMessage(x, {})._to_proto_str())
pcoll.element_type = bytes
return pcoll.apply(
ExternalTransform(self.URN,
NamedTupleBasedPayloadBuilder(self.params),
self.expansion_service))
def test_generate_sequence_java_class_lookup_payload_builder(self):
port = os.environ.get('EXPANSION_PORT')
address = 'localhost:%s' % port
with TestPipeline() as p:
payload_builder = JavaClassLookupPayloadBuilder(
'org.apache.beam.sdk.io.GenerateSequence')
payload_builder.with_constructor_method('from', 1)
payload_builder.add_builder_method('to', 10)
res = (p
| ExternalTransform(
None, payload_builder, expansion_service=address))
assert_that(res, equal_to(list(range(1, 10))))
def expand(self, pbegin):
return (pbegin | ExternalTransform(
self.URN,
NamedTupleBasedPayloadBuilder(self.params),
self.expansion_service,
) | ParDo(_JsonStringToDictionaries()))