def test_generate_sequence_java_external_transform(self):
port = os.environ.get('EXPANSION_PORT')
address = 'localhost:%s' % port
with TestPipeline() as p:
java_transform = JavaExternalTransform(
'org.apache.beam.sdk.io.GenerateSequence',
expansion_service=address)
# We have to use 'getattr' below for builder method 'from' of Java
# 'GenerateSequence' class since 'from' is a reserved keyword for Python.
java_transform = getattr(java_transform, 'from')(1).to(10)
res = (p | java_transform)
assert_that(res, equal_to(list(range(1, 10))))
def run(output_path, expansion_service_port, pipeline_args):
pipeline_options = PipelineOptions(pipeline_args)
with beam.Pipeline(options=pipeline_options) as p:
DataConfig = typing.NamedTuple(
'DataConfig', [('prefix', str), ('length', int), ('suffix', str)])
data_config = DataConfig(prefix='start', length=20, suffix='end')
java_transform = JavaExternalTransform(
'org.apache.beam.examples.multilanguage.JavaDataGenerator',
expansion_service=('localhost:%s' % expansion_service_port)).create(np.int32(100)).withDataConfig(data_config)
data = p | 'Generate' >> java_transform
data | 'Write' >> WriteToText(output_path)
def test_implicit_builder_with_constructor(self):
constructor_transform = (JavaExternalTransform('org.pkg.MyTransform')(
'abc').withIntProperty(5))
payload_bytes = constructor_transform._payload_builder.payload()
payload_from_bytes = proto_utils.parse_Bytes(payload_bytes,
JavaClassLookupPayload)
self.assertEqual('org.pkg.MyTransform', payload_from_bytes.class_name)
self._verify_row(payload_from_bytes.constructor_schema,
payload_from_bytes.constructor_payload,
{'ignore0': 'abc'})
builder_method = payload_from_bytes.builder_methods[0]
self.assertEqual('withIntProperty', builder_method.name)
self._verify_row(builder_method.schema, builder_method.payload,
{'ignore0': 5})
def test_implicit_builder_with_constructor_method(self):
constructor_transform = JavaExternalTransform(
'org.pkg.MyTransform').of(str_field='abc').withProperty(
int_field=1234).build()
payload_bytes = constructor_transform._payload_builder.payload()
payload_from_bytes = proto_utils.parse_Bytes(payload_bytes,
JavaClassLookupPayload)
self.assertEqual('of', payload_from_bytes.constructor_method)
self._verify_row(payload_from_bytes.constructor_schema,
payload_from_bytes.constructor_payload,
{'str_field': 'abc'})
with_property_method = payload_from_bytes.builder_methods[0]
self.assertEqual('withProperty', with_property_method.name)
self._verify_row(with_property_method.schema,
with_property_method.payload, {'int_field': 1234})
build_method = payload_from_bytes.builder_methods[1]
self.assertEqual('build', build_method.name)
self._verify_row(build_method.schema, build_method.payload, {})