def test_generate_sequence_java_external_transform(self): port = os.environ.get('EXPANSION_PORT') address = 'localhost:%s' % port with TestPipeline() as p: java_transform = JavaExternalTransform( 'org.apache.beam.sdk.io.GenerateSequence', expansion_service=address) # We have to use 'getattr' below for builder method 'from' of Java # 'GenerateSequence' class since 'from' is a reserved keyword for Python. java_transform = getattr(java_transform, 'from')(1).to(10) res = (p | java_transform) assert_that(res, equal_to(list(range(1, 10))))
def run(output_path, expansion_service_port, pipeline_args): pipeline_options = PipelineOptions(pipeline_args) with beam.Pipeline(options=pipeline_options) as p: DataConfig = typing.NamedTuple( 'DataConfig', [('prefix', str), ('length', int), ('suffix', str)]) data_config = DataConfig(prefix='start', length=20, suffix='end') java_transform = JavaExternalTransform( 'org.apache.beam.examples.multilanguage.JavaDataGenerator', expansion_service=('localhost:%s' % expansion_service_port)).create(np.int32(100)).withDataConfig(data_config) data = p | 'Generate' >> java_transform data | 'Write' >> WriteToText(output_path)
def test_implicit_builder_with_constructor(self): constructor_transform = (JavaExternalTransform('org.pkg.MyTransform')( 'abc').withIntProperty(5)) payload_bytes = constructor_transform._payload_builder.payload() payload_from_bytes = proto_utils.parse_Bytes(payload_bytes, JavaClassLookupPayload) self.assertEqual('org.pkg.MyTransform', payload_from_bytes.class_name) self._verify_row(payload_from_bytes.constructor_schema, payload_from_bytes.constructor_payload, {'ignore0': 'abc'}) builder_method = payload_from_bytes.builder_methods[0] self.assertEqual('withIntProperty', builder_method.name) self._verify_row(builder_method.schema, builder_method.payload, {'ignore0': 5})
def test_implicit_builder_with_constructor_method(self): constructor_transform = JavaExternalTransform( 'org.pkg.MyTransform').of(str_field='abc').withProperty( int_field=1234).build() payload_bytes = constructor_transform._payload_builder.payload() payload_from_bytes = proto_utils.parse_Bytes(payload_bytes, JavaClassLookupPayload) self.assertEqual('of', payload_from_bytes.constructor_method) self._verify_row(payload_from_bytes.constructor_schema, payload_from_bytes.constructor_payload, {'str_field': 'abc'}) with_property_method = payload_from_bytes.builder_methods[0] self.assertEqual('withProperty', with_property_method.name) self._verify_row(with_property_method.schema, with_property_method.payload, {'int_field': 1234}) build_method = payload_from_bytes.builder_methods[1] self.assertEqual('build', build_method.name) self._verify_row(build_method.schema, build_method.payload, {})