def test_resolve_by_path_segment(self, mock_import_module): mock_import_module.return_value = None with FullyQualifiedNamedTransform.with_filter('*'): FullyQualifiedNamedTransform._resolve('a.b.c.d') mock_import_module.assert_has_calls( [call('a'), call('a.b'), call('a.b.c'), call('a.b.c.d')])
def test_static_constructor(self): with FullyQualifiedNamedTransform.with_filter('*'): with beam.Pipeline() as p: assert_that( p | beam.Create(['a', 'b', 'c']) | FullyQualifiedNamedTransform( 'apache_beam.transforms.fully_qualified_named_transform_test' '._TestTransform.create', ('x', ), {'suffix': 'y'}), equal_to(['xay', 'xby', 'xcy']))
def test_resolve(self): # test _resolve with the module that is not exposed to the top level with FullyQualifiedNamedTransform.with_filter('*'): dataframe_transform = FullyQualifiedNamedTransform._resolve( 'apache_beam.dataframe.transforms.DataframeTransform') from apache_beam.dataframe.transforms import DataframeTransform self.assertIs(dataframe_transform, DataframeTransform) # test _resolve with the module that will never be exposed # to the top level in the future with FullyQualifiedNamedTransform.with_filter('*'): argument_placeholder = FullyQualifiedNamedTransform._resolve( 'apache_beam.internal.util.ArgumentPlaceholder') from apache_beam.internal.util import ArgumentPlaceholder self.assertIs(argument_placeholder, ArgumentPlaceholder)
def test_callable_transform(self): with FullyQualifiedNamedTransform.with_filter('*'): with beam.Pipeline() as p: assert_that( p | beam.Create(['a', 'b', 'c']) | FullyQualifiedNamedTransform( '__callable__', # the next argument is a callable to be applied ( python_callable.PythonCallableWithSource(""" def func(pcoll, x): return pcoll | beam.Map(lambda e: e + x) """), 'x' # arguments passed to the callable ), {}), equal_to(['ax', 'bx', 'cx']))
def test_constructor_transform(self): with FullyQualifiedNamedTransform.with_filter('*'): with beam.Pipeline() as p: assert_that( p | beam.Create(['a', 'b', 'c']) | FullyQualifiedNamedTransform( '__constructor__', # the next argument constructs a PTransform (), { 'source': python_callable.PythonCallableWithSource(""" class MyTransform(beam.PTransform): def __init__(self, x): self._x = x def expand(self, pcoll): return pcoll | beam.Map(lambda e: e + self._x) """), 'x': 'x' # arguments passed to the above constructor } ), equal_to(['ax', 'bx', 'cx']))
def test_glob_filter(self): with FullyQualifiedNamedTransform.with_filter('*'): self.assertIs( FullyQualifiedNamedTransform._resolve('apache_beam.Row'), beam.Row) with FullyQualifiedNamedTransform.with_filter('apache_beam.*'): self.assertIs( FullyQualifiedNamedTransform._resolve('apache_beam.Row'), beam.Row) with FullyQualifiedNamedTransform.with_filter('apache_beam.foo.*'): with self.assertRaises(ValueError): FullyQualifiedNamedTransform._resolve('apache_beam.Row')
def test_as_external_transform_no_kwargs(self): with FullyQualifiedNamedTransform.with_filter('*'): with beam.Pipeline() as p: assert_that( p | beam.Create(['a', 'b', 'c']) | beam.ExternalTransform( PYTHON_FULLY_QUALIFIED_NAMED_TRANSFORM_URN, ImplicitSchemaPayloadBuilder({ 'constructor': 'apache_beam.transforms' '.fully_qualified_named_transform_test._TestTransform', 'args': beam.Row(arg0='x', arg1='y'), }), expansion_service.ExpansionServiceServicer()), equal_to(['xay', 'xby', 'xcy']))