def test_glob_filter(self):
with FullyQualifiedNamedTransform.with_filter('*'):
self.assertIs(
FullyQualifiedNamedTransform._resolve('apache_beam.Row'), beam.Row)
with FullyQualifiedNamedTransform.with_filter('apache_beam.*'):
self.assertIs(
FullyQualifiedNamedTransform._resolve('apache_beam.Row'), beam.Row)
with FullyQualifiedNamedTransform.with_filter('apache_beam.foo.*'):
with self.assertRaises(ValueError):
FullyQualifiedNamedTransform._resolve('apache_beam.Row')
def test_resolve(self):
# test _resolve with the module that is not exposed to the top level
with FullyQualifiedNamedTransform.with_filter('*'):
dataframe_transform = FullyQualifiedNamedTransform._resolve(
'apache_beam.dataframe.transforms.DataframeTransform')
from apache_beam.dataframe.transforms import DataframeTransform
self.assertIs(dataframe_transform, DataframeTransform)
# test _resolve with the module that will never be exposed
# to the top level in the future
with FullyQualifiedNamedTransform.with_filter('*'):
argument_placeholder = FullyQualifiedNamedTransform._resolve(
'apache_beam.internal.util.ArgumentPlaceholder')
from apache_beam.internal.util import ArgumentPlaceholder
self.assertIs(argument_placeholder, ArgumentPlaceholder)
def test_resolve_by_path_segment(self, mock_import_module):
mock_import_module.return_value = None
with FullyQualifiedNamedTransform.with_filter('*'):
FullyQualifiedNamedTransform._resolve('a.b.c.d')
mock_import_module.assert_has_calls(
[call('a'), call('a.b'),
call('a.b.c'),
call('a.b.c.d')])
def test_static_constructor(self):
with FullyQualifiedNamedTransform.with_filter('*'):
with beam.Pipeline() as p:
assert_that(
p | beam.Create(['a', 'b', 'c'])
| FullyQualifiedNamedTransform(
'apache_beam.transforms.fully_qualified_named_transform_test'
'._TestTransform.create', ('x', ), {'suffix': 'y'}),
equal_to(['xay', 'xby', 'xcy']))
def test_as_external_transform_no_kwargs(self):
with FullyQualifiedNamedTransform.with_filter('*'):
with beam.Pipeline() as p:
assert_that(
p
| beam.Create(['a', 'b', 'c'])
| beam.ExternalTransform(
PYTHON_FULLY_QUALIFIED_NAMED_TRANSFORM_URN,
ImplicitSchemaPayloadBuilder({
'constructor': 'apache_beam.transforms'
'.fully_qualified_named_transform_test._TestTransform',
'args': beam.Row(arg0='x', arg1='y'),
}),
expansion_service.ExpansionServiceServicer()),
equal_to(['xay', 'xby', 'xcy']))
def test_callable_transform(self):
with FullyQualifiedNamedTransform.with_filter('*'):
with beam.Pipeline() as p:
assert_that(
p | beam.Create(['a', 'b', 'c'])
| FullyQualifiedNamedTransform(
'__callable__', # the next argument is a callable to be applied
(
python_callable.PythonCallableWithSource("""
def func(pcoll, x):
return pcoll | beam.Map(lambda e: e + x)
"""),
'x' # arguments passed to the callable
),
{}),
equal_to(['ax', 'bx', 'cx']))
def test_constructor_transform(self):
with FullyQualifiedNamedTransform.with_filter('*'):
with beam.Pipeline() as p:
assert_that(
p | beam.Create(['a', 'b', 'c'])
| FullyQualifiedNamedTransform(
'__constructor__', # the next argument constructs a PTransform
(),
{
'source': python_callable.PythonCallableWithSource("""
class MyTransform(beam.PTransform):
def __init__(self, x):
self._x = x
def expand(self, pcoll):
return pcoll | beam.Map(lambda e: e + self._x)
"""),
'x': 'x' # arguments passed to the above constructor
}
),
equal_to(['ax', 'bx', 'cx']))
