def normalize(x, none_as_type=False):
# None is inconsistantly used for Any, unknown, or NoneType.
if none_as_type and x is None:
return type(None)
elif x in _KNOWN_PRIMITIVE_TYPES:
return _KNOWN_PRIMITIVE_TYPES[x]
elif getattr(x, '__module__', None) == 'typing':
# Avoid circular imports
from apache_beam.typehints import native_type_compatibility
beam_type = native_type_compatibility.convert_to_beam_type(x)
if beam_type != x:
# We were able to do the conversion.
return beam_type
else:
# It might be a compatible type we don't understand.
return Any
return x
def test_convert_to_beam_type(self):
test_cases = [
('raw bytes', bytes, bytes),
('raw int', int, int),
('raw float', float, float),
('any', typing.Any, typehints.Any),
('simple dict', typing.Dict[bytes, int],
typehints.Dict[bytes, int]),
('simple list', typing.List[int], typehints.List[int]),
('simple optional', typing.Optional[int], typehints.Optional[int]),
('simple set', typing.Set[float], typehints.Set[float]),
('simple unary tuple', typing.Tuple[bytes],
typehints.Tuple[bytes]),
('simple union', typing.Union[int, bytes, float],
typehints.Union[int, bytes, float]),
('namedtuple', _TestNamedTuple, typehints.Any),
('test class', _TestClass, _TestClass),
('test class in list', typing.List[_TestClass],
typehints.List[_TestClass]),
('complex tuple', typing.Tuple[bytes, typing.List[typing.Tuple[
bytes, typing.Union[int, bytes, float]]]],
typehints.Tuple[bytes, typehints.List[typehints.Tuple[
bytes, typehints.Union[int, bytes, float]]]]),
('flat alias', _TestFlatAlias, typehints.Tuple[bytes, float]),
('nested alias', _TestNestedAlias,
typehints.List[typehints.Tuple[bytes, float]]),
('complex dict',
typing.Dict[bytes, typing.List[typing.Tuple[bytes, _TestClass]]],
typehints.Dict[bytes, typehints.List[typehints.Tuple[
bytes, _TestClass]]])
]
for test_case in test_cases:
# Unlike typing types, Beam types are guaranteed to compare equal.
description = test_case[0]
typing_type = test_case[1]
beam_type = test_case[2]
self.assertEqual(
native_type_compatibility.convert_to_beam_type(typing_type),
beam_type, description)
def test_generator_converted_to_iterator(self):
self.assertEqual(
typehints.Iterator[int],
convert_to_beam_type(typing.Generator[int, None, None]))
def with_output_types(*return_type_hint, **kwargs):
"""A decorator that type-checks defined type-hints for return values(s).
This decorator will type-check the return value(s) of the decorated function.
Only a single type-hint is accepted to specify the return type of the return
value. If the function to be decorated has multiple return values, then one
should use: ``Tuple[type_1, type_2]`` to annotate the types of the return
values.
If the ultimate return value for the function violates the specified type-hint
a :class:`TypeCheckError` will be raised detailing the type-constraint
violation.
This decorator is intended to be used like:
.. testcode::
from apache_beam.typehints import with_output_types
from apache_beam.typehints import Set
class Coordinate(object):
def __init__(self, x, y):
self.x = x
self.y = y
@with_output_types(Set[Coordinate])
def parse_ints(ints):
return {Coordinate(i, i) for i in ints}
Or with a simple type-hint:
.. testcode::
from apache_beam.typehints import with_output_types
@with_output_types(bool)
def negate(p):
return not p if p else p
Args:
*return_type_hint: A type-hint specifying the proper return type of the
function. This argument should either be a built-in Python type or an
instance of a :class:`~apache_beam.typehints.typehints.TypeConstraint`
created by 'indexing' a
:class:`~apache_beam.typehints.typehints.CompositeTypeHint`.
**kwargs: Not used.
Raises:
:class:`~exceptions.ValueError`: If any kwarg parameters are passed in,
or the length of **return_type_hint** is greater than ``1``. Or if the
inner wrapper function isn't passed a function object.
:class:`TypeCheckError`: If the **return_type_hint** object is
in invalid type-hint.
Returns:
The original function decorated such that it enforces type-hint constraints
for all return values.
"""
if kwargs:
raise ValueError("All arguments for the 'returns' decorator must be "
"positional arguments.")
if len(return_type_hint) != 1:
raise ValueError("'returns' accepts only a single positional argument. In "
"order to specify multiple return types, use the 'Tuple' "
"type-hint.")
return_type_hint = native_type_compatibility.convert_to_beam_type(
return_type_hint[0])
validate_composite_type_param(
return_type_hint,
error_msg_prefix='All type hint arguments'
)
def annotate(f):
get_type_hints(f).set_output_types(return_type_hint)
return f
return annotate
def test_convert_to_beam_type(self):
test_cases = [
('raw bytes', bytes, bytes),
('raw int', int, int),
('raw float', float, float),
('any', typing.Any, typehints.Any),
('simple dict', typing.Dict[bytes, int], typehints.Dict[bytes,
int]),
('simple list', typing.List[int], typehints.List[int]),
('simple iterable', typing.Iterable[int], typehints.Iterable[int]),
('simple optional', typing.Optional[int], typehints.Optional[int]),
('simple set', typing.Set[float], typehints.Set[float]),
('simple unary tuple', typing.Tuple[bytes],
typehints.Tuple[bytes]),
('simple union', typing.Union[int, bytes,
float], typehints.Union[int, bytes,
float]),
('namedtuple', _TestNamedTuple, _TestNamedTuple),
('test class', _TestClass, _TestClass),
('test class in list', typing.List[_TestClass],
typehints.List[_TestClass]),
('complex tuple',
typing.Tuple[bytes,
typing.List[typing.Tuple[bytes,
typing.Union[int, bytes,
float]]]],
typehints.Tuple[bytes, typehints.List[typehints.Tuple[
bytes, typehints.Union[int, bytes, float]]]]),
# TODO(BEAM-7713): This case seems to fail on Py3.5.2 but not 3.5.4.
('arbitrary-length tuple', typing.Tuple[int, ...],
typehints.Tuple[int, ...]) if sys.version_info >=
(3, 5, 4) else None,
('flat alias', _TestFlatAlias,
typehints.Tuple[bytes, float]), # type: ignore[misc]
('nested alias', _TestNestedAlias,
typehints.List[typehints.Tuple[bytes, float]]),
('complex dict',
typing.Dict[bytes, typing.List[typing.Tuple[bytes, _TestClass]]],
typehints.Dict[bytes,
typehints.List[typehints.Tuple[bytes,
_TestClass]]]),
('type var', typing.TypeVar('T'), typehints.TypeVariable('T')),
('nested type var', typing.Tuple[typing.TypeVar('K'),
typing.TypeVar('V')],
typehints.Tuple[typehints.TypeVariable('K'),
typehints.TypeVariable('V')]),
('iterator', typing.Iterator[typing.Any],
typehints.Iterator[typehints.Any]),
]
for test_case in test_cases:
if test_case is None:
continue
# Unlike typing types, Beam types are guaranteed to compare equal.
description = test_case[0]
typing_type = test_case[1]
expected_beam_type = test_case[2]
converted_beam_type = convert_to_beam_type(typing_type)
self.assertEqual(converted_beam_type, expected_beam_type,
description)
converted_typing_type = convert_to_typing_type(converted_beam_type)
self.assertEqual(converted_typing_type, typing_type, description)
def assertNotCompatible(self, base, sub): # pylint: disable=invalid-name
base, sub = native_type_compatibility.convert_to_beam_type([base, sub])
self.assertFalse(
is_consistent_with(sub, base),
'%s is consistent with %s' % (sub, base))
def with_output_types(*return_type_hint, **kwargs):
"""A decorator that type-checks defined type-hints for return values(s).
This decorator will type-check the return value(s) of the decorated function.
Only a single type-hint is accepted to specify the return type of the return
value. If the function to be decorated has multiple return values, then one
should use: ``Tuple[type_1, type_2]`` to annotate the types of the return
values.
If the ultimate return value for the function violates the specified type-hint
a :class:`TypeCheckError` will be raised detailing the type-constraint
violation.
This decorator is intended to be used like:
.. testcode::
from apache_beam.typehints import with_output_types
from apache_beam.typehints import Set
class Coordinate(object):
def __init__(self, x, y):
self.x = x
self.y = y
@with_output_types(Set[Coordinate])
def parse_ints(ints):
return {Coordinate(i, i) for i in ints}
Or with a simple type-hint:
.. testcode::
from apache_beam.typehints import with_output_types
@with_output_types(bool)
def negate(p):
return not p if p else p
Args:
*return_type_hint: A type-hint specifying the proper return type of the
function. This argument should either be a built-in Python type or an
instance of a :class:`~apache_beam.typehints.typehints.TypeConstraint`
created by 'indexing' a
:class:`~apache_beam.typehints.typehints.CompositeTypeHint`.
**kwargs: Not used.
Raises:
:class:`~exceptions.ValueError`: If any kwarg parameters are passed in,
or the length of **return_type_hint** is greater than ``1``. Or if the
inner wrapper function isn't passed a function object.
:class:`TypeCheckError`: If the **return_type_hint** object is
in invalid type-hint.
Returns:
The original function decorated such that it enforces type-hint constraints
for all return values.
"""
if kwargs:
raise ValueError("All arguments for the 'returns' decorator must be "
"positional arguments.")
if len(return_type_hint) != 1:
raise ValueError("'returns' accepts only a single positional argument. In "
"order to specify multiple return types, use the 'Tuple' "
"type-hint.")
return_type_hint = native_type_compatibility.convert_to_beam_type(
return_type_hint[0])
validate_composite_type_param(
return_type_hint,
error_msg_prefix='All type hint arguments'
)
def annotate(f):
get_type_hints(f).set_output_types(return_type_hint)
return f
return annotate
def test_forward_reference(self):
self.assertEqual(typehints.Any, convert_to_beam_type('int'))
self.assertEqual(typehints.Any,
convert_to_beam_type('typing.List[int]'))
self.assertEqual(typehints.List[typehints.Any],
convert_to_beam_type(typing.List['int']))
def test_newtype(self):
self.assertEqual(typehints.Any,
convert_to_beam_type(typing.NewType('Number', int)))
def with_output_types(*return_type_hint, **kwargs):
"""A decorator that type-checks defined type-hints for return values(s).
This decorator will type-check the return value(s) of the decorated function.
Only a single type-hint is accepted to specify the return type of the return
value. If the function to be decorated has multiple return values, then one
should use: 'Tuple[type_1, type_2]' to annotate the types of the return
values.
If the ultimate return value for the function violates the specified type-hint
a TypeCheckError will be raised detailing the type-constraint violation.
This decorator is intended to be used like::
* @with_output_types(Set[Coordinate])
def parse_ints(ints):
....
return [Coordinate.from_int(i) for i in ints]
Or with a simple type-hint::
* @with_output_types(bool)
def negate(p):
return not p if p else p
Args:
*return_type_hint: A type-hint specifying the proper return type of the
function. This argument should either be a built-in Python type or an
instance of a 'TypeConstraint' created by 'indexing' a
'CompositeTypeHint'.
**kwargs: Not used.
Raises:
ValueError: If any kwarg parameters are passed in, or the length of
'return_type_hint' is greater than 1. Or if the inner wrapper function
isn't passed a function object.
TypeCheckError: If the 'return_type_hint' object is in invalid type-hint.
Returns:
The original function decorated such that it enforces type-hint constraints
for all return values.
"""
if kwargs:
raise ValueError("All arguments for the 'returns' decorator must be "
"positional arguments.")
if len(return_type_hint) != 1:
raise ValueError(
"'returns' accepts only a single positional argument. In "
"order to specify multiple return types, use the 'Tuple' "
"type-hint.")
return_type_hint = native_type_compatibility.convert_to_beam_type(
return_type_hint[0])
validate_composite_type_param(return_type_hint,
error_msg_prefix='All type hint arguments')
def annotate(f):
get_type_hints(f).set_output_types(return_type_hint)
return f
return annotate
def test_string_literal_converted_to_any(self):
self.assertEqual(typehints.Any,
convert_to_beam_type('typing.List[int]'))
def test_convert_to_beam_type(self):
test_cases = [
('raw bytes', bytes, bytes),
('raw int', int, int),
('raw float', float, float),
('any', typing.Any, typehints.Any),
('simple dict', typing.Dict[bytes, int], typehints.Dict[bytes,
int]),
('simple list', typing.List[int], typehints.List[int]),
('simple iterable', typing.Iterable[int], typehints.Iterable[int]),
('simple optional', typing.Optional[int], typehints.Optional[int]),
('simple set', typing.Set[float], typehints.Set[float]),
('simple frozenset', typing.FrozenSet[float],
typehints.FrozenSet[float]),
('simple unary tuple', typing.Tuple[bytes],
typehints.Tuple[bytes]),
('simple union', typing.Union[int, bytes,
float], typehints.Union[int, bytes,
float]),
('namedtuple', _TestNamedTuple, _TestNamedTuple),
('test class', _TestClass, _TestClass),
('test class in list', typing.List[_TestClass],
typehints.List[_TestClass]),
('generic bare', _TestGeneric, _TestGeneric),
('generic subscripted', _TestGeneric[int], _TestGeneric[int]),
('complex tuple',
typing.Tuple[bytes,
typing.List[typing.Tuple[bytes,
typing.Union[int, bytes,
float]]]],
typehints.Tuple[bytes, typehints.List[typehints.Tuple[
bytes, typehints.Union[int, bytes, float]]]]),
('arbitrary-length tuple', typing.Tuple[int, ...],
typehints.Tuple[int, ...]),
('flat alias', _TestFlatAlias,
typehints.Tuple[bytes, float]), # type: ignore[misc]
('nested alias', _TestNestedAlias,
typehints.List[typehints.Tuple[bytes, float]]),
('complex dict',
typing.Dict[bytes, typing.List[typing.Tuple[bytes, _TestClass]]],
typehints.Dict[bytes,
typehints.List[typehints.Tuple[bytes,
_TestClass]]]),
('type var', typing.TypeVar('T'), typehints.TypeVariable('T')),
('nested type var', typing.Tuple[typing.TypeVar('K'),
typing.TypeVar('V')],
typehints.Tuple[typehints.TypeVariable('K'),
typehints.TypeVariable('V')]),
('iterator', typing.Iterator[typing.Any],
typehints.Iterator[typehints.Any]),
('nested generic bare', typing.List[_TestGeneric],
typehints.List[_TestGeneric]),
('nested generic subscripted', typing.List[_TestGeneric[int]],
typehints.List[_TestGeneric[int]]),
]
for test_case in test_cases:
if test_case is None:
continue
# Unlike typing types, Beam types are guaranteed to compare equal.
description = test_case[0]
typing_type = test_case[1]
expected_beam_type = test_case[2]
converted_beam_type = convert_to_beam_type(typing_type)
self.assertEqual(converted_beam_type, expected_beam_type,
description)
converted_typing_type = convert_to_typing_type(converted_beam_type)
self.assertEqual(converted_typing_type, typing_type, description)
def element_type_from_coder_id(self, coder_id):
if self.use_fake_coders or coder_id not in self.coders:
return pickler.loads(coder_id)
else:
return native_type_compatibility.convert_to_beam_type(
self.coders[coder_id].to_type_hint())
