def test_eq_with_different_name_checks(self):
self.assertEqual(
typehints.Set[typehints.TypeVariable('T', True)],
typehints.Set[typehints.TypeVariable('T2', False)])
self.assertEqual(
typehints.Set[typehints.TypeVariable('T', False)],
typehints.Set[typehints.TypeVariable('T2', True)])
def test_convert_bare_types(self):
# Conversions for unsubscripted types that have implicit subscripts.
test_cases = [
('bare list', typing.List,
typehints.List[typehints.TypeVariable('T')]),
('bare dict', typing.Dict,
typehints.Dict[typehints.TypeVariable('KT'),
typehints.TypeVariable('VT')]),
('bare tuple', typing.Tuple,
typehints.Tuple[typehints.TypeVariable('T'), ...]),
('bare set', typing.Set,
typehints.Set[typehints.TypeVariable('T')]),
('bare frozenset', typing.FrozenSet,
typehints.FrozenSet[typehints.TypeVariable(
'T', use_name_in_eq=False)]),
('bare iterator', typing.Iterator,
typehints.Iterator[typehints.TypeVariable('T_co')]),
('bare iterable', typing.Iterable,
typehints.Iterable[typehints.TypeVariable('T_co')]),
('nested bare', typing.Tuple[typing.Iterator], typehints.Tuple[
typehints.Iterator[typehints.TypeVariable('T_co')]]),
]
if sys.version_info >= (3, 7):
test_cases += [
('bare generator', typing.Generator,
typehints.Generator[typehints.TypeVariable('T_co')]),
]
for test_case in test_cases:
description = test_case[0]
typing_type = test_case[1]
expected_beam_type = test_case[2]
converted_beam_type = convert_to_beam_type(typing_type)
self.assertEqual(expected_beam_type, converted_beam_type,
description)
def test_match_type_variables(self):
S = typehints.TypeVariable('S') # pylint: disable=invalid-name
T = typehints.TypeVariable('T') # pylint: disable=invalid-name
hint = typehints.Dict[S, T]
self.assertEqual({
S: int,
T: str
}, hint.match_type_variables(typehints.Dict[int, str]))
def test_convert_to_beam_type(self):
test_cases = [
('raw bytes', bytes, bytes),
('raw int', int, int),
('raw float', float, float),
('any', typing.Any, typehints.Any),
('simple dict', typing.Dict[bytes, int],
typehints.Dict[bytes, int]),
('simple list', typing.List[int], typehints.List[int]),
('simple iterable', typing.Iterable[int], typehints.Iterable[int]),
('simple optional', typing.Optional[int], typehints.Optional[int]),
('simple set', typing.Set[float], typehints.Set[float]),
('simple frozenset',
typing.FrozenSet[float],
typehints.FrozenSet[float]),
('simple unary tuple', typing.Tuple[bytes],
typehints.Tuple[bytes]),
('simple union', typing.Union[int, bytes, float],
typehints.Union[int, bytes, float]),
('namedtuple', _TestNamedTuple, _TestNamedTuple),
('test class', _TestClass, _TestClass),
('test class in list', typing.List[_TestClass],
typehints.List[_TestClass]),
('complex tuple', typing.Tuple[bytes, typing.List[typing.Tuple[
bytes, typing.Union[int, bytes, float]]]],
typehints.Tuple[bytes, typehints.List[typehints.Tuple[
bytes, typehints.Union[int, bytes, float]]]]),
# TODO(BEAM-7713): This case seems to fail on Py3.5.2 but not 3.5.4.
('arbitrary-length tuple', typing.Tuple[int, ...],
typehints.Tuple[int, ...])
if sys.version_info >= (3, 5, 4) else None,
('flat alias', _TestFlatAlias, typehints.Tuple[bytes, float]), # type: ignore[misc]
('nested alias', _TestNestedAlias,
typehints.List[typehints.Tuple[bytes, float]]),
('complex dict',
typing.Dict[bytes, typing.List[typing.Tuple[bytes, _TestClass]]],
typehints.Dict[bytes, typehints.List[typehints.Tuple[
bytes, _TestClass]]]),
('type var', typing.TypeVar('T'), typehints.TypeVariable('T')),
('nested type var',
typing.Tuple[typing.TypeVar('K'), typing.TypeVar('V')],
typehints.Tuple[typehints.TypeVariable('K'),
typehints.TypeVariable('V')]),
('iterator', typing.Iterator[typing.Any],
typehints.Iterator[typehints.Any]),
]
for test_case in test_cases:
if test_case is None:
continue
# Unlike typing types, Beam types are guaranteed to compare equal.
description = test_case[0]
typing_type = test_case[1]
expected_beam_type = test_case[2]
converted_beam_type = convert_to_beam_type(typing_type)
self.assertEqual(converted_beam_type, expected_beam_type, description)
converted_typing_type = convert_to_typing_type(converted_beam_type)
self.assertEqual(converted_typing_type, typing_type, description)
def test_getitem(self):
K = typehints.TypeVariable('K') # pylint: disable=invalid-name
T = typehints.TypeVariable('T') # pylint: disable=invalid-name
with self.assertRaisesRegex(TypeError,
'Parameter to ShardedKeyType hint.*'):
_ = ShardedKeyType[K, T]
with self.assertRaisesRegex(TypeError,
'Parameter to ShardedKeyType hint.*'):
_ = ShardedKeyType[(K, T)]
def test_convert_to_beam_type(self):
test_cases = [
('raw bytes', bytes, bytes),
('raw int', int, int),
('raw float', float, float),
('any', typing.Any, typehints.Any),
('simple dict', typing.Dict[bytes, int], typehints.Dict[bytes,
int]),
('simple list', typing.List[int], typehints.List[int]),
('simple iterable', typing.Iterable[int], typehints.Iterable[int]),
('simple optional', typing.Optional[int], typehints.Optional[int]),
('simple set', typing.Set[float], typehints.Set[float]),
('simple unary tuple', typing.Tuple[bytes],
typehints.Tuple[bytes]),
('simple union', typing.Union[int, bytes,
float], typehints.Union[int, bytes,
float]),
('namedtuple', _TestNamedTuple, _TestNamedTuple),
('test class', _TestClass, _TestClass),
('test class in list', typing.List[_TestClass],
typehints.List[_TestClass]),
('complex tuple',
typing.Tuple[bytes,
typing.List[typing.Tuple[bytes,
typing.Union[int, bytes,
float]]]],
typehints.Tuple[bytes, typehints.List[typehints.Tuple[
bytes, typehints.Union[int, bytes, float]]]]),
# TODO(BEAM-7713): This case seems to fail on Py3.5.2 but not 3.5.4.
# ('arbitrary-length tuple', typing.Tuple[int, ...],
# typehints.Tuple[int, ...]),
('flat alias', _TestFlatAlias, typehints.Tuple[bytes, float]),
('nested alias', _TestNestedAlias,
typehints.List[typehints.Tuple[bytes, float]]),
('complex dict',
typing.Dict[bytes, typing.List[typing.Tuple[bytes, _TestClass]]],
typehints.Dict[bytes,
typehints.List[typehints.Tuple[bytes,
_TestClass]]]),
('type var', typing.TypeVar('T'), typehints.TypeVariable('T')),
('nested type var', typing.Tuple[typing.TypeVar('K'),
typing.TypeVar('V')],
typehints.Tuple[typehints.TypeVariable('K'),
typehints.TypeVariable('V')]),
]
for test_case in test_cases:
# Unlike typing types, Beam types are guaranteed to compare equal.
description = test_case[0]
typing_type = test_case[1]
beam_type = test_case[2]
self.assertEqual(
native_type_compatibility.convert_to_beam_type(typing_type),
beam_type, description)
def convert_to_beam_type(typ):
"""Convert a given typing type to a Beam type.
Args:
typ (`typing.Union[type, str]`): typing type or string literal representing
a type.
Returns:
type: The given type converted to a Beam type as far as we can do the
conversion.
Raises:
ValueError: The type was malformed.
"""
if isinstance(typ, typing.TypeVar):
# This is a special case, as it's not parameterized by types.
# Also, identity must be preserved through conversion (i.e. the same
# TypeVar instance must get converted into the same TypeVariable instance).
# A global cache should be OK as the number of distinct type variables
# is generally small.
if id(typ) not in _type_var_cache:
new_type_variable = typehints.TypeVariable(typ.__name__)
_type_var_cache[id(typ)] = new_type_variable
_type_var_cache[id(new_type_variable)] = typ
return _type_var_cache[id(typ)]
elif isinstance(typ, str):
# Special case for forward references.
# TODO(https://github.com/apache/beam/issues/19954): Currently unhandled.
_LOGGER.info('Converting string literal type hint to Any: "%s"', typ)
return typehints.Any
elif getattr(typ, '__module__', None) != 'typing':
# Only translate types from the typing module.
return typ
type_map = [
# TODO(https://github.com/apache/beam/issues/20076): Currently
# unsupported.
_TypeMapEntry(match=is_new_type, arity=0, beam_type=typehints.Any),
# TODO(https://github.com/apache/beam/issues/19954): Currently
# unsupported.
_TypeMapEntry(match=is_forward_ref, arity=0, beam_type=typehints.Any),
_TypeMapEntry(match=is_any, arity=0, beam_type=typehints.Any),
_TypeMapEntry(match=_match_issubclass(typing.Dict),
arity=2,
beam_type=typehints.Dict),
_TypeMapEntry(match=_match_is_exactly_iterable,
arity=1,
beam_type=typehints.Iterable),
_TypeMapEntry(match=_match_issubclass(typing.List),
arity=1,
beam_type=typehints.List),
_TypeMapEntry(match=_match_issubclass(typing.Set),
arity=1,
beam_type=typehints.Set),
_TypeMapEntry(match=_match_issubclass(typing.FrozenSet),
arity=1,
beam_type=typehints.FrozenSet),
# NamedTuple is a subclass of Tuple, but it needs special handling.
# We just convert it to Any for now.
# This MUST appear before the entry for the normal Tuple.
_TypeMapEntry(match=match_is_named_tuple,
arity=0,
beam_type=typehints.Any),
_TypeMapEntry(match=_match_issubclass(typing.Tuple),
arity=-1,
beam_type=typehints.Tuple),
_TypeMapEntry(match=_match_is_union,
arity=-1,
beam_type=typehints.Union),
_TypeMapEntry(match=_match_issubclass(typing.Generator),
arity=3,
beam_type=typehints.Generator),
_TypeMapEntry(match=_match_issubclass(typing.Iterator),
arity=1,
beam_type=typehints.Iterator),
]
# Find the first matching entry.
matched_entry = next((entry for entry in type_map if entry.match(typ)),
None)
if not matched_entry:
# Please add missing type support if you see this message.
_LOGGER.info('Using Any for unsupported type: %s', typ)
return typehints.Any
args = _get_args(typ)
len_args = len(args)
if len_args == 0 and len_args != matched_entry.arity:
arity = matched_entry.arity
# Handle unsubscripted types.
if _match_issubclass(typing.Tuple)(typ):
args = (typehints.TypeVariable('T'), Ellipsis)
elif _match_is_union(typ):
raise ValueError('Unsupported Union with no arguments.')
elif _match_issubclass(typing.Generator)(typ):
# Assume a simple generator.
args = (typehints.TypeVariable('T_co'), type(None), type(None))
elif _match_issubclass(typing.Dict)(typ):
args = (typehints.TypeVariable('KT'), typehints.TypeVariable('VT'))
elif (_match_issubclass(typing.Iterator)(typ)
or _match_is_exactly_iterable(typ)):
args = (typehints.TypeVariable('T_co'), )
else:
args = (typehints.TypeVariable('T'), ) * arity
elif matched_entry.arity == -1:
arity = len_args
else:
arity = matched_entry.arity
if len_args != arity:
raise ValueError(
'expecting type %s to have arity %d, had arity %d '
'instead' % (str(typ), arity, len_args))
typs = convert_to_beam_types(args)
if arity == 0:
# Nullary types (e.g. Any) don't accept empty tuples as arguments.
return matched_entry.beam_type
elif arity == 1:
# Unary types (e.g. Set) don't accept 1-tuples as arguments
return matched_entry.beam_type[typs[0]]
else:
return matched_entry.beam_type[tuple(typs)]
def test_match_type_variables(self):
K = typehints.TypeVariable('K') # pylint: disable=invalid-name
constraint = ShardedKeyType[K]
self.assertEqual({K: int},
constraint.match_type_variables(ShardedKeyType[int]))
def test_eq_without_name_check(self):
use_name_in_eq = False
self.assertEqual(
typehints.Set[typehints.TypeVariable('T', use_name_in_eq)],
typehints.Set[typehints.TypeVariable('T2', use_name_in_eq)])
def convert_to_beam_type(typ):
"""Convert a given typing type to a Beam type.
Args:
typ (type): typing type.
Returns:
type: The given type converted to a Beam type as far as we can do the
conversion.
Raises:
~exceptions.ValueError: The type was malformed.
"""
if isinstance(typ, typing.TypeVar):
# This is a special case, as it's not parameterized by types.
# Also, identity must be preserved through conversion (i.e. the same
# TypeVar instance must get converted into the same TypeVariable instance).
# A global cache should be OK as the number of distinct type variables
# is generally small.
if id(typ) not in _type_var_cache:
new_type_variable = typehints.TypeVariable(typ.__name__)
_type_var_cache[id(typ)] = new_type_variable
_type_var_cache[id(new_type_variable)] = typ
return _type_var_cache[id(typ)]
elif getattr(typ, '__module__', None) != 'typing':
# Only translate types from the typing module.
return typ
type_map = [
_TypeMapEntry(match=_match_same_type(typing.Any),
arity=0,
beam_type=typehints.Any),
_TypeMapEntry(match=_match_issubclass(typing.Dict),
arity=2,
beam_type=typehints.Dict),
_TypeMapEntry(match=_match_is_exactly_iterable,
arity=1,
beam_type=typehints.Iterable),
_TypeMapEntry(match=_match_issubclass(typing.List),
arity=1,
beam_type=typehints.List),
_TypeMapEntry(match=_match_issubclass(typing.Set),
arity=1,
beam_type=typehints.Set),
# NamedTuple is a subclass of Tuple, but it needs special handling.
# We just convert it to Any for now.
# This MUST appear before the entry for the normal Tuple.
_TypeMapEntry(match=_match_is_named_tuple,
arity=0,
beam_type=typehints.Any),
_TypeMapEntry(match=_match_issubclass(typing.Tuple),
arity=-1,
beam_type=typehints.Tuple),
_TypeMapEntry(match=_match_is_union,
arity=-1,
beam_type=typehints.Union),
_TypeMapEntry(match=_match_issubclass(typing.Generator),
arity=3,
beam_type=typehints.Generator),
_TypeMapEntry(match=_match_issubclass(typing.Iterator),
arity=1,
beam_type=typehints.Iterator),
]
# Find the first matching entry.
matched_entry = next((entry for entry in type_map if entry.match(typ)),
None)
if not matched_entry:
# No match: return original type.
return typ
if matched_entry.arity == -1:
arity = _len_arg(typ)
else:
arity = matched_entry.arity
if _len_arg(typ) != arity:
raise ValueError(
'expecting type %s to have arity %d, had arity %d '
'instead' % (str(typ), arity, _len_arg(typ)))
typs = [convert_to_beam_type(_get_arg(typ, i)) for i in range(arity)]
if arity == 0:
# Nullary types (e.g. Any) don't accept empty tuples as arguments.
return matched_entry.beam_type
elif arity == 1:
# Unary types (e.g. Set) don't accept 1-tuples as arguments
return matched_entry.beam_type[typs[0]]
else:
return matched_entry.beam_type[tuple(typs)]
