def test_eq_with_different_name_checks(self): self.assertEqual( typehints.Set[typehints.TypeVariable('T', True)], typehints.Set[typehints.TypeVariable('T2', False)]) self.assertEqual( typehints.Set[typehints.TypeVariable('T', False)], typehints.Set[typehints.TypeVariable('T2', True)])
def test_convert_bare_types(self): # Conversions for unsubscripted types that have implicit subscripts. test_cases = [ ('bare list', typing.List, typehints.List[typehints.TypeVariable('T')]), ('bare dict', typing.Dict, typehints.Dict[typehints.TypeVariable('KT'), typehints.TypeVariable('VT')]), ('bare tuple', typing.Tuple, typehints.Tuple[typehints.TypeVariable('T'), ...]), ('bare set', typing.Set, typehints.Set[typehints.TypeVariable('T')]), ('bare frozenset', typing.FrozenSet, typehints.FrozenSet[typehints.TypeVariable( 'T', use_name_in_eq=False)]), ('bare iterator', typing.Iterator, typehints.Iterator[typehints.TypeVariable('T_co')]), ('bare iterable', typing.Iterable, typehints.Iterable[typehints.TypeVariable('T_co')]), ('nested bare', typing.Tuple[typing.Iterator], typehints.Tuple[ typehints.Iterator[typehints.TypeVariable('T_co')]]), ] if sys.version_info >= (3, 7): test_cases += [ ('bare generator', typing.Generator, typehints.Generator[typehints.TypeVariable('T_co')]), ] for test_case in test_cases: description = test_case[0] typing_type = test_case[1] expected_beam_type = test_case[2] converted_beam_type = convert_to_beam_type(typing_type) self.assertEqual(expected_beam_type, converted_beam_type, description)
def test_match_type_variables(self): S = typehints.TypeVariable('S') # pylint: disable=invalid-name T = typehints.TypeVariable('T') # pylint: disable=invalid-name hint = typehints.Dict[S, T] self.assertEqual({ S: int, T: str }, hint.match_type_variables(typehints.Dict[int, str]))
def test_convert_to_beam_type(self): test_cases = [ ('raw bytes', bytes, bytes), ('raw int', int, int), ('raw float', float, float), ('any', typing.Any, typehints.Any), ('simple dict', typing.Dict[bytes, int], typehints.Dict[bytes, int]), ('simple list', typing.List[int], typehints.List[int]), ('simple iterable', typing.Iterable[int], typehints.Iterable[int]), ('simple optional', typing.Optional[int], typehints.Optional[int]), ('simple set', typing.Set[float], typehints.Set[float]), ('simple frozenset', typing.FrozenSet[float], typehints.FrozenSet[float]), ('simple unary tuple', typing.Tuple[bytes], typehints.Tuple[bytes]), ('simple union', typing.Union[int, bytes, float], typehints.Union[int, bytes, float]), ('namedtuple', _TestNamedTuple, _TestNamedTuple), ('test class', _TestClass, _TestClass), ('test class in list', typing.List[_TestClass], typehints.List[_TestClass]), ('complex tuple', typing.Tuple[bytes, typing.List[typing.Tuple[ bytes, typing.Union[int, bytes, float]]]], typehints.Tuple[bytes, typehints.List[typehints.Tuple[ bytes, typehints.Union[int, bytes, float]]]]), # TODO(BEAM-7713): This case seems to fail on Py3.5.2 but not 3.5.4. ('arbitrary-length tuple', typing.Tuple[int, ...], typehints.Tuple[int, ...]) if sys.version_info >= (3, 5, 4) else None, ('flat alias', _TestFlatAlias, typehints.Tuple[bytes, float]), # type: ignore[misc] ('nested alias', _TestNestedAlias, typehints.List[typehints.Tuple[bytes, float]]), ('complex dict', typing.Dict[bytes, typing.List[typing.Tuple[bytes, _TestClass]]], typehints.Dict[bytes, typehints.List[typehints.Tuple[ bytes, _TestClass]]]), ('type var', typing.TypeVar('T'), typehints.TypeVariable('T')), ('nested type var', typing.Tuple[typing.TypeVar('K'), typing.TypeVar('V')], typehints.Tuple[typehints.TypeVariable('K'), typehints.TypeVariable('V')]), ('iterator', typing.Iterator[typing.Any], typehints.Iterator[typehints.Any]), ] for test_case in test_cases: if test_case is None: continue # Unlike typing types, Beam types are guaranteed to compare equal. description = test_case[0] typing_type = test_case[1] expected_beam_type = test_case[2] converted_beam_type = convert_to_beam_type(typing_type) self.assertEqual(converted_beam_type, expected_beam_type, description) converted_typing_type = convert_to_typing_type(converted_beam_type) self.assertEqual(converted_typing_type, typing_type, description)
def test_getitem(self): K = typehints.TypeVariable('K') # pylint: disable=invalid-name T = typehints.TypeVariable('T') # pylint: disable=invalid-name with self.assertRaisesRegex(TypeError, 'Parameter to ShardedKeyType hint.*'): _ = ShardedKeyType[K, T] with self.assertRaisesRegex(TypeError, 'Parameter to ShardedKeyType hint.*'): _ = ShardedKeyType[(K, T)]
def test_convert_to_beam_type(self): test_cases = [ ('raw bytes', bytes, bytes), ('raw int', int, int), ('raw float', float, float), ('any', typing.Any, typehints.Any), ('simple dict', typing.Dict[bytes, int], typehints.Dict[bytes, int]), ('simple list', typing.List[int], typehints.List[int]), ('simple iterable', typing.Iterable[int], typehints.Iterable[int]), ('simple optional', typing.Optional[int], typehints.Optional[int]), ('simple set', typing.Set[float], typehints.Set[float]), ('simple unary tuple', typing.Tuple[bytes], typehints.Tuple[bytes]), ('simple union', typing.Union[int, bytes, float], typehints.Union[int, bytes, float]), ('namedtuple', _TestNamedTuple, _TestNamedTuple), ('test class', _TestClass, _TestClass), ('test class in list', typing.List[_TestClass], typehints.List[_TestClass]), ('complex tuple', typing.Tuple[bytes, typing.List[typing.Tuple[bytes, typing.Union[int, bytes, float]]]], typehints.Tuple[bytes, typehints.List[typehints.Tuple[ bytes, typehints.Union[int, bytes, float]]]]), # TODO(BEAM-7713): This case seems to fail on Py3.5.2 but not 3.5.4. # ('arbitrary-length tuple', typing.Tuple[int, ...], # typehints.Tuple[int, ...]), ('flat alias', _TestFlatAlias, typehints.Tuple[bytes, float]), ('nested alias', _TestNestedAlias, typehints.List[typehints.Tuple[bytes, float]]), ('complex dict', typing.Dict[bytes, typing.List[typing.Tuple[bytes, _TestClass]]], typehints.Dict[bytes, typehints.List[typehints.Tuple[bytes, _TestClass]]]), ('type var', typing.TypeVar('T'), typehints.TypeVariable('T')), ('nested type var', typing.Tuple[typing.TypeVar('K'), typing.TypeVar('V')], typehints.Tuple[typehints.TypeVariable('K'), typehints.TypeVariable('V')]), ] for test_case in test_cases: # Unlike typing types, Beam types are guaranteed to compare equal. description = test_case[0] typing_type = test_case[1] beam_type = test_case[2] self.assertEqual( native_type_compatibility.convert_to_beam_type(typing_type), beam_type, description)
def convert_to_beam_type(typ): """Convert a given typing type to a Beam type. Args: typ (`typing.Union[type, str]`): typing type or string literal representing a type. Returns: type: The given type converted to a Beam type as far as we can do the conversion. Raises: ValueError: The type was malformed. """ if isinstance(typ, typing.TypeVar): # This is a special case, as it's not parameterized by types. # Also, identity must be preserved through conversion (i.e. the same # TypeVar instance must get converted into the same TypeVariable instance). # A global cache should be OK as the number of distinct type variables # is generally small. if id(typ) not in _type_var_cache: new_type_variable = typehints.TypeVariable(typ.__name__) _type_var_cache[id(typ)] = new_type_variable _type_var_cache[id(new_type_variable)] = typ return _type_var_cache[id(typ)] elif isinstance(typ, str): # Special case for forward references. # TODO(https://github.com/apache/beam/issues/19954): Currently unhandled. _LOGGER.info('Converting string literal type hint to Any: "%s"', typ) return typehints.Any elif getattr(typ, '__module__', None) != 'typing': # Only translate types from the typing module. return typ type_map = [ # TODO(https://github.com/apache/beam/issues/20076): Currently # unsupported. _TypeMapEntry(match=is_new_type, arity=0, beam_type=typehints.Any), # TODO(https://github.com/apache/beam/issues/19954): Currently # unsupported. _TypeMapEntry(match=is_forward_ref, arity=0, beam_type=typehints.Any), _TypeMapEntry(match=is_any, arity=0, beam_type=typehints.Any), _TypeMapEntry(match=_match_issubclass(typing.Dict), arity=2, beam_type=typehints.Dict), _TypeMapEntry(match=_match_is_exactly_iterable, arity=1, beam_type=typehints.Iterable), _TypeMapEntry(match=_match_issubclass(typing.List), arity=1, beam_type=typehints.List), _TypeMapEntry(match=_match_issubclass(typing.Set), arity=1, beam_type=typehints.Set), _TypeMapEntry(match=_match_issubclass(typing.FrozenSet), arity=1, beam_type=typehints.FrozenSet), # NamedTuple is a subclass of Tuple, but it needs special handling. # We just convert it to Any for now. # This MUST appear before the entry for the normal Tuple. _TypeMapEntry(match=match_is_named_tuple, arity=0, beam_type=typehints.Any), _TypeMapEntry(match=_match_issubclass(typing.Tuple), arity=-1, beam_type=typehints.Tuple), _TypeMapEntry(match=_match_is_union, arity=-1, beam_type=typehints.Union), _TypeMapEntry(match=_match_issubclass(typing.Generator), arity=3, beam_type=typehints.Generator), _TypeMapEntry(match=_match_issubclass(typing.Iterator), arity=1, beam_type=typehints.Iterator), ] # Find the first matching entry. matched_entry = next((entry for entry in type_map if entry.match(typ)), None) if not matched_entry: # Please add missing type support if you see this message. _LOGGER.info('Using Any for unsupported type: %s', typ) return typehints.Any args = _get_args(typ) len_args = len(args) if len_args == 0 and len_args != matched_entry.arity: arity = matched_entry.arity # Handle unsubscripted types. if _match_issubclass(typing.Tuple)(typ): args = (typehints.TypeVariable('T'), Ellipsis) elif _match_is_union(typ): raise ValueError('Unsupported Union with no arguments.') elif _match_issubclass(typing.Generator)(typ): # Assume a simple generator. args = (typehints.TypeVariable('T_co'), type(None), type(None)) elif _match_issubclass(typing.Dict)(typ): args = (typehints.TypeVariable('KT'), typehints.TypeVariable('VT')) elif (_match_issubclass(typing.Iterator)(typ) or _match_is_exactly_iterable(typ)): args = (typehints.TypeVariable('T_co'), ) else: args = (typehints.TypeVariable('T'), ) * arity elif matched_entry.arity == -1: arity = len_args else: arity = matched_entry.arity if len_args != arity: raise ValueError( 'expecting type %s to have arity %d, had arity %d ' 'instead' % (str(typ), arity, len_args)) typs = convert_to_beam_types(args) if arity == 0: # Nullary types (e.g. Any) don't accept empty tuples as arguments. return matched_entry.beam_type elif arity == 1: # Unary types (e.g. Set) don't accept 1-tuples as arguments return matched_entry.beam_type[typs[0]] else: return matched_entry.beam_type[tuple(typs)]
def test_match_type_variables(self): K = typehints.TypeVariable('K') # pylint: disable=invalid-name constraint = ShardedKeyType[K] self.assertEqual({K: int}, constraint.match_type_variables(ShardedKeyType[int]))
def test_eq_without_name_check(self): use_name_in_eq = False self.assertEqual( typehints.Set[typehints.TypeVariable('T', use_name_in_eq)], typehints.Set[typehints.TypeVariable('T2', use_name_in_eq)])
def convert_to_beam_type(typ): """Convert a given typing type to a Beam type. Args: typ (type): typing type. Returns: type: The given type converted to a Beam type as far as we can do the conversion. Raises: ~exceptions.ValueError: The type was malformed. """ if isinstance(typ, typing.TypeVar): # This is a special case, as it's not parameterized by types. # Also, identity must be preserved through conversion (i.e. the same # TypeVar instance must get converted into the same TypeVariable instance). # A global cache should be OK as the number of distinct type variables # is generally small. if id(typ) not in _type_var_cache: new_type_variable = typehints.TypeVariable(typ.__name__) _type_var_cache[id(typ)] = new_type_variable _type_var_cache[id(new_type_variable)] = typ return _type_var_cache[id(typ)] elif getattr(typ, '__module__', None) != 'typing': # Only translate types from the typing module. return typ type_map = [ _TypeMapEntry(match=_match_same_type(typing.Any), arity=0, beam_type=typehints.Any), _TypeMapEntry(match=_match_issubclass(typing.Dict), arity=2, beam_type=typehints.Dict), _TypeMapEntry(match=_match_is_exactly_iterable, arity=1, beam_type=typehints.Iterable), _TypeMapEntry(match=_match_issubclass(typing.List), arity=1, beam_type=typehints.List), _TypeMapEntry(match=_match_issubclass(typing.Set), arity=1, beam_type=typehints.Set), # NamedTuple is a subclass of Tuple, but it needs special handling. # We just convert it to Any for now. # This MUST appear before the entry for the normal Tuple. _TypeMapEntry(match=_match_is_named_tuple, arity=0, beam_type=typehints.Any), _TypeMapEntry(match=_match_issubclass(typing.Tuple), arity=-1, beam_type=typehints.Tuple), _TypeMapEntry(match=_match_is_union, arity=-1, beam_type=typehints.Union), _TypeMapEntry(match=_match_issubclass(typing.Generator), arity=3, beam_type=typehints.Generator), _TypeMapEntry(match=_match_issubclass(typing.Iterator), arity=1, beam_type=typehints.Iterator), ] # Find the first matching entry. matched_entry = next((entry for entry in type_map if entry.match(typ)), None) if not matched_entry: # No match: return original type. return typ if matched_entry.arity == -1: arity = _len_arg(typ) else: arity = matched_entry.arity if _len_arg(typ) != arity: raise ValueError( 'expecting type %s to have arity %d, had arity %d ' 'instead' % (str(typ), arity, _len_arg(typ))) typs = [convert_to_beam_type(_get_arg(typ, i)) for i in range(arity)] if arity == 0: # Nullary types (e.g. Any) don't accept empty tuples as arguments. return matched_entry.beam_type elif arity == 1: # Unary types (e.g. Set) don't accept 1-tuples as arguments return matched_entry.beam_type[typs[0]] else: return matched_entry.beam_type[tuple(typs)]