def normalize(x, none_as_type=False): # None is inconsistantly used for Any, unknown, or NoneType. if none_as_type and x is None: return type(None) elif x in _KNOWN_PRIMITIVE_TYPES: return _KNOWN_PRIMITIVE_TYPES[x] elif getattr(x, '__module__', None) == 'typing': # Avoid circular imports from apache_beam.typehints import native_type_compatibility beam_type = native_type_compatibility.convert_to_beam_type(x) if beam_type != x: # We were able to do the conversion. return beam_type else: # It might be a compatible type we don't understand. return Any return x
def test_convert_to_beam_type(self): test_cases = [ ('raw bytes', bytes, bytes), ('raw int', int, int), ('raw float', float, float), ('any', typing.Any, typehints.Any), ('simple dict', typing.Dict[bytes, int], typehints.Dict[bytes, int]), ('simple list', typing.List[int], typehints.List[int]), ('simple optional', typing.Optional[int], typehints.Optional[int]), ('simple set', typing.Set[float], typehints.Set[float]), ('simple unary tuple', typing.Tuple[bytes], typehints.Tuple[bytes]), ('simple union', typing.Union[int, bytes, float], typehints.Union[int, bytes, float]), ('namedtuple', _TestNamedTuple, typehints.Any), ('test class', _TestClass, _TestClass), ('test class in list', typing.List[_TestClass], typehints.List[_TestClass]), ('complex tuple', typing.Tuple[bytes, typing.List[typing.Tuple[ bytes, typing.Union[int, bytes, float]]]], typehints.Tuple[bytes, typehints.List[typehints.Tuple[ bytes, typehints.Union[int, bytes, float]]]]), ('flat alias', _TestFlatAlias, typehints.Tuple[bytes, float]), ('nested alias', _TestNestedAlias, typehints.List[typehints.Tuple[bytes, float]]), ('complex dict', typing.Dict[bytes, typing.List[typing.Tuple[bytes, _TestClass]]], typehints.Dict[bytes, typehints.List[typehints.Tuple[ bytes, _TestClass]]]) ] for test_case in test_cases: # Unlike typing types, Beam types are guaranteed to compare equal. description = test_case[0] typing_type = test_case[1] beam_type = test_case[2] self.assertEqual( native_type_compatibility.convert_to_beam_type(typing_type), beam_type, description)
def test_generator_converted_to_iterator(self): self.assertEqual( typehints.Iterator[int], convert_to_beam_type(typing.Generator[int, None, None]))
def with_output_types(*return_type_hint, **kwargs): """A decorator that type-checks defined type-hints for return values(s). This decorator will type-check the return value(s) of the decorated function. Only a single type-hint is accepted to specify the return type of the return value. If the function to be decorated has multiple return values, then one should use: ``Tuple[type_1, type_2]`` to annotate the types of the return values. If the ultimate return value for the function violates the specified type-hint a :class:`TypeCheckError` will be raised detailing the type-constraint violation. This decorator is intended to be used like: .. testcode:: from apache_beam.typehints import with_output_types from apache_beam.typehints import Set class Coordinate(object): def __init__(self, x, y): self.x = x self.y = y @with_output_types(Set[Coordinate]) def parse_ints(ints): return {Coordinate(i, i) for i in ints} Or with a simple type-hint: .. testcode:: from apache_beam.typehints import with_output_types @with_output_types(bool) def negate(p): return not p if p else p Args: *return_type_hint: A type-hint specifying the proper return type of the function. This argument should either be a built-in Python type or an instance of a :class:`~apache_beam.typehints.typehints.TypeConstraint` created by 'indexing' a :class:`~apache_beam.typehints.typehints.CompositeTypeHint`. **kwargs: Not used. Raises: :class:`~exceptions.ValueError`: If any kwarg parameters are passed in, or the length of **return_type_hint** is greater than ``1``. Or if the inner wrapper function isn't passed a function object. :class:`TypeCheckError`: If the **return_type_hint** object is in invalid type-hint. Returns: The original function decorated such that it enforces type-hint constraints for all return values. """ if kwargs: raise ValueError("All arguments for the 'returns' decorator must be " "positional arguments.") if len(return_type_hint) != 1: raise ValueError("'returns' accepts only a single positional argument. In " "order to specify multiple return types, use the 'Tuple' " "type-hint.") return_type_hint = native_type_compatibility.convert_to_beam_type( return_type_hint[0]) validate_composite_type_param( return_type_hint, error_msg_prefix='All type hint arguments' ) def annotate(f): get_type_hints(f).set_output_types(return_type_hint) return f return annotate
def test_convert_to_beam_type(self): test_cases = [ ('raw bytes', bytes, bytes), ('raw int', int, int), ('raw float', float, float), ('any', typing.Any, typehints.Any), ('simple dict', typing.Dict[bytes, int], typehints.Dict[bytes, int]), ('simple list', typing.List[int], typehints.List[int]), ('simple iterable', typing.Iterable[int], typehints.Iterable[int]), ('simple optional', typing.Optional[int], typehints.Optional[int]), ('simple set', typing.Set[float], typehints.Set[float]), ('simple unary tuple', typing.Tuple[bytes], typehints.Tuple[bytes]), ('simple union', typing.Union[int, bytes, float], typehints.Union[int, bytes, float]), ('namedtuple', _TestNamedTuple, _TestNamedTuple), ('test class', _TestClass, _TestClass), ('test class in list', typing.List[_TestClass], typehints.List[_TestClass]), ('complex tuple', typing.Tuple[bytes, typing.List[typing.Tuple[bytes, typing.Union[int, bytes, float]]]], typehints.Tuple[bytes, typehints.List[typehints.Tuple[ bytes, typehints.Union[int, bytes, float]]]]), # TODO(BEAM-7713): This case seems to fail on Py3.5.2 but not 3.5.4. ('arbitrary-length tuple', typing.Tuple[int, ...], typehints.Tuple[int, ...]) if sys.version_info >= (3, 5, 4) else None, ('flat alias', _TestFlatAlias, typehints.Tuple[bytes, float]), # type: ignore[misc] ('nested alias', _TestNestedAlias, typehints.List[typehints.Tuple[bytes, float]]), ('complex dict', typing.Dict[bytes, typing.List[typing.Tuple[bytes, _TestClass]]], typehints.Dict[bytes, typehints.List[typehints.Tuple[bytes, _TestClass]]]), ('type var', typing.TypeVar('T'), typehints.TypeVariable('T')), ('nested type var', typing.Tuple[typing.TypeVar('K'), typing.TypeVar('V')], typehints.Tuple[typehints.TypeVariable('K'), typehints.TypeVariable('V')]), ('iterator', typing.Iterator[typing.Any], typehints.Iterator[typehints.Any]), ] for test_case in test_cases: if test_case is None: continue # Unlike typing types, Beam types are guaranteed to compare equal. description = test_case[0] typing_type = test_case[1] expected_beam_type = test_case[2] converted_beam_type = convert_to_beam_type(typing_type) self.assertEqual(converted_beam_type, expected_beam_type, description) converted_typing_type = convert_to_typing_type(converted_beam_type) self.assertEqual(converted_typing_type, typing_type, description)
def assertNotCompatible(self, base, sub): # pylint: disable=invalid-name base, sub = native_type_compatibility.convert_to_beam_type([base, sub]) self.assertFalse( is_consistent_with(sub, base), '%s is consistent with %s' % (sub, base))
def with_output_types(*return_type_hint, **kwargs): """A decorator that type-checks defined type-hints for return values(s). This decorator will type-check the return value(s) of the decorated function. Only a single type-hint is accepted to specify the return type of the return value. If the function to be decorated has multiple return values, then one should use: ``Tuple[type_1, type_2]`` to annotate the types of the return values. If the ultimate return value for the function violates the specified type-hint a :class:`TypeCheckError` will be raised detailing the type-constraint violation. This decorator is intended to be used like: .. testcode:: from apache_beam.typehints import with_output_types from apache_beam.typehints import Set class Coordinate(object): def __init__(self, x, y): self.x = x self.y = y @with_output_types(Set[Coordinate]) def parse_ints(ints): return {Coordinate(i, i) for i in ints} Or with a simple type-hint: .. testcode:: from apache_beam.typehints import with_output_types @with_output_types(bool) def negate(p): return not p if p else p Args: *return_type_hint: A type-hint specifying the proper return type of the function. This argument should either be a built-in Python type or an instance of a :class:`~apache_beam.typehints.typehints.TypeConstraint` created by 'indexing' a :class:`~apache_beam.typehints.typehints.CompositeTypeHint`. **kwargs: Not used. Raises: :class:`~exceptions.ValueError`: If any kwarg parameters are passed in, or the length of **return_type_hint** is greater than ``1``. Or if the inner wrapper function isn't passed a function object. :class:`TypeCheckError`: If the **return_type_hint** object is in invalid type-hint. Returns: The original function decorated such that it enforces type-hint constraints for all return values. """ if kwargs: raise ValueError("All arguments for the 'returns' decorator must be " "positional arguments.") if len(return_type_hint) != 1: raise ValueError("'returns' accepts only a single positional argument. In " "order to specify multiple return types, use the 'Tuple' " "type-hint.") return_type_hint = native_type_compatibility.convert_to_beam_type( return_type_hint[0]) validate_composite_type_param( return_type_hint, error_msg_prefix='All type hint arguments' ) def annotate(f): get_type_hints(f).set_output_types(return_type_hint) return f return annotate
def test_forward_reference(self): self.assertEqual(typehints.Any, convert_to_beam_type('int')) self.assertEqual(typehints.Any, convert_to_beam_type('typing.List[int]')) self.assertEqual(typehints.List[typehints.Any], convert_to_beam_type(typing.List['int']))
def test_newtype(self): self.assertEqual(typehints.Any, convert_to_beam_type(typing.NewType('Number', int)))
def with_output_types(*return_type_hint, **kwargs): """A decorator that type-checks defined type-hints for return values(s). This decorator will type-check the return value(s) of the decorated function. Only a single type-hint is accepted to specify the return type of the return value. If the function to be decorated has multiple return values, then one should use: 'Tuple[type_1, type_2]' to annotate the types of the return values. If the ultimate return value for the function violates the specified type-hint a TypeCheckError will be raised detailing the type-constraint violation. This decorator is intended to be used like:: * @with_output_types(Set[Coordinate]) def parse_ints(ints): .... return [Coordinate.from_int(i) for i in ints] Or with a simple type-hint:: * @with_output_types(bool) def negate(p): return not p if p else p Args: *return_type_hint: A type-hint specifying the proper return type of the function. This argument should either be a built-in Python type or an instance of a 'TypeConstraint' created by 'indexing' a 'CompositeTypeHint'. **kwargs: Not used. Raises: ValueError: If any kwarg parameters are passed in, or the length of 'return_type_hint' is greater than 1. Or if the inner wrapper function isn't passed a function object. TypeCheckError: If the 'return_type_hint' object is in invalid type-hint. Returns: The original function decorated such that it enforces type-hint constraints for all return values. """ if kwargs: raise ValueError("All arguments for the 'returns' decorator must be " "positional arguments.") if len(return_type_hint) != 1: raise ValueError( "'returns' accepts only a single positional argument. In " "order to specify multiple return types, use the 'Tuple' " "type-hint.") return_type_hint = native_type_compatibility.convert_to_beam_type( return_type_hint[0]) validate_composite_type_param(return_type_hint, error_msg_prefix='All type hint arguments') def annotate(f): get_type_hints(f).set_output_types(return_type_hint) return f return annotate
def test_string_literal_converted_to_any(self): self.assertEqual(typehints.Any, convert_to_beam_type('typing.List[int]'))
def test_convert_to_beam_type(self): test_cases = [ ('raw bytes', bytes, bytes), ('raw int', int, int), ('raw float', float, float), ('any', typing.Any, typehints.Any), ('simple dict', typing.Dict[bytes, int], typehints.Dict[bytes, int]), ('simple list', typing.List[int], typehints.List[int]), ('simple iterable', typing.Iterable[int], typehints.Iterable[int]), ('simple optional', typing.Optional[int], typehints.Optional[int]), ('simple set', typing.Set[float], typehints.Set[float]), ('simple frozenset', typing.FrozenSet[float], typehints.FrozenSet[float]), ('simple unary tuple', typing.Tuple[bytes], typehints.Tuple[bytes]), ('simple union', typing.Union[int, bytes, float], typehints.Union[int, bytes, float]), ('namedtuple', _TestNamedTuple, _TestNamedTuple), ('test class', _TestClass, _TestClass), ('test class in list', typing.List[_TestClass], typehints.List[_TestClass]), ('generic bare', _TestGeneric, _TestGeneric), ('generic subscripted', _TestGeneric[int], _TestGeneric[int]), ('complex tuple', typing.Tuple[bytes, typing.List[typing.Tuple[bytes, typing.Union[int, bytes, float]]]], typehints.Tuple[bytes, typehints.List[typehints.Tuple[ bytes, typehints.Union[int, bytes, float]]]]), ('arbitrary-length tuple', typing.Tuple[int, ...], typehints.Tuple[int, ...]), ('flat alias', _TestFlatAlias, typehints.Tuple[bytes, float]), # type: ignore[misc] ('nested alias', _TestNestedAlias, typehints.List[typehints.Tuple[bytes, float]]), ('complex dict', typing.Dict[bytes, typing.List[typing.Tuple[bytes, _TestClass]]], typehints.Dict[bytes, typehints.List[typehints.Tuple[bytes, _TestClass]]]), ('type var', typing.TypeVar('T'), typehints.TypeVariable('T')), ('nested type var', typing.Tuple[typing.TypeVar('K'), typing.TypeVar('V')], typehints.Tuple[typehints.TypeVariable('K'), typehints.TypeVariable('V')]), ('iterator', typing.Iterator[typing.Any], typehints.Iterator[typehints.Any]), ('nested generic bare', typing.List[_TestGeneric], typehints.List[_TestGeneric]), ('nested generic subscripted', typing.List[_TestGeneric[int]], typehints.List[_TestGeneric[int]]), ] for test_case in test_cases: if test_case is None: continue # Unlike typing types, Beam types are guaranteed to compare equal. description = test_case[0] typing_type = test_case[1] expected_beam_type = test_case[2] converted_beam_type = convert_to_beam_type(typing_type) self.assertEqual(converted_beam_type, expected_beam_type, description) converted_typing_type = convert_to_typing_type(converted_beam_type) self.assertEqual(converted_typing_type, typing_type, description)
def element_type_from_coder_id(self, coder_id): if self.use_fake_coders or coder_id not in self.coders: return pickler.loads(coder_id) else: return native_type_compatibility.convert_to_beam_type( self.coders[coder_id].to_type_hint())