def to_runner_api_parameter(self, unused_context): _args_schema = named_fields_to_schema([ (f'arg{ix}', convert_to_typing_type(instance_to_type(value))) for (ix, value) in enumerate(self._args) ]) _kwargs_schema = named_fields_to_schema([ (key, convert_to_typing_type(instance_to_type(value))) for (key, value) in self._kwargs.items() ]) payload_schema = named_fields_to_schema({ 'constructor': str, 'args': _args_schema, 'kwargs': _kwargs_schema, }) return (PYTHON_FULLY_QUALIFIED_NAMED_TRANSFORM_URN, external_transforms_pb2.ExternalConfigurationPayload( schema=payload_schema, payload=coders.RowCoder(payload_schema).encode( Row(constructor=self._constructor, args=Row( **{ f'arg{ix}': arg for (ix, arg) in enumerate(self._args) }), kwargs=Row(**self._kwargs)), )))
def _get_schema_proto_and_payload(self, *args, **kwargs): named_fields = [] fields_to_values = OrderedDict() next_field_id = 0 for value in args: if value is None: raise ValueError( 'Received value None. None values are currently not supported' ) named_fields.append( ((JavaClassLookupPayloadBuilder.IGNORED_ARG_FORMAT % next_field_id), convert_to_typing_type(instance_to_type(value)))) fields_to_values[( JavaClassLookupPayloadBuilder.IGNORED_ARG_FORMAT % next_field_id)] = value next_field_id += 1 for key, value in kwargs.items(): if not key: raise ValueError('Parameter name cannot be empty') if value is None: raise ValueError( 'Received value None for key %s. None values are currently not ' 'supported' % key) named_fields.append( (key, convert_to_typing_type(instance_to_type(value)))) fields_to_values[key] = value schema_proto = named_fields_to_schema(named_fields) row = named_tuple_from_schema(schema_proto)(**fields_to_values) schema = named_tuple_to_schema(type(row)) payload = RowCoder(schema).encode(row) return (schema_proto, payload)
def _get_named_tuple_instance(self): schema = named_fields_to_schema([ (k, convert_to_typing_type(v)) for k, v in self._transform.__init__.__annotations__.items() if k in self._values ]) return named_tuple_from_schema(schema)(**self._values)
def _get_named_tuple_instance(self): # omit fields with value=None since we can't infer their type values = { key: value for key, value in self._values.items() if value is not None } # In python 2 named_fields_to_schema will not accept str because its # ambiguous. This converts str hints to ByteString recursively so its clear # we intend to use BYTES. # TODO(BEAM-7372): Remove coercion to ByteString def coerce_str_to_bytes(typ): if typ == str: return ByteString elif hasattr(typ, '__args__') and hasattr(typ, '__origin__'): # Create a new type rather than modifying the existing one typ = typ.__origin__[tuple( map(coerce_str_to_bytes, typ.__args__))] return typ if sys.version_info[0] >= 3: coerce_str_to_bytes = lambda x: x schema = named_fields_to_schema([ (key, coerce_str_to_bytes( convert_to_typing_type(instance_to_type(value)))) for key, value in values.items() ]) return named_tuple_from_schema(schema)(**values)
def _get_named_tuple_instance(self): # omit fields with value=None since we can't infer their type values = { key: value for key, value in self._values.items() if value is not None } # TODO(BEAM-7372): Remove coercion to ByteString def coerce_str_to_bytes(typ): if typ == str: return ByteString elif hasattr(typ, '__args__'): typ.__args__ = tuple(map(coerce_str_to_bytes, typ.__args__)) return typ if str == unicode: coerce_str_to_bytes = lambda x: x schema = named_fields_to_schema([ (key, coerce_str_to_bytes( convert_to_typing_type(instance_to_type(value)))) for key, value in values.items() ]) return named_tuple_from_schema(schema)(**values)
def _get_named_tuple_instance(self): import dataclasses schema = named_fields_to_schema([ (field.name, convert_to_typing_type(field.type)) for field in dataclasses.fields(self._transform) ]) return named_tuple_from_schema(schema)( **dataclasses.asdict(self._transform))
def test_convert_to_beam_type(self): test_cases = [ ('raw bytes', bytes, bytes), ('raw int', int, int), ('raw float', float, float), ('any', typing.Any, typehints.Any), ('simple dict', typing.Dict[bytes, int], typehints.Dict[bytes, int]), ('simple list', typing.List[int], typehints.List[int]), ('simple iterable', typing.Iterable[int], typehints.Iterable[int]), ('simple optional', typing.Optional[int], typehints.Optional[int]), ('simple set', typing.Set[float], typehints.Set[float]), ('simple frozenset', typing.FrozenSet[float], typehints.FrozenSet[float]), ('simple unary tuple', typing.Tuple[bytes], typehints.Tuple[bytes]), ('simple union', typing.Union[int, bytes, float], typehints.Union[int, bytes, float]), ('namedtuple', _TestNamedTuple, _TestNamedTuple), ('test class', _TestClass, _TestClass), ('test class in list', typing.List[_TestClass], typehints.List[_TestClass]), ('complex tuple', typing.Tuple[bytes, typing.List[typing.Tuple[ bytes, typing.Union[int, bytes, float]]]], typehints.Tuple[bytes, typehints.List[typehints.Tuple[ bytes, typehints.Union[int, bytes, float]]]]), # TODO(BEAM-7713): This case seems to fail on Py3.5.2 but not 3.5.4. ('arbitrary-length tuple', typing.Tuple[int, ...], typehints.Tuple[int, ...]) if sys.version_info >= (3, 5, 4) else None, ('flat alias', _TestFlatAlias, typehints.Tuple[bytes, float]), # type: ignore[misc] ('nested alias', _TestNestedAlias, typehints.List[typehints.Tuple[bytes, float]]), ('complex dict', typing.Dict[bytes, typing.List[typing.Tuple[bytes, _TestClass]]], typehints.Dict[bytes, typehints.List[typehints.Tuple[ bytes, _TestClass]]]), ('type var', typing.TypeVar('T'), typehints.TypeVariable('T')), ('nested type var', typing.Tuple[typing.TypeVar('K'), typing.TypeVar('V')], typehints.Tuple[typehints.TypeVariable('K'), typehints.TypeVariable('V')]), ('iterator', typing.Iterator[typing.Any], typehints.Iterator[typehints.Any]), ] for test_case in test_cases: if test_case is None: continue # Unlike typing types, Beam types are guaranteed to compare equal. description = test_case[0] typing_type = test_case[1] expected_beam_type = test_case[2] converted_beam_type = convert_to_beam_type(typing_type) self.assertEqual(converted_beam_type, expected_beam_type, description) converted_typing_type = convert_to_typing_type(converted_beam_type) self.assertEqual(converted_typing_type, typing_type, description)
def _get_named_tuple_instance(self): # omit fields with value=None since we can't infer their type values = { key: value for key, value in self._values.items() if value is not None } schema = named_fields_to_schema([ (key, convert_to_typing_type(instance_to_type(value))) for key, value in values.items() ]) return named_tuple_from_schema(schema)(**values)
def convert_to_typing_type(type_): if isinstance(type_, row_type.RowTypeConstraint): return named_tuple_from_schema(named_fields_to_schema(type_._fields)) else: return native_type_compatibility.convert_to_typing_type(type_)