def test_type_resolution(): assert type(TypeEngine.get_transformer(typing.List[int])) == ListTransformer assert type(TypeEngine.get_transformer(typing.List)) == ListTransformer assert type(TypeEngine.get_transformer(list)) == ListTransformer assert type(TypeEngine.get_transformer(typing.Dict[str, int])) == DictTransformer assert type(TypeEngine.get_transformer(typing.Dict)) == DictTransformer assert type(TypeEngine.get_transformer(dict)) == DictTransformer assert type(TypeEngine.get_transformer(int)) == SimpleTransformer assert type(TypeEngine.get_transformer(os.PathLike)) == PathLikeTransformer
def _change_unrecognized_type_to_pickle(t: Type[T]) -> Type[T]: try: if hasattr(t, "__origin__") and hasattr(t, "__args__"): if t.__origin__ == list: return typing.List[_change_unrecognized_type_to_pickle( t.__args__[0])] elif t.__origin__ == dict and t.__args__[0] == str: return typing.Dict[ str, _change_unrecognized_type_to_pickle(t.__args__[1])] else: TypeEngine.get_transformer(t) except ValueError: logger.warning( f"Unsupported Type {t} found, Flyte will default to use PickleFile as the transport. " f"Pickle can only be used to send objects between the exact same version of Python, " f"and we strongly recommend to use python type that flyte support." ) return FlytePickle[t] return t
def test_file_format_getting_python_value(): transformer = TypeEngine.get_transformer(FlyteFile) ctx = FlyteContext.current_context() # This file probably won't exist, but it's okay. It won't be downloaded unless we try to read the thing returned lv = Literal(scalar=Scalar(blob=Blob(metadata=BlobMetadata( type=BlobType(format="txt", dimensionality=0)), uri="file:///tmp/test"))) pv = transformer.to_python_value(ctx, lv, expected_python_type=FlyteFile["txt"]) assert isinstance(pv, FlyteFile) assert pv.extension() == "txt"
def test_file_guess(): transformer = TypeEngine.get_transformer(FlyteFile) lt = transformer.get_literal_type(FlyteFile["txt"]) assert lt.blob.format == "txt" assert lt.blob.dimensionality == 0 fft = transformer.guess_python_type(lt) assert issubclass(fft, FlyteFile) assert fft.extension() == "txt" lt = transformer.get_literal_type(FlyteFile) assert lt.blob.format == "" assert lt.blob.dimensionality == 0 fft = transformer.guess_python_type(lt) assert issubclass(fft, FlyteFile) assert fft.extension() == ""
def test_file_formats_getting_literal_type(): transformer = TypeEngine.get_transformer(FlyteFile) lt = transformer.get_literal_type(FlyteFile) assert lt.blob.format == "" # Works with formats that we define lt = transformer.get_literal_type(FlyteFile["txt"]) assert lt.blob.format == "txt" lt = transformer.get_literal_type(FlyteFile[typing.TypeVar("jpg")]) assert lt.blob.format == "jpg" # Empty default to the default lt = transformer.get_literal_type(FlyteFile) assert lt.blob.format == "" lt = transformer.get_literal_type(FlyteFile[typing.TypeVar(".png")]) assert lt.blob.format == "png"
def test_type_resolution(): assert type(TypeEngine.get_transformer( PipelineModel)) == PySparkPipelineModelTransformer