def test_reindexing_with_float64_NA_log():
# GH 47055
s = Series([1.0, NA], dtype=Float64Dtype())
s_reindex = s.reindex(range(3))
result = s_reindex.values._data
expected = np.array([1, np.NaN, np.NaN])
tm.assert_numpy_array_equal(result, expected)
with tm.assert_produces_warning(None):
result_log = np.log(s_reindex)
expected_log = Series([0, np.NaN, np.NaN], dtype=Float64Dtype())
tm.assert_series_equal(result_log, expected_log)
def time_frame_from_scalar_ea_float64_na(self):
DataFrame(
NA,
index=range(self.nrows),
columns=list("abc"),
dtype=Float64Dtype(),
)
def test_as_spark_type_extension_float_dtypes(self):
from pandas import Float32Dtype, Float64Dtype
type_mapper = {
Float32Dtype(): FloatType(),
Float64Dtype(): DoubleType(),
}
for extension_dtype, spark_type in type_mapper.items():
self.assertEqual(as_spark_type(extension_dtype), spark_type)
def spark_type_to_pandas_dtype(
spark_type: types.DataType, *, use_extension_dtypes: bool = False
) -> Dtype:
"""Return the given Spark DataType to pandas dtype."""
if use_extension_dtypes and extension_dtypes_available:
# IntegralType
if isinstance(spark_type, types.ByteType):
return Int8Dtype()
elif isinstance(spark_type, types.ShortType):
return Int16Dtype()
elif isinstance(spark_type, types.IntegerType):
return Int32Dtype()
elif isinstance(spark_type, types.LongType):
return Int64Dtype()
if extension_object_dtypes_available:
# BooleanType
if isinstance(spark_type, types.BooleanType):
return BooleanDtype()
# StringType
elif isinstance(spark_type, types.StringType):
return StringDtype()
# FractionalType
if extension_float_dtypes_available:
if isinstance(spark_type, types.FloatType):
return Float32Dtype()
elif isinstance(spark_type, types.DoubleType):
return Float64Dtype()
if isinstance(
spark_type,
(
types.DateType,
types.NullType,
types.ArrayType,
types.MapType,
types.StructType,
types.UserDefinedType,
),
):
return np.dtype("object")
elif isinstance(spark_type, types.TimestampType):
return np.dtype("datetime64[ns]")
else:
return np.dtype(to_arrow_type(spark_type).to_pandas_dtype())
def fractional_extension_dtypes(self):
return (
["Float32", "Float64", Float32Dtype(), Float64Dtype()]
if extension_float_dtypes_available
else []
)
