def apply_standard(self) -> FrameOrSeriesUnion:
f = self.f
obj = self.obj
with np.errstate(all="ignore"):
if isinstance(f, np.ufunc):
return f(obj)
# row-wise access
if is_extension_array_dtype(obj.dtype) and hasattr(
obj._values, "map"):
# GH#23179 some EAs do not have `map`
mapped = obj._values.map(f)
else:
values = obj.astype(object)._values
mapped = lib.map_infer(values, f, convert=self.convert_dtype)
if len(mapped) and isinstance(mapped[0], ABCSeries):
# GH 25959 use pd.array instead of tolist
# so extension arrays can be used
return obj._constructor_expanddim(pd_array(mapped),
index=obj.index)
else:
return obj._constructor(mapped, index=obj.index).__finalize__(
obj, method="apply")
def apply_standard(self) -> FrameOrSeriesUnion:
f = self.f
obj = self.obj
with np.errstate(all="ignore"):
if isinstance(f, np.ufunc):
return f(obj)
# row-wise access
if is_extension_array_dtype(obj.dtype) and hasattr(obj._values, "map"):
# GH#23179 some EAs do not have `map`
mapped = obj._values.map(f)
else:
values = obj.astype(object)._values
# error: Argument 2 to "map_infer" has incompatible type
# "Union[Callable[..., Any], str, List[Union[Callable[..., Any], str]],
# Dict[Hashable, Union[Union[Callable[..., Any], str],
# List[Union[Callable[..., Any], str]]]]]"; expected
# "Callable[[Any], Any]"
mapped = lib.map_infer(
values,
f, # type: ignore[arg-type]
convert=self.convert_dtype,
)
if len(mapped) and isinstance(mapped[0], ABCSeries):
# GH 25959 use pd.array instead of tolist
# so extension arrays can be used
return obj._constructor_expanddim(pd_array(mapped), index=obj.index)
else:
return obj._constructor(mapped, index=obj.index).__finalize__(
obj, method="apply"
)
def _cast_to_common_type(arr: ArrayLike, dtype: DtypeObj) -> ArrayLike:
"""
Helper function for `arr.astype(common_dtype)` but handling all special
cases.
"""
if (is_categorical_dtype(arr.dtype) and isinstance(dtype, np.dtype)
and np.issubdtype(dtype, np.integer)):
# problem case: categorical of int -> gives int as result dtype,
# but categorical can contain NAs -> fall back to object dtype
try:
return arr.astype(dtype, copy=False)
except ValueError:
return arr.astype(object, copy=False)
if is_sparse(arr) and not is_sparse(dtype):
# problem case: SparseArray.astype(dtype) doesn't follow the specified
# dtype exactly, but converts this to Sparse[dtype] -> first manually
# convert to dense array
arr = cast(SparseArray, arr)
return arr.to_dense().astype(dtype, copy=False)
if (isinstance(arr, np.ndarray) and arr.dtype.kind in ["m", "M"]
and dtype is np.dtype("object")):
# wrap datetime-likes in EA to ensure astype(object) gives Timestamp/Timedelta
# this can happen when concat_compat is called directly on arrays (when arrays
# are not coming from Index/Series._values), eg in BlockManager.quantile
arr = ensure_wrapped_if_datetimelike(arr)
if is_extension_array_dtype(dtype) and isinstance(arr, np.ndarray):
# numpy's astype cannot handle ExtensionDtypes
return pd_array(arr, dtype=dtype, copy=False)
return arr.astype(dtype, copy=False)
def apply_standard(self) -> FrameOrSeriesUnion:
f = self.f
obj = self.obj
with np.errstate(all="ignore"):
if isinstance(f, np.ufunc):
# error: Argument 1 to "__call__" of "ufunc" has incompatible type
# "Series"; expected "Union[Union[int, float, complex, str, bytes,
# generic], Sequence[Union[int, float, complex, str, bytes, generic]],
# Sequence[Sequence[Any]], _SupportsArray]"
return f(obj) # type: ignore[arg-type]
# row-wise access
if is_extension_array_dtype(obj.dtype) and hasattr(obj._values, "map"):
# GH#23179 some EAs do not have `map`
mapped = obj._values.map(f)
else:
values = obj.astype(object)._values
mapped = lib.map_infer(values, f, convert=self.convert_dtype)
if len(mapped) and isinstance(mapped[0], ABCSeries):
# GH 25959 use pd.array instead of tolist
# so extension arrays can be used
return obj._constructor_expanddim(pd_array(mapped), index=obj.index)
else:
return obj._constructor(mapped, index=obj.index).__finalize__(
obj, method="apply"
)
def _wrap_datetimelike(arr):
"""
Wrap datetime64 and timedelta64 ndarrays in DatetimeArray/TimedeltaArray.
DTA/TDA handle .astype(object) correctly.
"""
from pandas.core.construction import array as pd_array, extract_array
arr = extract_array(arr, extract_numpy=True)
if isinstance(arr, np.ndarray) and arr.dtype.kind in ["m", "M"]:
arr = pd_array(arr)
return arr
def _convert_arr_indexer(self, keyarr):
if lib.infer_dtype(keyarr) == "string":
# Weak reasoning that indexer is a list of strings
# representing datetime or timedelta or period
try:
extension_arr = pd_array(keyarr, self.dtype)
except (ValueError, DateParseError):
# Fail to infer keyarr from self.dtype
return keyarr
converted_arr = extract_array(extension_arr, extract_numpy=True)
else:
converted_arr = com.asarray_tuplesafe(keyarr)
return converted_arr
def cast_to_common_type(arr: ArrayLike, dtype: DtypeObj) -> ArrayLike:
"""
Helper function for `arr.astype(common_dtype)` but handling all special
cases.
"""
if is_dtype_equal(arr.dtype, dtype):
return arr
if (
is_categorical_dtype(arr.dtype)
and isinstance(dtype, np.dtype)
and np.issubdtype(dtype, np.integer)
):
# problem case: categorical of int -> gives int as result dtype,
# but categorical can contain NAs -> fall back to object dtype
try:
return arr.astype(dtype, copy=False)
except ValueError:
return arr.astype(object, copy=False)
if is_sparse(arr) and not is_sparse(dtype):
# problem case: SparseArray.astype(dtype) doesn't follow the specified
# dtype exactly, but converts this to Sparse[dtype] -> first manually
# convert to dense array
# error: Argument 1 to "astype" of "_ArrayOrScalarCommon" has incompatible type
# "Union[dtype[Any], ExtensionDtype]"; expected "Union[dtype[Any], None, type, _
# SupportsDType[dtype[Any]], str, Union[Tuple[Any, int], Tuple[Any,
# Union[SupportsIndex, Sequence[SupportsIndex]]], List[Any], _DTypeDict,
# Tuple[Any, Any]]]" [arg-type]
arr = cast(SparseArray, arr)
return arr.to_dense().astype(dtype, copy=False) # type: ignore[arg-type]
if (
isinstance(arr, np.ndarray)
and arr.dtype.kind in ["m", "M"]
and dtype is np.dtype("object")
):
# wrap datetime-likes in EA to ensure astype(object) gives Timestamp/Timedelta
# this can happen when concat_compat is called directly on arrays (when arrays
# are not coming from Index/Series._values), eg in BlockManager.quantile
arr = ensure_wrapped_if_datetimelike(arr)
if isinstance(dtype, ExtensionDtype):
if isinstance(arr, np.ndarray):
# numpy's astype cannot handle ExtensionDtypes
return pd_array(arr, dtype=dtype, copy=False)
return arr.astype(dtype, copy=False)
return arr.astype(dtype, copy=False)
def check_array_indexer(array: AnyArrayLike, indexer: Any) -> Any:
"""
Check if `indexer` is a valid array indexer for `array`.
For a boolean mask, `array` and `indexer` are checked to have the same
length. The dtype is validated, and if it is an integer or boolean
ExtensionArray, it is checked if there are missing values present, and
it is converted to the appropriate numpy array. Other dtypes will raise
an error.
Non-array indexers (integer, slice, Ellipsis, tuples, ..) are passed
through as is.
.. versionadded:: 1.0.0
Parameters
----------
array : array-like
The array that is being indexed (only used for the length).
indexer : array-like or list-like
The array-like that's used to index. List-like input that is not yet
a numpy array or an ExtensionArray is converted to one. Other input
types are passed through as is.
Returns
-------
numpy.ndarray
The validated indexer as a numpy array that can be used to index.
Raises
------
IndexError
When the lengths don't match.
ValueError
When `indexer` cannot be converted to a numpy ndarray to index
(e.g. presence of missing values).
See Also
--------
api.types.is_bool_dtype : Check if `key` is of boolean dtype.
Examples
--------
When checking a boolean mask, a boolean ndarray is returned when the
arguments are all valid.
>>> mask = pd.array([True, False])
>>> arr = pd.array([1, 2])
>>> pd.api.indexers.check_array_indexer(arr, mask)
array([ True, False])
An IndexError is raised when the lengths don't match.
>>> mask = pd.array([True, False, True])
>>> pd.api.indexers.check_array_indexer(arr, mask)
Traceback (most recent call last):
...
IndexError: Boolean index has wrong length: 3 instead of 2.
NA values in a boolean array are treated as False.
>>> mask = pd.array([True, pd.NA])
>>> pd.api.indexers.check_array_indexer(arr, mask)
array([ True, False])
A numpy boolean mask will get passed through (if the length is correct):
>>> mask = np.array([True, False])
>>> pd.api.indexers.check_array_indexer(arr, mask)
array([ True, False])
Similarly for integer indexers, an integer ndarray is returned when it is
a valid indexer, otherwise an error is (for integer indexers, a matching
length is not required):
>>> indexer = pd.array([0, 2], dtype="Int64")
>>> arr = pd.array([1, 2, 3])
>>> pd.api.indexers.check_array_indexer(arr, indexer)
array([0, 2])
>>> indexer = pd.array([0, pd.NA], dtype="Int64")
>>> pd.api.indexers.check_array_indexer(arr, indexer)
Traceback (most recent call last):
...
ValueError: Cannot index with an integer indexer containing NA values
For non-integer/boolean dtypes, an appropriate error is raised:
>>> indexer = np.array([0., 2.], dtype="float64")
>>> pd.api.indexers.check_array_indexer(arr, indexer)
Traceback (most recent call last):
...
IndexError: arrays used as indices must be of integer or boolean type
"""
from pandas.core.construction import array as pd_array
# whatever is not an array-like is returned as-is (possible valid array
# indexers that are not array-like: integer, slice, Ellipsis, None)
# In this context, tuples are not considered as array-like, as they have
# a specific meaning in indexing (multi-dimensional indexing)
if is_list_like(indexer):
if isinstance(indexer, tuple):
return indexer
else:
return indexer
# convert list-likes to array
if not is_array_like(indexer):
indexer = pd_array(indexer)
if len(indexer) == 0:
# empty list is converted to float array by pd.array
indexer = np.array([], dtype=np.intp)
dtype = indexer.dtype
if is_bool_dtype(dtype):
if is_extension_array_dtype(dtype):
indexer = indexer.to_numpy(dtype=bool, na_value=False)
else:
indexer = np.asarray(indexer, dtype=bool)
# GH26658
if len(indexer) != len(array):
raise IndexError(
f"Boolean index has wrong length: "
f"{len(indexer)} instead of {len(array)}"
)
elif is_integer_dtype(dtype):
try:
indexer = np.asarray(indexer, dtype=np.intp)
except ValueError as err:
raise ValueError(
"Cannot index with an integer indexer containing NA values"
) from err
else:
raise IndexError("arrays used as indices must be of integer or boolean type")
return indexer
