class TestSparseArray: def setup_method(self, method): self.arr_data = np.array( [np.nan, np.nan, 1, 2, 3, np.nan, 4, 5, np.nan, 6]) self.arr = SparseArray(self.arr_data) self.zarr = SparseArray([0, 0, 1, 2, 3, 0, 4, 5, 0, 6], fill_value=0) def test_constructor_dtype(self): arr = SparseArray([np.nan, 1, 2, np.nan]) assert arr.dtype == SparseDtype(np.float64, np.nan) assert arr.dtype.subtype == np.float64 assert np.isnan(arr.fill_value) arr = SparseArray([np.nan, 1, 2, np.nan], fill_value=0) assert arr.dtype == SparseDtype(np.float64, 0) assert arr.fill_value == 0 arr = SparseArray([0, 1, 2, 4], dtype=np.float64) assert arr.dtype == SparseDtype(np.float64, np.nan) assert np.isnan(arr.fill_value) arr = SparseArray([0, 1, 2, 4], dtype=np.int64) assert arr.dtype == SparseDtype(np.int64, 0) assert arr.fill_value == 0 arr = SparseArray([0, 1, 2, 4], fill_value=0, dtype=np.int64) assert arr.dtype == SparseDtype(np.int64, 0) assert arr.fill_value == 0 arr = SparseArray([0, 1, 2, 4], dtype=None) assert arr.dtype == SparseDtype(np.int64, 0) assert arr.fill_value == 0 arr = SparseArray([0, 1, 2, 4], fill_value=0, dtype=None) assert arr.dtype == SparseDtype(np.int64, 0) assert arr.fill_value == 0 def test_constructor_dtype_str(self): result = SparseArray([1, 2, 3], dtype="int") expected = SparseArray([1, 2, 3], dtype=int) tm.assert_sp_array_equal(result, expected) def test_constructor_sparse_dtype(self): result = SparseArray([1, 0, 0, 1], dtype=SparseDtype("int64", -1)) expected = SparseArray([1, 0, 0, 1], fill_value=-1, dtype=np.int64) tm.assert_sp_array_equal(result, expected) assert result.sp_values.dtype == np.dtype("int64") def test_constructor_sparse_dtype_str(self): result = SparseArray([1, 0, 0, 1], dtype="Sparse[int32]") expected = SparseArray([1, 0, 0, 1], dtype=np.int32) tm.assert_sp_array_equal(result, expected) assert result.sp_values.dtype == np.dtype("int32") def test_constructor_object_dtype(self): # GH 11856 arr = SparseArray(["A", "A", np.nan, "B"], dtype=np.object) assert arr.dtype == SparseDtype(np.object) assert np.isnan(arr.fill_value) arr = SparseArray(["A", "A", np.nan, "B"], dtype=np.object, fill_value="A") assert arr.dtype == SparseDtype(np.object, "A") assert arr.fill_value == "A" # GH 17574 data = [False, 0, 100.0, 0.0] arr = SparseArray(data, dtype=np.object, fill_value=False) assert arr.dtype == SparseDtype(np.object, False) assert arr.fill_value is False arr_expected = np.array(data, dtype=np.object) it = (type(x) == type(y) and x == y for x, y in zip(arr, arr_expected)) assert np.fromiter(it, dtype=np.bool).all() @pytest.mark.parametrize("dtype", [SparseDtype(int, 0), int]) def test_constructor_na_dtype(self, dtype): with pytest.raises(ValueError, match="Cannot convert"): SparseArray([0, 1, np.nan], dtype=dtype) def test_constructor_warns_when_losing_timezone(self): # GH#32501 warn when losing timezone inforamtion dti = pd.date_range("2016-01-01", periods=3, tz="US/Pacific") expected = SparseArray(np.asarray(dti, dtype="datetime64[ns]")) with tm.assert_produces_warning(UserWarning): result = SparseArray(dti) tm.assert_sp_array_equal(result, expected) with tm.assert_produces_warning(UserWarning): result = SparseArray(pd.Series(dti)) tm.assert_sp_array_equal(result, expected) def test_constructor_spindex_dtype(self): arr = SparseArray(data=[1, 2], sparse_index=IntIndex(4, [1, 2])) # XXX: Behavior change: specifying SparseIndex no longer changes the # fill_value expected = SparseArray([0, 1, 2, 0], kind="integer") tm.assert_sp_array_equal(arr, expected) assert arr.dtype == SparseDtype(np.int64) assert arr.fill_value == 0 arr = SparseArray( data=[1, 2, 3], sparse_index=IntIndex(4, [1, 2, 3]), dtype=np.int64, fill_value=0, ) exp = SparseArray([0, 1, 2, 3], dtype=np.int64, fill_value=0) tm.assert_sp_array_equal(arr, exp) assert arr.dtype == SparseDtype(np.int64) assert arr.fill_value == 0 arr = SparseArray(data=[1, 2], sparse_index=IntIndex(4, [1, 2]), fill_value=0, dtype=np.int64) exp = SparseArray([0, 1, 2, 0], fill_value=0, dtype=np.int64) tm.assert_sp_array_equal(arr, exp) assert arr.dtype == SparseDtype(np.int64) assert arr.fill_value == 0 arr = SparseArray( data=[1, 2, 3], sparse_index=IntIndex(4, [1, 2, 3]), dtype=None, fill_value=0, ) exp = SparseArray([0, 1, 2, 3], dtype=None) tm.assert_sp_array_equal(arr, exp) assert arr.dtype == SparseDtype(np.int64) assert arr.fill_value == 0 @pytest.mark.parametrize("sparse_index", [None, IntIndex(1, [0])]) def test_constructor_spindex_dtype_scalar(self, sparse_index): # scalar input arr = SparseArray(data=1, sparse_index=sparse_index, dtype=None) exp = SparseArray([1], dtype=None) tm.assert_sp_array_equal(arr, exp) assert arr.dtype == SparseDtype(np.int64) assert arr.fill_value == 0 arr = SparseArray(data=1, sparse_index=IntIndex(1, [0]), dtype=None) exp = SparseArray([1], dtype=None) tm.assert_sp_array_equal(arr, exp) assert arr.dtype == SparseDtype(np.int64) assert arr.fill_value == 0 def test_constructor_spindex_dtype_scalar_broadcasts(self): arr = SparseArray(data=[1, 2], sparse_index=IntIndex(4, [1, 2]), fill_value=0, dtype=None) exp = SparseArray([0, 1, 2, 0], fill_value=0, dtype=None) tm.assert_sp_array_equal(arr, exp) assert arr.dtype == SparseDtype(np.int64) assert arr.fill_value == 0 @pytest.mark.parametrize( "data, fill_value", [ (np.array([1, 2]), 0), (np.array([1.0, 2.0]), np.nan), ([True, False], False), ([pd.Timestamp("2017-01-01")], pd.NaT), ], ) def test_constructor_inferred_fill_value(self, data, fill_value): result = SparseArray(data).fill_value if pd.isna(fill_value): assert pd.isna(result) else: assert result == fill_value @pytest.mark.parametrize("format", ["coo", "csc", "csr"]) @pytest.mark.parametrize( "size", [ pytest.param( 0, marks=td.skip_if_np_lt("1.16", reason="NumPy-11383")), 10 ], ) @td.skip_if_no_scipy def test_from_spmatrix(self, size, format): import scipy.sparse mat = scipy.sparse.random(size, 1, density=0.5, format=format) result = SparseArray.from_spmatrix(mat) result = np.asarray(result) expected = mat.toarray().ravel() tm.assert_numpy_array_equal(result, expected) @pytest.mark.parametrize("format", ["coo", "csc", "csr"]) @td.skip_if_no_scipy def test_from_spmatrix_including_explicit_zero(self, format): import scipy.sparse mat = scipy.sparse.random(10, 1, density=0.5, format=format) mat.data[0] = 0 result = SparseArray.from_spmatrix(mat) result = np.asarray(result) expected = mat.toarray().ravel() tm.assert_numpy_array_equal(result, expected) @td.skip_if_no_scipy def test_csr_to_spmatrix_with_explicit_zero(self): # similar to above, using exact methods in issue #28992 import scipy.sparse randmat = scipy.sparse.random(5, 5, density=0.5) mat = scipy.sparse.csr_matrix(randmat) mat.data[0] = 0 dfmat = pd.DataFrame.sparse.from_spmatrix(mat) densemat = mat.todense() tm.assert_numpy_array_equal(np.array(dfmat), np.array(densemat)) @td.skip_if_no_scipy def test_from_spmatrix_raises(self): import scipy.sparse mat = scipy.sparse.eye(5, 4, format="csc") with pytest.raises(ValueError, match="not '4'"): SparseArray.from_spmatrix(mat) @pytest.mark.parametrize( "scalar,dtype", [ (False, SparseDtype(bool, False)), (0.0, SparseDtype("float64", 0)), (1, SparseDtype("int64", 1)), ("z", SparseDtype("object", "z")), ], ) def test_scalar_with_index_infer_dtype(self, scalar, dtype): # GH 19163 arr = SparseArray(scalar, index=[1, 2, 3], fill_value=scalar) exp = SparseArray([scalar, scalar, scalar], fill_value=scalar) tm.assert_sp_array_equal(arr, exp) assert arr.dtype == dtype assert exp.dtype == dtype def test_get_item(self): assert np.isnan(self.arr[1]) assert self.arr[2] == 1 assert self.arr[7] == 5 assert self.zarr[0] == 0 assert self.zarr[2] == 1 assert self.zarr[7] == 5 errmsg = re.compile("bounds") with pytest.raises(IndexError, match=errmsg): self.arr[11] with pytest.raises(IndexError, match=errmsg): self.arr[-11] assert self.arr[-1] == self.arr[len(self.arr) - 1] def test_take_scalar_raises(self): msg = "'indices' must be an array, not a scalar '2'." with pytest.raises(ValueError, match=msg): self.arr.take(2) def test_take(self): exp = SparseArray(np.take(self.arr_data, [2, 3])) tm.assert_sp_array_equal(self.arr.take([2, 3]), exp) exp = SparseArray(np.take(self.arr_data, [0, 1, 2])) tm.assert_sp_array_equal(self.arr.take([0, 1, 2]), exp) def test_take_fill_value(self): data = np.array([1, np.nan, 0, 3, 0]) sparse = SparseArray(data, fill_value=0) exp = SparseArray(np.take(data, [0]), fill_value=0) tm.assert_sp_array_equal(sparse.take([0]), exp) exp = SparseArray(np.take(data, [1, 3, 4]), fill_value=0) tm.assert_sp_array_equal(sparse.take([1, 3, 4]), exp) def test_take_negative(self): exp = SparseArray(np.take(self.arr_data, [-1])) tm.assert_sp_array_equal(self.arr.take([-1]), exp) exp = SparseArray(np.take(self.arr_data, [-4, -3, -2])) tm.assert_sp_array_equal(self.arr.take([-4, -3, -2]), exp) @pytest.mark.parametrize("fill_value", [0, None, np.nan]) def test_shift_fill_value(self, fill_value): # GH #24128 sparse = SparseArray(np.array([1, 0, 0, 3, 0]), fill_value=8.0) res = sparse.shift(1, fill_value=fill_value) if isna(fill_value): fill_value = res.dtype.na_value exp = SparseArray(np.array([fill_value, 1, 0, 0, 3]), fill_value=8.0) tm.assert_sp_array_equal(res, exp) def test_bad_take(self): with pytest.raises(IndexError, match="bounds"): self.arr.take([11]) def test_take_filling(self): # similar tests as GH 12631 sparse = SparseArray([np.nan, np.nan, 1, np.nan, 4]) result = sparse.take(np.array([1, 0, -1])) expected = SparseArray([np.nan, np.nan, 4]) tm.assert_sp_array_equal(result, expected) # XXX: test change: fill_value=True -> allow_fill=True result = sparse.take(np.array([1, 0, -1]), allow_fill=True) expected = SparseArray([np.nan, np.nan, np.nan]) tm.assert_sp_array_equal(result, expected) # allow_fill=False result = sparse.take(np.array([1, 0, -1]), allow_fill=False, fill_value=True) expected = SparseArray([np.nan, np.nan, 4]) tm.assert_sp_array_equal(result, expected) msg = "Invalid value in 'indices'" with pytest.raises(ValueError, match=msg): sparse.take(np.array([1, 0, -2]), allow_fill=True) with pytest.raises(ValueError, match=msg): sparse.take(np.array([1, 0, -5]), allow_fill=True) msg = "out of bounds value in 'indices'" with pytest.raises(IndexError, match=msg): sparse.take(np.array([1, -6])) with pytest.raises(IndexError, match=msg): sparse.take(np.array([1, 5])) with pytest.raises(IndexError, match=msg): sparse.take(np.array([1, 5]), allow_fill=True) def test_take_filling_fill_value(self): # same tests as GH 12631 sparse = SparseArray([np.nan, 0, 1, 0, 4], fill_value=0) result = sparse.take(np.array([1, 0, -1])) expected = SparseArray([0, np.nan, 4], fill_value=0) tm.assert_sp_array_equal(result, expected) # fill_value result = sparse.take(np.array([1, 0, -1]), allow_fill=True) # XXX: behavior change. # the old way of filling self.fill_value doesn't follow EA rules. # It's supposed to be self.dtype.na_value (nan in this case) expected = SparseArray([0, np.nan, np.nan], fill_value=0) tm.assert_sp_array_equal(result, expected) # allow_fill=False result = sparse.take(np.array([1, 0, -1]), allow_fill=False, fill_value=True) expected = SparseArray([0, np.nan, 4], fill_value=0) tm.assert_sp_array_equal(result, expected) msg = "Invalid value in 'indices'." with pytest.raises(ValueError, match=msg): sparse.take(np.array([1, 0, -2]), allow_fill=True) with pytest.raises(ValueError, match=msg): sparse.take(np.array([1, 0, -5]), allow_fill=True) msg = "out of bounds value in 'indices'" with pytest.raises(IndexError, match=msg): sparse.take(np.array([1, -6])) with pytest.raises(IndexError, match=msg): sparse.take(np.array([1, 5])) with pytest.raises(IndexError, match=msg): sparse.take(np.array([1, 5]), fill_value=True) def test_take_filling_all_nan(self): sparse = SparseArray([np.nan, np.nan, np.nan, np.nan, np.nan]) # XXX: did the default kind from take change? result = sparse.take(np.array([1, 0, -1])) expected = SparseArray([np.nan, np.nan, np.nan], kind="block") tm.assert_sp_array_equal(result, expected) result = sparse.take(np.array([1, 0, -1]), fill_value=True) expected = SparseArray([np.nan, np.nan, np.nan], kind="block") tm.assert_sp_array_equal(result, expected) msg = "out of bounds value in 'indices'" with pytest.raises(IndexError, match=msg): sparse.take(np.array([1, -6])) with pytest.raises(IndexError, match=msg): sparse.take(np.array([1, 5])) with pytest.raises(IndexError, match=msg): sparse.take(np.array([1, 5]), fill_value=True) def test_set_item(self): def setitem(): self.arr[5] = 3 def setslice(): self.arr[1:5] = 2 with pytest.raises(TypeError, match="assignment via setitem"): setitem() with pytest.raises(TypeError, match="assignment via setitem"): setslice() def test_constructor_from_too_large_array(self): with pytest.raises(TypeError, match="expected dimension <= 1 data"): SparseArray(np.arange(10).reshape((2, 5))) def test_constructor_from_sparse(self): res = SparseArray(self.zarr) assert res.fill_value == 0 tm.assert_almost_equal(res.sp_values, self.zarr.sp_values) def test_constructor_copy(self): cp = SparseArray(self.arr, copy=True) cp.sp_values[:3] = 0 assert not (self.arr.sp_values[:3] == 0).any() not_copy = SparseArray(self.arr) not_copy.sp_values[:3] = 0 assert (self.arr.sp_values[:3] == 0).all() def test_constructor_bool(self): # GH 10648 data = np.array([False, False, True, True, False, False]) arr = SparseArray(data, fill_value=False, dtype=bool) assert arr.dtype == SparseDtype(bool) tm.assert_numpy_array_equal(arr.sp_values, np.array([True, True])) # Behavior change: np.asarray densifies. # tm.assert_numpy_array_equal(arr.sp_values, np.asarray(arr)) tm.assert_numpy_array_equal(arr.sp_index.indices, np.array([2, 3], np.int32)) dense = arr.to_dense() assert dense.dtype == bool tm.assert_numpy_array_equal(dense, data) def test_constructor_bool_fill_value(self): arr = SparseArray([True, False, True], dtype=None) assert arr.dtype == SparseDtype(np.bool) assert not arr.fill_value arr = SparseArray([True, False, True], dtype=np.bool) assert arr.dtype == SparseDtype(np.bool) assert not arr.fill_value arr = SparseArray([True, False, True], dtype=np.bool, fill_value=True) assert arr.dtype == SparseDtype(np.bool, True) assert arr.fill_value def test_constructor_float32(self): # GH 10648 data = np.array([1.0, np.nan, 3], dtype=np.float32) arr = SparseArray(data, dtype=np.float32) assert arr.dtype == SparseDtype(np.float32) tm.assert_numpy_array_equal(arr.sp_values, np.array([1, 3], dtype=np.float32)) # Behavior change: np.asarray densifies. # tm.assert_numpy_array_equal(arr.sp_values, np.asarray(arr)) tm.assert_numpy_array_equal(arr.sp_index.indices, np.array([0, 2], dtype=np.int32)) dense = arr.to_dense() assert dense.dtype == np.float32 tm.assert_numpy_array_equal(dense, data) def test_astype(self): # float -> float arr = SparseArray([None, None, 0, 2]) result = arr.astype("Sparse[float32]") expected = SparseArray([None, None, 0, 2], dtype=np.dtype("float32")) tm.assert_sp_array_equal(result, expected) dtype = SparseDtype("float64", fill_value=0) result = arr.astype(dtype) expected = SparseArray._simple_new( np.array([0.0, 2.0], dtype=dtype.subtype), IntIndex(4, [2, 3]), dtype) tm.assert_sp_array_equal(result, expected) dtype = SparseDtype("int64", 0) result = arr.astype(dtype) expected = SparseArray._simple_new(np.array([0, 2], dtype=np.int64), IntIndex(4, [2, 3]), dtype) tm.assert_sp_array_equal(result, expected) arr = SparseArray([0, np.nan, 0, 1], fill_value=0) with pytest.raises(ValueError, match="NA"): arr.astype("Sparse[i8]") def test_astype_bool(self): a = SparseArray([1, 0, 0, 1], dtype=SparseDtype(int, 0)) result = a.astype(bool) expected = SparseArray([True, 0, 0, True], dtype=SparseDtype(bool, 0)) tm.assert_sp_array_equal(result, expected) # update fill value result = a.astype(SparseDtype(bool, False)) expected = SparseArray([True, False, False, True], dtype=SparseDtype(bool, False)) tm.assert_sp_array_equal(result, expected) def test_astype_all(self, any_real_dtype): vals = np.array([1, 2, 3]) arr = SparseArray(vals, fill_value=1) typ = np.dtype(any_real_dtype) res = arr.astype(typ) assert res.dtype == SparseDtype(typ, 1) assert res.sp_values.dtype == typ tm.assert_numpy_array_equal(np.asarray(res.to_dense()), vals.astype(typ)) @pytest.mark.parametrize( "array, dtype, expected", [ ( SparseArray([0, 1]), "float", SparseArray([0.0, 1.0], dtype=SparseDtype(float, 0.0)), ), (SparseArray([0, 1]), bool, SparseArray([False, True])), ( SparseArray([0, 1], fill_value=1), bool, SparseArray([False, True], dtype=SparseDtype(bool, True)), ), pytest.param( SparseArray([0, 1]), "datetime64[ns]", SparseArray( np.array([0, 1], dtype="datetime64[ns]"), dtype=SparseDtype("datetime64[ns]", pd.Timestamp("1970")), ), marks=[pytest.mark.xfail(reason="NumPy-7619")], ), ( SparseArray([0, 1, 10]), str, SparseArray(["0", "1", "10"], dtype=SparseDtype(str, "0")), ), (SparseArray(["10", "20"]), float, SparseArray([10.0, 20.0])), ( SparseArray([0, 1, 0]), object, SparseArray([0, 1, 0], dtype=SparseDtype(object, 0)), ), ], ) def test_astype_more(self, array, dtype, expected): result = array.astype(dtype) tm.assert_sp_array_equal(result, expected) def test_astype_nan_raises(self): arr = SparseArray([1.0, np.nan]) with pytest.raises(ValueError, match="Cannot convert non-finite"): arr.astype(int) def test_set_fill_value(self): arr = SparseArray([1.0, np.nan, 2.0], fill_value=np.nan) arr.fill_value = 2 assert arr.fill_value == 2 arr = SparseArray([1, 0, 2], fill_value=0, dtype=np.int64) arr.fill_value = 2 assert arr.fill_value == 2 # XXX: this seems fine? You can construct an integer # sparsearray with NaN fill value, why not update one? # coerces to int # msg = "unable to set fill_value 3\\.1 to int64 dtype" # with pytest.raises(ValueError, match=msg): arr.fill_value = 3.1 assert arr.fill_value == 3.1 # msg = "unable to set fill_value nan to int64 dtype" # with pytest.raises(ValueError, match=msg): arr.fill_value = np.nan assert np.isnan(arr.fill_value) arr = SparseArray([True, False, True], fill_value=False, dtype=np.bool) arr.fill_value = True assert arr.fill_value # coerces to bool # msg = "unable to set fill_value 0 to bool dtype" # with pytest.raises(ValueError, match=msg): arr.fill_value = 0 assert arr.fill_value == 0 # msg = "unable to set fill_value nan to bool dtype" # with pytest.raises(ValueError, match=msg): arr.fill_value = np.nan assert np.isnan(arr.fill_value) @pytest.mark.parametrize("val", [[1, 2, 3], np.array([1, 2]), (1, 2, 3)]) def test_set_fill_invalid_non_scalar(self, val): arr = SparseArray([True, False, True], fill_value=False, dtype=np.bool) msg = "fill_value must be a scalar" with pytest.raises(ValueError, match=msg): arr.fill_value = val def test_copy(self): arr2 = self.arr.copy() assert arr2.sp_values is not self.arr.sp_values assert arr2.sp_index is self.arr.sp_index def test_values_asarray(self): tm.assert_almost_equal(self.arr.to_dense(), self.arr_data) @pytest.mark.parametrize( "data,shape,dtype", [ ([0, 0, 0, 0, 0], (5, ), None), ([], (0, ), None), ([0], (1, ), None), (["A", "A", np.nan, "B"], (4, ), np.object), ], ) def test_shape(self, data, shape, dtype): # GH 21126 out = SparseArray(data, dtype=dtype) assert out.shape == shape @pytest.mark.parametrize( "vals", [ [np.nan, np.nan, np.nan, np.nan, np.nan], [1, np.nan, np.nan, 3, np.nan], [1, np.nan, 0, 3, 0], ], ) @pytest.mark.parametrize("fill_value", [None, 0]) def test_dense_repr(self, vals, fill_value): vals = np.array(vals) arr = SparseArray(vals, fill_value=fill_value) res = arr.to_dense() tm.assert_numpy_array_equal(res, vals) res2 = arr._internal_get_values() tm.assert_numpy_array_equal(res2, vals) def test_getitem(self): def _checkit(i): tm.assert_almost_equal(self.arr[i], self.arr.to_dense()[i]) for i in range(len(self.arr)): _checkit(i) _checkit(-i) def test_getitem_arraylike_mask(self): arr = SparseArray([0, 1, 2]) result = arr[[True, False, True]] expected = SparseArray([0, 2]) tm.assert_sp_array_equal(result, expected) def test_getslice(self): result = self.arr[:-3] exp = SparseArray(self.arr.to_dense()[:-3]) tm.assert_sp_array_equal(result, exp) result = self.arr[-4:] exp = SparseArray(self.arr.to_dense()[-4:]) tm.assert_sp_array_equal(result, exp) # two corner cases from Series result = self.arr[-12:] exp = SparseArray(self.arr) tm.assert_sp_array_equal(result, exp) result = self.arr[:-12] exp = SparseArray(self.arr.to_dense()[:0]) tm.assert_sp_array_equal(result, exp) def test_getslice_tuple(self): dense = np.array([np.nan, 0, 3, 4, 0, 5, np.nan, np.nan, 0]) sparse = SparseArray(dense) res = sparse[4:, ] # noqa: E231 exp = SparseArray(dense[4:, ]) # noqa: E231 tm.assert_sp_array_equal(res, exp) sparse = SparseArray(dense, fill_value=0) res = sparse[4:, ] # noqa: E231 exp = SparseArray(dense[4:, ], fill_value=0) # noqa: E231 tm.assert_sp_array_equal(res, exp) msg = "too many indices for array" with pytest.raises(IndexError, match=msg): sparse[4:, :] with pytest.raises(IndexError, match=msg): # check numpy compat dense[4:, :] def test_boolean_slice_empty(self): arr = SparseArray([0, 1, 2]) res = arr[[False, False, False]] assert res.dtype == arr.dtype @pytest.mark.parametrize( "op", ["add", "sub", "mul", "truediv", "floordiv", "pow"]) def test_binary_operators(self, op): op = getattr(operator, op) data1 = np.random.randn(20) data2 = np.random.randn(20) data1[::2] = np.nan data2[::3] = np.nan arr1 = SparseArray(data1) arr2 = SparseArray(data2) data1[::2] = 3 data2[::3] = 3 farr1 = SparseArray(data1, fill_value=3) farr2 = SparseArray(data2, fill_value=3) def _check_op(op, first, second): res = op(first, second) exp = SparseArray(op(first.to_dense(), second.to_dense()), fill_value=first.fill_value) assert isinstance(res, SparseArray) tm.assert_almost_equal(res.to_dense(), exp.to_dense()) res2 = op(first, second.to_dense()) assert isinstance(res2, SparseArray) tm.assert_sp_array_equal(res, res2) res3 = op(first.to_dense(), second) assert isinstance(res3, SparseArray) tm.assert_sp_array_equal(res, res3) res4 = op(first, 4) assert isinstance(res4, SparseArray) # Ignore this if the actual op raises (e.g. pow). try: exp = op(first.to_dense(), 4) exp_fv = op(first.fill_value, 4) except ValueError: pass else: tm.assert_almost_equal(res4.fill_value, exp_fv) tm.assert_almost_equal(res4.to_dense(), exp) with np.errstate(all="ignore"): for first_arr, second_arr in [(arr1, arr2), (farr1, farr2)]: _check_op(op, first_arr, second_arr) def test_pickle(self): def _check_roundtrip(obj): unpickled = tm.round_trip_pickle(obj) tm.assert_sp_array_equal(unpickled, obj) _check_roundtrip(self.arr) _check_roundtrip(self.zarr) def test_generator_warnings(self): sp_arr = SparseArray([1, 2, 3]) with warnings.catch_warnings(record=True) as w: warnings.filterwarnings(action="always", category=DeprecationWarning) warnings.filterwarnings(action="always", category=PendingDeprecationWarning) for _ in sp_arr: pass assert len(w) == 0 def test_fillna(self): s = SparseArray([1, np.nan, np.nan, 3, np.nan]) res = s.fillna(-1) exp = SparseArray([1, -1, -1, 3, -1], fill_value=-1, dtype=np.float64) tm.assert_sp_array_equal(res, exp) s = SparseArray([1, np.nan, np.nan, 3, np.nan], fill_value=0) res = s.fillna(-1) exp = SparseArray([1, -1, -1, 3, -1], fill_value=0, dtype=np.float64) tm.assert_sp_array_equal(res, exp) s = SparseArray([1, np.nan, 0, 3, 0]) res = s.fillna(-1) exp = SparseArray([1, -1, 0, 3, 0], fill_value=-1, dtype=np.float64) tm.assert_sp_array_equal(res, exp) s = SparseArray([1, np.nan, 0, 3, 0], fill_value=0) res = s.fillna(-1) exp = SparseArray([1, -1, 0, 3, 0], fill_value=0, dtype=np.float64) tm.assert_sp_array_equal(res, exp) s = SparseArray([np.nan, np.nan, np.nan, np.nan]) res = s.fillna(-1) exp = SparseArray([-1, -1, -1, -1], fill_value=-1, dtype=np.float64) tm.assert_sp_array_equal(res, exp) s = SparseArray([np.nan, np.nan, np.nan, np.nan], fill_value=0) res = s.fillna(-1) exp = SparseArray([-1, -1, -1, -1], fill_value=0, dtype=np.float64) tm.assert_sp_array_equal(res, exp) # float dtype's fill_value is np.nan, replaced by -1 s = SparseArray([0.0, 0.0, 0.0, 0.0]) res = s.fillna(-1) exp = SparseArray([0.0, 0.0, 0.0, 0.0], fill_value=-1) tm.assert_sp_array_equal(res, exp) # int dtype shouldn't have missing. No changes. s = SparseArray([0, 0, 0, 0]) assert s.dtype == SparseDtype(np.int64) assert s.fill_value == 0 res = s.fillna(-1) tm.assert_sp_array_equal(res, s) s = SparseArray([0, 0, 0, 0], fill_value=0) assert s.dtype == SparseDtype(np.int64) assert s.fill_value == 0 res = s.fillna(-1) exp = SparseArray([0, 0, 0, 0], fill_value=0) tm.assert_sp_array_equal(res, exp) # fill_value can be nan if there is no missing hole. # only fill_value will be changed s = SparseArray([0, 0, 0, 0], fill_value=np.nan) assert s.dtype == SparseDtype(np.int64, fill_value=np.nan) assert np.isnan(s.fill_value) res = s.fillna(-1) exp = SparseArray([0, 0, 0, 0], fill_value=-1) tm.assert_sp_array_equal(res, exp) def test_fillna_overlap(self): s = SparseArray([1, np.nan, np.nan, 3, np.nan]) # filling with existing value doesn't replace existing value with # fill_value, i.e. existing 3 remains in sp_values res = s.fillna(3) exp = np.array([1, 3, 3, 3, 3], dtype=np.float64) tm.assert_numpy_array_equal(res.to_dense(), exp) s = SparseArray([1, np.nan, np.nan, 3, np.nan], fill_value=0) res = s.fillna(3) exp = SparseArray([1, 3, 3, 3, 3], fill_value=0, dtype=np.float64) tm.assert_sp_array_equal(res, exp) def test_nonzero(self): # Tests regression #21172. sa = SparseArray( [float("nan"), float("nan"), 1, 0, 0, 2, 0, 0, 0, 3, 0, 0]) expected = np.array([2, 5, 9], dtype=np.int32) (result, ) = sa.nonzero() tm.assert_numpy_array_equal(expected, result) sa = SparseArray([0, 0, 1, 0, 0, 2, 0, 0, 0, 3, 0, 0]) (result, ) = sa.nonzero() tm.assert_numpy_array_equal(expected, result)
class TestDataFrameAnalytics: # --------------------------------------------------------------------- # Reductions def test_stat_op_api(self, float_frame, float_string_frame): assert_stat_op_api( "count", float_frame, float_string_frame, has_numeric_only=True ) assert_stat_op_api( "sum", float_frame, float_string_frame, has_numeric_only=True ) assert_stat_op_api("nunique", float_frame, float_string_frame) assert_stat_op_api("mean", float_frame, float_string_frame) assert_stat_op_api("product", float_frame, float_string_frame) assert_stat_op_api("median", float_frame, float_string_frame) assert_stat_op_api("min", float_frame, float_string_frame) assert_stat_op_api("max", float_frame, float_string_frame) assert_stat_op_api("mad", float_frame, float_string_frame) assert_stat_op_api("var", float_frame, float_string_frame) assert_stat_op_api("std", float_frame, float_string_frame) assert_stat_op_api("sem", float_frame, float_string_frame) assert_stat_op_api("median", float_frame, float_string_frame) try: from scipy.stats import skew, kurtosis # noqa:F401 assert_stat_op_api("skew", float_frame, float_string_frame) assert_stat_op_api("kurt", float_frame, float_string_frame) except ImportError: pass def test_stat_op_calc(self, float_frame_with_na, mixed_float_frame): def count(s): return notna(s).sum() def nunique(s): return len(algorithms.unique1d(s.dropna())) def mad(x): return np.abs(x - x.mean()).mean() def var(x): return np.var(x, ddof=1) def std(x): return np.std(x, ddof=1) def sem(x): return np.std(x, ddof=1) / np.sqrt(len(x)) def skewness(x): from scipy.stats import skew # noqa:F811 if len(x) < 3: return np.nan return skew(x, bias=False) def kurt(x): from scipy.stats import kurtosis # noqa:F811 if len(x) < 4: return np.nan return kurtosis(x, bias=False) assert_stat_op_calc( "nunique", nunique, float_frame_with_na, has_skipna=False, check_dtype=False, check_dates=True, ) # GH#32571 check_less_precise is needed on apparently-random # py37-npdev builds and OSX-PY36-min_version builds # mixed types (with upcasting happening) assert_stat_op_calc( "sum", np.sum, mixed_float_frame.astype("float32"), check_dtype=False, check_less_precise=True, ) assert_stat_op_calc( "sum", np.sum, float_frame_with_na, skipna_alternative=np.nansum ) assert_stat_op_calc("mean", np.mean, float_frame_with_na, check_dates=True) assert_stat_op_calc( "product", np.prod, float_frame_with_na, skipna_alternative=np.nanprod ) assert_stat_op_calc("mad", mad, float_frame_with_na) assert_stat_op_calc("var", var, float_frame_with_na) assert_stat_op_calc("std", std, float_frame_with_na) assert_stat_op_calc("sem", sem, float_frame_with_na) assert_stat_op_calc( "count", count, float_frame_with_na, has_skipna=False, check_dtype=False, check_dates=True, ) try: from scipy import skew, kurtosis # noqa:F401 assert_stat_op_calc("skew", skewness, float_frame_with_na) assert_stat_op_calc("kurt", kurt, float_frame_with_na) except ImportError: pass # TODO: Ensure warning isn't emitted in the first place @pytest.mark.filterwarnings("ignore:All-NaN:RuntimeWarning") def test_median(self, float_frame_with_na, int_frame): def wrapper(x): if isna(x).any(): return np.nan return np.median(x) assert_stat_op_calc("median", wrapper, float_frame_with_na, check_dates=True) assert_stat_op_calc( "median", wrapper, int_frame, check_dtype=False, check_dates=True ) @pytest.mark.parametrize( "method", ["sum", "mean", "prod", "var", "std", "skew", "min", "max"] ) def test_stat_operators_attempt_obj_array(self, method): # GH#676 data = { "a": [ -0.00049987540199591344, -0.0016467257772919831, 0.00067695870775883013, ], "b": [-0, -0, 0.0], "c": [ 0.00031111847529610595, 0.0014902627951905339, -0.00094099200035979691, ], } df1 = DataFrame(data, index=["foo", "bar", "baz"], dtype="O") df2 = DataFrame({0: [np.nan, 2], 1: [np.nan, 3], 2: [np.nan, 4]}, dtype=object) for df in [df1, df2]: assert df.values.dtype == np.object_ result = getattr(df, method)(1) expected = getattr(df.astype("f8"), method)(1) if method in ["sum", "prod"]: tm.assert_series_equal(result, expected) @pytest.mark.parametrize("op", ["mean", "std", "var", "skew", "kurt", "sem"]) def test_mixed_ops(self, op): # GH#16116 df = DataFrame( { "int": [1, 2, 3, 4], "float": [1.0, 2.0, 3.0, 4.0], "str": ["a", "b", "c", "d"], } ) result = getattr(df, op)() assert len(result) == 2 with pd.option_context("use_bottleneck", False): result = getattr(df, op)() assert len(result) == 2 def test_reduce_mixed_frame(self): # GH 6806 df = DataFrame( { "bool_data": [True, True, False, False, False], "int_data": [10, 20, 30, 40, 50], "string_data": ["a", "b", "c", "d", "e"], } ) df.reindex(columns=["bool_data", "int_data", "string_data"]) test = df.sum(axis=0) tm.assert_numpy_array_equal( test.values, np.array([2, 150, "abcde"], dtype=object) ) tm.assert_series_equal(test, df.T.sum(axis=1)) def test_nunique(self): df = DataFrame({"A": [1, 1, 1], "B": [1, 2, 3], "C": [1, np.nan, 3]}) tm.assert_series_equal(df.nunique(), Series({"A": 1, "B": 3, "C": 2})) tm.assert_series_equal( df.nunique(dropna=False), Series({"A": 1, "B": 3, "C": 3}) ) tm.assert_series_equal(df.nunique(axis=1), Series({0: 1, 1: 2, 2: 2})) tm.assert_series_equal( df.nunique(axis=1, dropna=False), Series({0: 1, 1: 3, 2: 2}) ) @pytest.mark.parametrize("tz", [None, "UTC"]) def test_mean_mixed_datetime_numeric(self, tz): # https://github.com/pandas-dev/pandas/issues/24752 df = pd.DataFrame({"A": [1, 1], "B": [pd.Timestamp("2000", tz=tz)] * 2}) with tm.assert_produces_warning(FutureWarning): result = df.mean() expected = pd.Series([1.0], index=["A"]) tm.assert_series_equal(result, expected) @pytest.mark.parametrize("tz", [None, "UTC"]) def test_mean_excludes_datetimes(self, tz): # https://github.com/pandas-dev/pandas/issues/24752 # Our long-term desired behavior is unclear, but the behavior in # 0.24.0rc1 was buggy. df = pd.DataFrame({"A": [pd.Timestamp("2000", tz=tz)] * 2}) with tm.assert_produces_warning(FutureWarning): result = df.mean() expected = pd.Series(dtype=np.float64) tm.assert_series_equal(result, expected) def test_mean_mixed_string_decimal(self): # GH 11670 # possible bug when calculating mean of DataFrame? d = [ {"A": 2, "B": None, "C": Decimal("628.00")}, {"A": 1, "B": None, "C": Decimal("383.00")}, {"A": 3, "B": None, "C": Decimal("651.00")}, {"A": 2, "B": None, "C": Decimal("575.00")}, {"A": 4, "B": None, "C": Decimal("1114.00")}, {"A": 1, "B": "TEST", "C": Decimal("241.00")}, {"A": 2, "B": None, "C": Decimal("572.00")}, {"A": 4, "B": None, "C": Decimal("609.00")}, {"A": 3, "B": None, "C": Decimal("820.00")}, {"A": 5, "B": None, "C": Decimal("1223.00")}, ] df = pd.DataFrame(d) result = df.mean() expected = pd.Series([2.7, 681.6], index=["A", "C"]) tm.assert_series_equal(result, expected) def test_var_std(self, datetime_frame): result = datetime_frame.std(ddof=4) expected = datetime_frame.apply(lambda x: x.std(ddof=4)) tm.assert_almost_equal(result, expected) result = datetime_frame.var(ddof=4) expected = datetime_frame.apply(lambda x: x.var(ddof=4)) tm.assert_almost_equal(result, expected) arr = np.repeat(np.random.random((1, 1000)), 1000, 0) result = nanops.nanvar(arr, axis=0) assert not (result < 0).any() with pd.option_context("use_bottleneck", False): result = nanops.nanvar(arr, axis=0) assert not (result < 0).any() @pytest.mark.parametrize("meth", ["sem", "var", "std"]) def test_numeric_only_flag(self, meth): # GH 9201 df1 = DataFrame(np.random.randn(5, 3), columns=["foo", "bar", "baz"]) # set one entry to a number in str format df1.loc[0, "foo"] = "100" df2 = DataFrame(np.random.randn(5, 3), columns=["foo", "bar", "baz"]) # set one entry to a non-number str df2.loc[0, "foo"] = "a" result = getattr(df1, meth)(axis=1, numeric_only=True) expected = getattr(df1[["bar", "baz"]], meth)(axis=1) tm.assert_series_equal(expected, result) result = getattr(df2, meth)(axis=1, numeric_only=True) expected = getattr(df2[["bar", "baz"]], meth)(axis=1) tm.assert_series_equal(expected, result) # df1 has all numbers, df2 has a letter inside msg = r"unsupported operand type\(s\) for -: 'float' and 'str'" with pytest.raises(TypeError, match=msg): getattr(df1, meth)(axis=1, numeric_only=False) msg = "could not convert string to float: 'a'" with pytest.raises(TypeError, match=msg): getattr(df2, meth)(axis=1, numeric_only=False) def test_sem(self, datetime_frame): result = datetime_frame.sem(ddof=4) expected = datetime_frame.apply(lambda x: x.std(ddof=4) / np.sqrt(len(x))) tm.assert_almost_equal(result, expected) arr = np.repeat(np.random.random((1, 1000)), 1000, 0) result = nanops.nansem(arr, axis=0) assert not (result < 0).any() with pd.option_context("use_bottleneck", False): result = nanops.nansem(arr, axis=0) assert not (result < 0).any() @td.skip_if_no_scipy def test_kurt(self): index = MultiIndex( levels=[["bar"], ["one", "two", "three"], [0, 1]], codes=[[0, 0, 0, 0, 0, 0], [0, 1, 2, 0, 1, 2], [0, 1, 0, 1, 0, 1]], ) df = DataFrame(np.random.randn(6, 3), index=index) kurt = df.kurt() kurt2 = df.kurt(level=0).xs("bar") tm.assert_series_equal(kurt, kurt2, check_names=False) assert kurt.name is None assert kurt2.name == "bar" @pytest.mark.parametrize( "dropna, expected", [ ( True, { "A": [12], "B": [10.0], "C": [1.0], "D": ["a"], "E": Categorical(["a"], categories=["a"]), "F": to_datetime(["2000-1-2"]), "G": to_timedelta(["1 days"]), }, ), ( False, { "A": [12], "B": [10.0], "C": [np.nan], "D": np.array([np.nan], dtype=object), "E": Categorical([np.nan], categories=["a"]), "F": [pd.NaT], "G": to_timedelta([pd.NaT]), }, ), ( True, { "H": [8, 9, np.nan, np.nan], "I": [8, 9, np.nan, np.nan], "J": [1, np.nan, np.nan, np.nan], "K": Categorical(["a", np.nan, np.nan, np.nan], categories=["a"]), "L": to_datetime(["2000-1-2", "NaT", "NaT", "NaT"]), "M": to_timedelta(["1 days", "nan", "nan", "nan"]), "N": [0, 1, 2, 3], }, ), ( False, { "H": [8, 9, np.nan, np.nan], "I": [8, 9, np.nan, np.nan], "J": [1, np.nan, np.nan, np.nan], "K": Categorical([np.nan, "a", np.nan, np.nan], categories=["a"]), "L": to_datetime(["NaT", "2000-1-2", "NaT", "NaT"]), "M": to_timedelta(["nan", "1 days", "nan", "nan"]), "N": [0, 1, 2, 3], }, ), ], ) def test_mode_dropna(self, dropna, expected): df = DataFrame( { "A": [12, 12, 19, 11], "B": [10, 10, np.nan, 3], "C": [1, np.nan, np.nan, np.nan], "D": [np.nan, np.nan, "a", np.nan], "E": Categorical([np.nan, np.nan, "a", np.nan]), "F": to_datetime(["NaT", "2000-1-2", "NaT", "NaT"]), "G": to_timedelta(["1 days", "nan", "nan", "nan"]), "H": [8, 8, 9, 9], "I": [9, 9, 8, 8], "J": [1, 1, np.nan, np.nan], "K": Categorical(["a", np.nan, "a", np.nan]), "L": to_datetime(["2000-1-2", "2000-1-2", "NaT", "NaT"]), "M": to_timedelta(["1 days", "nan", "1 days", "nan"]), "N": np.arange(4, dtype="int64"), } ) result = df[sorted(expected.keys())].mode(dropna=dropna) expected = DataFrame(expected) tm.assert_frame_equal(result, expected) def test_mode_sortwarning(self): # Check for the warning that is raised when the mode # results cannot be sorted df = DataFrame({"A": [np.nan, np.nan, "a", "a"]}) expected = DataFrame({"A": ["a", np.nan]}) with tm.assert_produces_warning(UserWarning, check_stacklevel=False): result = df.mode(dropna=False) result = result.sort_values(by="A").reset_index(drop=True) tm.assert_frame_equal(result, expected) def test_operators_timedelta64(self): df = DataFrame( dict( A=date_range("2012-1-1", periods=3, freq="D"), B=date_range("2012-1-2", periods=3, freq="D"), C=Timestamp("20120101") - timedelta(minutes=5, seconds=5), ) ) diffs = DataFrame(dict(A=df["A"] - df["C"], B=df["A"] - df["B"])) # min result = diffs.min() assert result[0] == diffs.loc[0, "A"] assert result[1] == diffs.loc[0, "B"] result = diffs.min(axis=1) assert (result == diffs.loc[0, "B"]).all() # max result = diffs.max() assert result[0] == diffs.loc[2, "A"] assert result[1] == diffs.loc[2, "B"] result = diffs.max(axis=1) assert (result == diffs["A"]).all() # abs result = diffs.abs() result2 = abs(diffs) expected = DataFrame(dict(A=df["A"] - df["C"], B=df["B"] - df["A"])) tm.assert_frame_equal(result, expected) tm.assert_frame_equal(result2, expected) # mixed frame mixed = diffs.copy() mixed["C"] = "foo" mixed["D"] = 1 mixed["E"] = 1.0 mixed["F"] = Timestamp("20130101") # results in an object array result = mixed.min() expected = Series( [ pd.Timedelta(timedelta(seconds=5 * 60 + 5)), pd.Timedelta(timedelta(days=-1)), "foo", 1, 1.0, Timestamp("20130101"), ], index=mixed.columns, ) tm.assert_series_equal(result, expected) # excludes numeric result = mixed.min(axis=1) expected = Series([1, 1, 1.0], index=[0, 1, 2]) tm.assert_series_equal(result, expected) # works when only those columns are selected result = mixed[["A", "B"]].min(1) expected = Series([timedelta(days=-1)] * 3) tm.assert_series_equal(result, expected) result = mixed[["A", "B"]].min() expected = Series( [timedelta(seconds=5 * 60 + 5), timedelta(days=-1)], index=["A", "B"] ) tm.assert_series_equal(result, expected) # GH 3106 df = DataFrame( { "time": date_range("20130102", periods=5), "time2": date_range("20130105", periods=5), } ) df["off1"] = df["time2"] - df["time"] assert df["off1"].dtype == "timedelta64[ns]" df["off2"] = df["time"] - df["time2"] df._consolidate_inplace() assert df["off1"].dtype == "timedelta64[ns]" assert df["off2"].dtype == "timedelta64[ns]" def test_sum_corner(self): empty_frame = DataFrame() axis0 = empty_frame.sum(0) axis1 = empty_frame.sum(1) assert isinstance(axis0, Series) assert isinstance(axis1, Series) assert len(axis0) == 0 assert len(axis1) == 0 @pytest.mark.parametrize("method, unit", [("sum", 0), ("prod", 1)]) def test_sum_prod_nanops(self, method, unit): idx = ["a", "b", "c"] df = pd.DataFrame( {"a": [unit, unit], "b": [unit, np.nan], "c": [np.nan, np.nan]} ) # The default result = getattr(df, method) expected = pd.Series([unit, unit, unit], index=idx, dtype="float64") # min_count=1 result = getattr(df, method)(min_count=1) expected = pd.Series([unit, unit, np.nan], index=idx) tm.assert_series_equal(result, expected) # min_count=0 result = getattr(df, method)(min_count=0) expected = pd.Series([unit, unit, unit], index=idx, dtype="float64") tm.assert_series_equal(result, expected) result = getattr(df.iloc[1:], method)(min_count=1) expected = pd.Series([unit, np.nan, np.nan], index=idx) tm.assert_series_equal(result, expected) # min_count > 1 df = pd.DataFrame({"A": [unit] * 10, "B": [unit] * 5 + [np.nan] * 5}) result = getattr(df, method)(min_count=5) expected = pd.Series(result, index=["A", "B"]) tm.assert_series_equal(result, expected) result = getattr(df, method)(min_count=6) expected = pd.Series(result, index=["A", "B"]) tm.assert_series_equal(result, expected) def test_sum_nanops_timedelta(self): # prod isn't defined on timedeltas idx = ["a", "b", "c"] df = pd.DataFrame({"a": [0, 0], "b": [0, np.nan], "c": [np.nan, np.nan]}) df2 = df.apply(pd.to_timedelta) # 0 by default result = df2.sum() expected = pd.Series([0, 0, 0], dtype="m8[ns]", index=idx) tm.assert_series_equal(result, expected) # min_count=0 result = df2.sum(min_count=0) tm.assert_series_equal(result, expected) # min_count=1 result = df2.sum(min_count=1) expected = pd.Series([0, 0, np.nan], dtype="m8[ns]", index=idx) tm.assert_series_equal(result, expected) def test_sum_object(self, float_frame): values = float_frame.values.astype(int) frame = DataFrame(values, index=float_frame.index, columns=float_frame.columns) deltas = frame * timedelta(1) deltas.sum() def test_sum_bool(self, float_frame): # ensure this works, bug report bools = np.isnan(float_frame) bools.sum(1) bools.sum(0) def test_sum_mixed_datetime(self): # GH#30886 df = pd.DataFrame( {"A": pd.date_range("2000", periods=4), "B": [1, 2, 3, 4]} ).reindex([2, 3, 4]) result = df.sum() expected = pd.Series({"B": 7.0}) tm.assert_series_equal(result, expected) def test_mean_corner(self, float_frame, float_string_frame): # unit test when have object data the_mean = float_string_frame.mean(axis=0) the_sum = float_string_frame.sum(axis=0, numeric_only=True) tm.assert_index_equal(the_sum.index, the_mean.index) assert len(the_mean.index) < len(float_string_frame.columns) # xs sum mixed type, just want to know it works... the_mean = float_string_frame.mean(axis=1) the_sum = float_string_frame.sum(axis=1, numeric_only=True) tm.assert_index_equal(the_sum.index, the_mean.index) # take mean of boolean column float_frame["bool"] = float_frame["A"] > 0 means = float_frame.mean(0) assert means["bool"] == float_frame["bool"].values.mean() def test_mean_datetimelike(self): # GH#24757 check that datetimelike are excluded by default, handled # correctly with numeric_only=True df = pd.DataFrame( { "A": np.arange(3), "B": pd.date_range("2016-01-01", periods=3), "C": pd.timedelta_range("1D", periods=3), "D": pd.period_range("2016", periods=3, freq="A"), } ) result = df.mean(numeric_only=True) expected = pd.Series({"A": 1.0}) tm.assert_series_equal(result, expected) with tm.assert_produces_warning(FutureWarning): # in the future datetime columns will be included result = df.mean() expected = pd.Series({"A": 1.0, "C": df.loc[1, "C"]}) tm.assert_series_equal(result, expected) def test_mean_datetimelike_numeric_only_false(self): df = pd.DataFrame( { "A": np.arange(3), "B": pd.date_range("2016-01-01", periods=3), "C": pd.timedelta_range("1D", periods=3), } ) # datetime(tz) and timedelta work result = df.mean(numeric_only=False) expected = pd.Series({"A": 1, "B": df.loc[1, "B"], "C": df.loc[1, "C"]}) tm.assert_series_equal(result, expected) # mean of period is not allowed df["D"] = pd.period_range("2016", periods=3, freq="A") with pytest.raises(TypeError, match="mean is not implemented for Period"): df.mean(numeric_only=False) def test_mean_extensionarray_numeric_only_true(self): # https://github.com/pandas-dev/pandas/issues/33256 arr = np.random.randint(1000, size=(10, 5)) df = pd.DataFrame(arr, dtype="Int64") result = df.mean(numeric_only=True) expected = pd.DataFrame(arr).mean() tm.assert_series_equal(result, expected) def test_stats_mixed_type(self, float_string_frame): # don't blow up float_string_frame.std(1) float_string_frame.var(1) float_string_frame.mean(1) float_string_frame.skew(1) def test_sum_bools(self): df = DataFrame(index=range(1), columns=range(10)) bools = isna(df) assert bools.sum(axis=1)[0] == 10 # ---------------------------------------------------------------------- # Index of max / min def test_idxmin(self, float_frame, int_frame): frame = float_frame frame.iloc[5:10] = np.nan frame.iloc[15:20, -2:] = np.nan for skipna in [True, False]: for axis in [0, 1]: for df in [frame, int_frame]: result = df.idxmin(axis=axis, skipna=skipna) expected = df.apply(Series.idxmin, axis=axis, skipna=skipna) tm.assert_series_equal(result, expected) msg = "No axis named 2 for object type DataFrame" with pytest.raises(ValueError, match=msg): frame.idxmin(axis=2) def test_idxmax(self, float_frame, int_frame): frame = float_frame frame.iloc[5:10] = np.nan frame.iloc[15:20, -2:] = np.nan for skipna in [True, False]: for axis in [0, 1]: for df in [frame, int_frame]: result = df.idxmax(axis=axis, skipna=skipna) expected = df.apply(Series.idxmax, axis=axis, skipna=skipna) tm.assert_series_equal(result, expected) msg = "No axis named 2 for object type DataFrame" with pytest.raises(ValueError, match=msg): frame.idxmax(axis=2) # ---------------------------------------------------------------------- # Logical reductions @pytest.mark.parametrize("opname", ["any", "all"]) def test_any_all(self, opname, bool_frame_with_na, float_string_frame): assert_bool_op_calc( opname, getattr(np, opname), bool_frame_with_na, has_skipna=True ) assert_bool_op_api( opname, bool_frame_with_na, float_string_frame, has_bool_only=True ) def test_any_all_extra(self): df = DataFrame( { "A": [True, False, False], "B": [True, True, False], "C": [True, True, True], }, index=["a", "b", "c"], ) result = df[["A", "B"]].any(1) expected = Series([True, True, False], index=["a", "b", "c"]) tm.assert_series_equal(result, expected) result = df[["A", "B"]].any(1, bool_only=True) tm.assert_series_equal(result, expected) result = df.all(1) expected = Series([True, False, False], index=["a", "b", "c"]) tm.assert_series_equal(result, expected) result = df.all(1, bool_only=True) tm.assert_series_equal(result, expected) # Axis is None result = df.all(axis=None).item() assert result is False result = df.any(axis=None).item() assert result is True result = df[["C"]].all(axis=None).item() assert result is True def test_any_datetime(self): # GH 23070 float_data = [1, np.nan, 3, np.nan] datetime_data = [ pd.Timestamp("1960-02-15"), pd.Timestamp("1960-02-16"), pd.NaT, pd.NaT, ] df = DataFrame({"A": float_data, "B": datetime_data}) result = df.any(1) expected = Series([True, True, True, False]) tm.assert_series_equal(result, expected) def test_any_all_bool_only(self): # GH 25101 df = DataFrame( {"col1": [1, 2, 3], "col2": [4, 5, 6], "col3": [None, None, None]} ) result = df.all(bool_only=True) expected = Series(dtype=np.bool_) tm.assert_series_equal(result, expected) df = DataFrame( { "col1": [1, 2, 3], "col2": [4, 5, 6], "col3": [None, None, None], "col4": [False, False, True], } ) result = df.all(bool_only=True) expected = Series({"col4": False}) tm.assert_series_equal(result, expected) @pytest.mark.parametrize( "func, data, expected", [ (np.any, {}, False), (np.all, {}, True), (np.any, {"A": []}, False), (np.all, {"A": []}, True), (np.any, {"A": [False, False]}, False), (np.all, {"A": [False, False]}, False), (np.any, {"A": [True, False]}, True), (np.all, {"A": [True, False]}, False), (np.any, {"A": [True, True]}, True), (np.all, {"A": [True, True]}, True), (np.any, {"A": [False], "B": [False]}, False), (np.all, {"A": [False], "B": [False]}, False), (np.any, {"A": [False, False], "B": [False, True]}, True), (np.all, {"A": [False, False], "B": [False, True]}, False), # other types (np.all, {"A": pd.Series([0.0, 1.0], dtype="float")}, False), (np.any, {"A": pd.Series([0.0, 1.0], dtype="float")}, True), (np.all, {"A": pd.Series([0, 1], dtype=int)}, False), (np.any, {"A": pd.Series([0, 1], dtype=int)}, True), pytest.param( np.all, {"A": pd.Series([0, 1], dtype="M8[ns]")}, False, marks=[td.skip_if_np_lt("1.15")], ), pytest.param( np.any, {"A": pd.Series([0, 1], dtype="M8[ns]")}, True, marks=[td.skip_if_np_lt("1.15")], ), pytest.param( np.all, {"A": pd.Series([1, 2], dtype="M8[ns]")}, True, marks=[td.skip_if_np_lt("1.15")], ), pytest.param( np.any, {"A": pd.Series([1, 2], dtype="M8[ns]")}, True, marks=[td.skip_if_np_lt("1.15")], ), pytest.param( np.all, {"A": pd.Series([0, 1], dtype="m8[ns]")}, False, marks=[td.skip_if_np_lt("1.15")], ), pytest.param( np.any, {"A": pd.Series([0, 1], dtype="m8[ns]")}, True, marks=[td.skip_if_np_lt("1.15")], ), pytest.param( np.all, {"A": pd.Series([1, 2], dtype="m8[ns]")}, True, marks=[td.skip_if_np_lt("1.15")], ), pytest.param( np.any, {"A": pd.Series([1, 2], dtype="m8[ns]")}, True, marks=[td.skip_if_np_lt("1.15")], ), (np.all, {"A": pd.Series([0, 1], dtype="category")}, False), (np.any, {"A": pd.Series([0, 1], dtype="category")}, True), (np.all, {"A": pd.Series([1, 2], dtype="category")}, True), (np.any, {"A": pd.Series([1, 2], dtype="category")}, True), # Mix GH#21484 pytest.param( np.all, { "A": pd.Series([10, 20], dtype="M8[ns]"), "B": pd.Series([10, 20], dtype="m8[ns]"), }, True, # In 1.13.3 and 1.14 np.all(df) returns a Timedelta here marks=[td.skip_if_np_lt("1.15")], ), ], ) def test_any_all_np_func(self, func, data, expected): # GH 19976 data = DataFrame(data) result = func(data) assert isinstance(result, np.bool_) assert result.item() is expected # method version result = getattr(DataFrame(data), func.__name__)(axis=None) assert isinstance(result, np.bool_) assert result.item() is expected def test_any_all_object(self): # GH 19976 result = np.all(DataFrame(columns=["a", "b"])).item() assert result is True result = np.any(DataFrame(columns=["a", "b"])).item() assert result is False @pytest.mark.parametrize("method", ["any", "all"]) def test_any_all_level_axis_none_raises(self, method): df = DataFrame( {"A": 1}, index=MultiIndex.from_product( [["A", "B"], ["a", "b"]], names=["out", "in"] ), ) xpr = "Must specify 'axis' when aggregating by level." with pytest.raises(ValueError, match=xpr): getattr(df, method)(axis=None, level="out") # --------------------------------------------------------------------- # Matrix-like def test_matmul(self): # matmul test is for GH 10259 a = DataFrame( np.random.randn(3, 4), index=["a", "b", "c"], columns=["p", "q", "r", "s"] ) b = DataFrame( np.random.randn(4, 2), index=["p", "q", "r", "s"], columns=["one", "two"] ) # DataFrame @ DataFrame result = operator.matmul(a, b) expected = DataFrame( np.dot(a.values, b.values), index=["a", "b", "c"], columns=["one", "two"] ) tm.assert_frame_equal(result, expected) # DataFrame @ Series result = operator.matmul(a, b.one) expected = Series(np.dot(a.values, b.one.values), index=["a", "b", "c"]) tm.assert_series_equal(result, expected) # np.array @ DataFrame result = operator.matmul(a.values, b) assert isinstance(result, DataFrame) assert result.columns.equals(b.columns) assert result.index.equals(pd.Index(range(3))) expected = np.dot(a.values, b.values) tm.assert_almost_equal(result.values, expected) # nested list @ DataFrame (__rmatmul__) result = operator.matmul(a.values.tolist(), b) expected = DataFrame( np.dot(a.values, b.values), index=["a", "b", "c"], columns=["one", "two"] ) tm.assert_almost_equal(result.values, expected.values) # mixed dtype DataFrame @ DataFrame a["q"] = a.q.round().astype(int) result = operator.matmul(a, b) expected = DataFrame( np.dot(a.values, b.values), index=["a", "b", "c"], columns=["one", "two"] ) tm.assert_frame_equal(result, expected) # different dtypes DataFrame @ DataFrame a = a.astype(int) result = operator.matmul(a, b) expected = DataFrame( np.dot(a.values, b.values), index=["a", "b", "c"], columns=["one", "two"] ) tm.assert_frame_equal(result, expected) # unaligned df = DataFrame(np.random.randn(3, 4), index=[1, 2, 3], columns=range(4)) df2 = DataFrame(np.random.randn(5, 3), index=range(5), columns=[1, 2, 3]) with pytest.raises(ValueError, match="aligned"): operator.matmul(df, df2) # --------------------------------------------------------------------- # Unsorted def test_series_broadcasting(self): # smoke test for numpy warnings # GH 16378, GH 16306 df = DataFrame([1.0, 1.0, 1.0]) df_nan = DataFrame({"A": [np.nan, 2.0, np.nan]}) s = Series([1, 1, 1]) s_nan = Series([np.nan, np.nan, 1]) with tm.assert_produces_warning(None): df_nan.clip(lower=s, axis=0) for op in ["lt", "le", "gt", "ge", "eq", "ne"]: getattr(df, op)(s_nan, axis=0)