def test_reductions_1D(dtype):
x = np.arange(5).astype(dtype)
a = da.from_array(x, chunks=(2,))
reduction_1d_test(da.sum, a, np.sum, x)
reduction_1d_test(da.prod, a, np.prod, x)
reduction_1d_test(da.mean, a, np.mean, x)
reduction_1d_test(da.var, a, np.var, x)
reduction_1d_test(da.std, a, np.std, x)
reduction_1d_test(da.min, a, np.min, x, False)
reduction_1d_test(da.max, a, np.max, x, False)
reduction_1d_test(da.any, a, np.any, x, False)
reduction_1d_test(da.all, a, np.all, x, False)
reduction_1d_test(da.nansum, a, np.nansum, x)
with ignoring(AttributeError):
reduction_1d_test(da.nanprod, a, np.nanprod, x)
reduction_1d_test(da.nanmean, a, np.mean, x)
reduction_1d_test(da.nanvar, a, np.var, x)
reduction_1d_test(da.nanstd, a, np.std, x)
reduction_1d_test(da.nanmin, a, np.nanmin, x, False)
reduction_1d_test(da.nanmax, a, np.nanmax, x, False)
assert eq(da.argmax(a, axis=0), np.argmax(x, axis=0))
assert eq(da.argmin(a, axis=0), np.argmin(x, axis=0))
assert eq(da.nanargmax(a, axis=0), np.nanargmax(x, axis=0))
assert eq(da.nanargmin(a, axis=0), np.nanargmin(x, axis=0))
assert eq(da.argmax(a, axis=0, split_every=2), np.argmax(x, axis=0))
assert eq(da.argmin(a, axis=0, split_every=2), np.argmin(x, axis=0))
assert eq(da.nanargmax(a, axis=0, split_every=2), np.nanargmax(x, axis=0))
assert eq(da.nanargmin(a, axis=0, split_every=2), np.nanargmin(x, axis=0))
def test_reductions_2D_int():
x = np.arange(1, 122).reshape((11, 11)).astype('i4')
a = da.from_array(x, chunks=(4, 4))
reduction_2d_test(da.sum, a, np.sum, x)
reduction_2d_test(da.prod, a, np.prod, x)
reduction_2d_test(da.mean, a, np.mean, x)
reduction_2d_test(da.var, a, np.var, x, False) # Difference in dtype algo
reduction_2d_test(da.std, a, np.std, x, False) # Difference in dtype algo
reduction_2d_test(da.min, a, np.min, x, False)
reduction_2d_test(da.max, a, np.max, x, False)
reduction_2d_test(da.any, a, np.any, x, False)
reduction_2d_test(da.all, a, np.all, x, False)
reduction_2d_test(da.nansum, a, np.nansum, x)
with ignoring(AttributeError):
reduction_2d_test(da.nanprod, a, np.nanprod, x)
reduction_2d_test(da.nanmean, a, np.mean, x)
reduction_2d_test(da.nanvar, a, np.nanvar, x,
False) # Difference in dtype algo
reduction_2d_test(da.nanstd, a, np.nanstd, x,
False) # Difference in dtype algo
reduction_2d_test(da.nanmin, a, np.nanmin, x, False)
reduction_2d_test(da.nanmax, a, np.nanmax, x, False)
assert eq(da.argmax(a, axis=0), np.argmax(x, axis=0))
assert eq(da.argmin(a, axis=0), np.argmin(x, axis=0))
assert eq(da.nanargmax(a, axis=0), np.nanargmax(x, axis=0))
assert eq(da.nanargmin(a, axis=0), np.nanargmin(x, axis=0))
assert eq(da.argmax(a, axis=1), np.argmax(x, axis=1))
assert eq(da.argmin(a, axis=1), np.argmin(x, axis=1))
assert eq(da.nanargmax(a, axis=1), np.nanargmax(x, axis=1))
assert eq(da.nanargmin(a, axis=1), np.nanargmin(x, axis=1))
def test_reductions_2D_int():
x = np.arange(1, 122).reshape((11, 11)).astype('i4')
a = da.from_array(x, chunks=(4, 4))
reduction_2d_test(da.sum, a, np.sum, x)
reduction_2d_test(da.prod, a, np.prod, x)
reduction_2d_test(da.mean, a, np.mean, x)
reduction_2d_test(da.var, a, np.var, x, False) # Difference in dtype algo
reduction_2d_test(da.std, a, np.std, x, False) # Difference in dtype algo
reduction_2d_test(da.min, a, np.min, x, False)
reduction_2d_test(da.max, a, np.max, x, False)
reduction_2d_test(da.any, a, np.any, x, False)
reduction_2d_test(da.all, a, np.all, x, False)
reduction_2d_test(da.nansum, a, np.nansum, x)
with ignoring(AttributeError):
reduction_2d_test(da.nanprod, a, np.nanprod, x)
reduction_2d_test(da.nanmean, a, np.mean, x)
reduction_2d_test(da.nanvar, a, np.nanvar, x, False) # Difference in dtype algo
reduction_2d_test(da.nanstd, a, np.nanstd, x, False) # Difference in dtype algo
reduction_2d_test(da.nanmin, a, np.nanmin, x, False)
reduction_2d_test(da.nanmax, a, np.nanmax, x, False)
assert eq(da.argmax(a, axis=0), np.argmax(x, axis=0))
assert eq(da.argmin(a, axis=0), np.argmin(x, axis=0))
assert eq(da.nanargmax(a, axis=0), np.nanargmax(x, axis=0))
assert eq(da.nanargmin(a, axis=0), np.nanargmin(x, axis=0))
assert eq(da.argmax(a, axis=1), np.argmax(x, axis=1))
assert eq(da.argmin(a, axis=1), np.argmin(x, axis=1))
assert eq(da.nanargmax(a, axis=1), np.nanargmax(x, axis=1))
assert eq(da.nanargmin(a, axis=1), np.nanargmin(x, axis=1))
def test_reductions_2D_nans():
# chunks are a mix of some/all/no NaNs
x = np.full((4, 4), np.nan)
x[:2, :2] = np.array([[1, 2], [3, 4]])
x[2, 2] = 5
x[3, 3] = 6
a = da.from_array(x, chunks=(2, 2))
reduction_2d_test(da.sum, a, np.sum, x, False, False)
reduction_2d_test(da.prod, a, np.prod, x, False, False)
reduction_2d_test(da.mean, a, np.mean, x, False, False)
reduction_2d_test(da.var, a, np.var, x, False, False)
reduction_2d_test(da.std, a, np.std, x, False, False)
reduction_2d_test(da.min, a, np.min, x, False, False)
reduction_2d_test(da.max, a, np.max, x, False, False)
reduction_2d_test(da.any, a, np.any, x, False, False)
reduction_2d_test(da.all, a, np.all, x, False, False)
reduction_2d_test(da.nansum, a, np.nansum, x, False, False)
reduction_2d_test(da.nanprod, a, np.nanprod, x, False, False)
reduction_2d_test(da.nanmean, a, np.nanmean, x, False, False)
with pytest.warns(None): # division by 0 warning
reduction_2d_test(da.nanvar, a, np.nanvar, x, False, False)
with pytest.warns(None): # division by 0 warning
reduction_2d_test(da.nanstd, a, np.nanstd, x, False, False)
with pytest.warns(None): # all NaN axis warning
reduction_2d_test(da.nanmin, a, np.nanmin, x, False, False)
with pytest.warns(None): # all NaN axis warning
reduction_2d_test(da.nanmax, a, np.nanmax, x, False, False)
assert_eq(da.argmax(a), np.argmax(x))
assert_eq(da.argmin(a), np.argmin(x))
with pytest.warns(None): # all NaN axis warning
assert_eq(da.nanargmax(a), np.nanargmax(x))
with pytest.warns(None): # all NaN axis warning
assert_eq(da.nanargmin(a), np.nanargmin(x))
assert_eq(da.argmax(a, axis=0), np.argmax(x, axis=0))
assert_eq(da.argmin(a, axis=0), np.argmin(x, axis=0))
with pytest.warns(None): # all NaN axis warning
assert_eq(da.nanargmax(a, axis=0), np.nanargmax(x, axis=0))
with pytest.warns(None): # all NaN axis warning
assert_eq(da.nanargmin(a, axis=0), np.nanargmin(x, axis=0))
assert_eq(da.argmax(a, axis=1), np.argmax(x, axis=1))
assert_eq(da.argmin(a, axis=1), np.argmin(x, axis=1))
with pytest.warns(None): # all NaN axis warning
assert_eq(da.nanargmax(a, axis=1), np.nanargmax(x, axis=1))
with pytest.warns(None): # all NaN axis warning
assert_eq(da.nanargmin(a, axis=1), np.nanargmin(x, axis=1))
def test_reductions_2D_nans():
# chunks are a mix of some/all/no NaNs
x = np.full((4, 4), np.nan)
x[:2, :2] = np.array([[1, 2], [3, 4]])
x[2, 2] = 5
x[3, 3] = 6
a = da.from_array(x, chunks=(2, 2))
reduction_2d_test(da.sum, a, np.sum, x, False, False)
reduction_2d_test(da.prod, a, np.prod, x, False, False)
reduction_2d_test(da.mean, a, np.mean, x, False, False)
reduction_2d_test(da.var, a, np.var, x, False, False)
reduction_2d_test(da.std, a, np.std, x, False, False)
reduction_2d_test(da.min, a, np.min, x, False, False)
reduction_2d_test(da.max, a, np.max, x, False, False)
reduction_2d_test(da.any, a, np.any, x, False, False)
reduction_2d_test(da.all, a, np.all, x, False, False)
reduction_2d_test(da.nansum, a, np.nansum, x, False, False)
reduction_2d_test(da.nanprod, a, nanprod, x, False, False)
reduction_2d_test(da.nanmean, a, np.nanmean, x, False, False)
with pytest.warns(None): # division by 0 warning
reduction_2d_test(da.nanvar, a, np.nanvar, x, False, False)
with pytest.warns(None): # division by 0 warning
reduction_2d_test(da.nanstd, a, np.nanstd, x, False, False)
with pytest.warns(None): # all NaN axis warning
reduction_2d_test(da.nanmin, a, np.nanmin, x, False, False)
with pytest.warns(None): # all NaN axis warning
reduction_2d_test(da.nanmax, a, np.nanmax, x, False, False)
assert_eq(da.argmax(a), np.argmax(x))
assert_eq(da.argmin(a), np.argmin(x))
with pytest.warns(None): # all NaN axis warning
assert_eq(da.nanargmax(a), np.nanargmax(x))
with pytest.warns(None): # all NaN axis warning
assert_eq(da.nanargmin(a), np.nanargmin(x))
assert_eq(da.argmax(a, axis=0), np.argmax(x, axis=0))
assert_eq(da.argmin(a, axis=0), np.argmin(x, axis=0))
with pytest.warns(None): # all NaN axis warning
assert_eq(da.nanargmax(a, axis=0), np.nanargmax(x, axis=0))
with pytest.warns(None): # all NaN axis warning
assert_eq(da.nanargmin(a, axis=0), np.nanargmin(x, axis=0))
assert_eq(da.argmax(a, axis=1), np.argmax(x, axis=1))
assert_eq(da.argmin(a, axis=1), np.argmin(x, axis=1))
with pytest.warns(None): # all NaN axis warning
assert_eq(da.nanargmax(a, axis=1), np.nanargmax(x, axis=1))
with pytest.warns(None): # all NaN axis warning
assert_eq(da.nanargmin(a, axis=1), np.nanargmin(x, axis=1))
def test_reductions_1D_int():
x = np.arange(5).astype('i4')
a = da.from_array(x, chunks=(2, ))
reduction_1d_test(da.sum, a, np.sum, x)
reduction_1d_test(da.prod, a, np.prod, x)
reduction_1d_test(da.mean, a, np.mean, x)
reduction_1d_test(da.var, a, np.var, x)
reduction_1d_test(da.std, a, np.std, x)
reduction_1d_test(da.min, a, np.min, x, False)
reduction_1d_test(da.max, a, np.max, x, False)
reduction_1d_test(da.any, a, np.any, x, False)
reduction_1d_test(da.all, a, np.all, x, False)
reduction_1d_test(da.nansum, a, np.nansum, x)
with ignoring(AttributeError):
reduction_1d_test(da.nanprod, a, np.nanprod, x)
reduction_1d_test(da.nanmean, a, np.mean, x)
reduction_1d_test(da.nanvar, a, np.var, x)
reduction_1d_test(da.nanstd, a, np.std, x)
reduction_1d_test(da.nanmin, a, np.nanmin, x, False)
reduction_1d_test(da.nanmax, a, np.nanmax, x, False)
assert eq(da.argmax(a, axis=0), np.argmax(x, axis=0))
assert eq(da.argmin(a, axis=0), np.argmin(x, axis=0))
assert eq(da.nanargmax(a, axis=0), np.nanargmax(x, axis=0))
assert eq(da.nanargmin(a, axis=0), np.nanargmin(x, axis=0))
def test_reductions_2D(dtype):
x = np.arange(1, 122).reshape((11, 11)).astype(dtype)
a = da.from_array(x, chunks=(4, 4))
b = a.sum(keepdims=True)
assert b._keys() == [[(b.name, 0, 0)]]
reduction_2d_test(da.sum, a, np.sum, x)
reduction_2d_test(da.prod, a, np.prod, x)
reduction_2d_test(da.mean, a, np.mean, x)
reduction_2d_test(da.var, a, np.var, x, False) # Difference in dtype algo
reduction_2d_test(da.std, a, np.std, x, False) # Difference in dtype algo
reduction_2d_test(da.min, a, np.min, x, False)
reduction_2d_test(da.max, a, np.max, x, False)
reduction_2d_test(da.any, a, np.any, x, False)
reduction_2d_test(da.all, a, np.all, x, False)
reduction_2d_test(da.nansum, a, np.nansum, x)
with ignoring(AttributeError):
reduction_2d_test(da.nanprod, a, np.nanprod, x)
reduction_2d_test(da.nanmean, a, np.mean, x)
reduction_2d_test(da.nanvar, a, np.nanvar, x, False) # Difference in dtype algo
reduction_2d_test(da.nanstd, a, np.nanstd, x, False) # Difference in dtype algo
reduction_2d_test(da.nanmin, a, np.nanmin, x, False)
reduction_2d_test(da.nanmax, a, np.nanmax, x, False)
assert eq(da.argmax(a, axis=0), np.argmax(x, axis=0))
assert eq(da.argmin(a, axis=0), np.argmin(x, axis=0))
assert eq(da.nanargmax(a, axis=0), np.nanargmax(x, axis=0))
assert eq(da.nanargmin(a, axis=0), np.nanargmin(x, axis=0))
assert eq(da.argmax(a, axis=1), np.argmax(x, axis=1))
assert eq(da.argmin(a, axis=1), np.argmin(x, axis=1))
assert eq(da.nanargmax(a, axis=1), np.nanargmax(x, axis=1))
assert eq(da.nanargmin(a, axis=1), np.nanargmin(x, axis=1))
assert eq(da.argmax(a, axis=0, split_every=2), np.argmax(x, axis=0))
assert eq(da.argmin(a, axis=0, split_every=2), np.argmin(x, axis=0))
assert eq(da.nanargmax(a, axis=0, split_every=2), np.nanargmax(x, axis=0))
assert eq(da.nanargmin(a, axis=0, split_every=2), np.nanargmin(x, axis=0))
assert eq(da.argmax(a, axis=1, split_every=2), np.argmax(x, axis=1))
assert eq(da.argmin(a, axis=1, split_every=2), np.argmin(x, axis=1))
assert eq(da.nanargmax(a, axis=1, split_every=2), np.nanargmax(x, axis=1))
assert eq(da.nanargmin(a, axis=1, split_every=2), np.nanargmin(x, axis=1))
def test_reductions_2D_nans():
# chunks are a mix of some/all/no NaNs
x = np.full((4, 4), np.nan)
x[:2, :2] = np.array([[1, 2], [3, 4]])
x[2, 2] = 5
x[3, 3] = 6
a = da.from_array(x, chunks=(2, 2))
reduction_2d_test(da.sum, a, np.sum, x, False, False)
reduction_2d_test(da.prod, a, np.prod, x, False, False)
reduction_2d_test(da.mean, a, np.mean, x, False, False)
reduction_2d_test(da.var, a, np.var, x, False, False)
reduction_2d_test(da.std, a, np.std, x, False, False)
reduction_2d_test(da.min, a, np.min, x, False, False)
reduction_2d_test(da.max, a, np.max, x, False, False)
reduction_2d_test(da.any, a, np.any, x, False, False)
reduction_2d_test(da.all, a, np.all, x, False, False)
reduction_2d_test(da.nansum, a, np.nansum, x, False, False)
reduction_2d_test(da.nanprod, a, np.nanprod, x, False, False)
with warnings.catch_warnings():
warnings.simplefilter("ignore", RuntimeWarning)
reduction_2d_test(da.nanmean, a, np.nanmean, x, False, False)
reduction_2d_test(da.nanvar, a, np.nanvar, x, False, False)
reduction_2d_test(da.nanstd, a, np.nanstd, x, False, False)
reduction_2d_test(da.nanmin, a, np.nanmin, x, False, False)
reduction_2d_test(da.nanmax, a, np.nanmax, x, False, False)
assert_eq(da.argmax(a), np.argmax(x))
assert_eq(da.argmin(a), np.argmin(x))
assert_eq(da.nanargmax(a), np.nanargmax(x))
assert_eq(da.nanargmin(a), np.nanargmin(x))
assert_eq(da.argmax(a, axis=0), np.argmax(x, axis=0))
assert_eq(da.argmin(a, axis=0), np.argmin(x, axis=0))
assert_eq(da.nanargmax(a, axis=0), np.nanargmax(x, axis=0))
assert_eq(da.nanargmin(a, axis=0), np.nanargmin(x, axis=0))
assert_eq(da.argmax(a, axis=1), np.argmax(x, axis=1))
assert_eq(da.argmin(a, axis=1), np.argmin(x, axis=1))
assert_eq(da.nanargmax(a, axis=1), np.nanargmax(x, axis=1))
assert_eq(da.nanargmin(a, axis=1), np.nanargmin(x, axis=1))
def test_reductions_2D_nans():
# chunks are a mix of some/all/no NaNs
x = np.full((4, 4), np.nan)
x[:2, :2] = np.array([[1, 2], [3, 4]])
x[2, 2] = 5
x[3, 3] = 6
a = da.from_array(x, chunks=(2, 2))
reduction_2d_test(da.sum, a, np.sum, x, False, False)
reduction_2d_test(da.prod, a, np.prod, x, False, False)
reduction_2d_test(da.mean, a, np.mean, x, False, False)
reduction_2d_test(da.var, a, np.var, x, False, False)
reduction_2d_test(da.std, a, np.std, x, False, False)
reduction_2d_test(da.min, a, np.min, x, False, False)
reduction_2d_test(da.max, a, np.max, x, False, False)
reduction_2d_test(da.any, a, np.any, x, False, False)
reduction_2d_test(da.all, a, np.all, x, False, False)
reduction_2d_test(da.nansum, a, np.nansum, x, False, False)
with ignoring(AttributeError):
reduction_2d_test(da.nanprod, a, np.nanprod, x, False, False)
reduction_2d_test(da.nanmean, a, np.nanmean, x, False, False)
reduction_2d_test(da.nanvar, a, np.nanvar, x, False, False)
reduction_2d_test(da.nanstd, a, np.nanstd, x, False, False)
reduction_2d_test(da.nanmin, a, np.nanmin, x, False, False)
reduction_2d_test(da.nanmax, a, np.nanmax, x, False, False)
assert eq(da.argmax(a), np.argmax(x))
assert eq(da.argmin(a), np.argmin(x))
assert eq(da.nanargmax(a), np.nanargmax(x))
assert eq(da.nanargmin(a), np.nanargmin(x))
assert eq(da.argmax(a, axis=0), np.argmax(x, axis=0))
assert eq(da.argmin(a, axis=0), np.argmin(x, axis=0))
assert eq(da.nanargmax(a, axis=0), np.nanargmax(x, axis=0))
assert eq(da.nanargmin(a, axis=0), np.nanargmin(x, axis=0))
assert eq(da.argmax(a, axis=1), np.argmax(x, axis=1))
assert eq(da.argmin(a, axis=1), np.argmin(x, axis=1))
assert eq(da.nanargmax(a, axis=1), np.nanargmax(x, axis=1))
assert eq(da.nanargmin(a, axis=1), np.nanargmin(x, axis=1))
def test_reductions_2D_nans():
# chunks are a mix of some/all/no NaNs
x = np.full((4, 4), np.nan)
x[:2, :2] = np.array([[1, 2], [3, 4]])
x[2, 2] = 5
x[3, 3] = 6
a = da.from_array(x, chunks=(2, 2))
reduction_2d_test(da.sum, a, np.sum, x, False, False)
reduction_2d_test(da.prod, a, np.prod, x, False, False)
reduction_2d_test(da.mean, a, np.mean, x, False, False)
reduction_2d_test(da.var, a, np.var, x, False, False)
reduction_2d_test(da.std, a, np.std, x, False, False)
reduction_2d_test(da.min, a, np.min, x, False, False)
reduction_2d_test(da.max, a, np.max, x, False, False)
reduction_2d_test(da.any, a, np.any, x, False, False)
reduction_2d_test(da.all, a, np.all, x, False, False)
reduction_2d_test(da.nansum, a, np.nansum, x, False, False)
with ignoring(AttributeError):
reduction_2d_test(da.nanprod, a, np.nanprod, x, False, False)
reduction_2d_test(da.nanmean, a, np.nanmean, x, False, False)
reduction_2d_test(da.nanvar, a, np.nanvar, x, False, False)
reduction_2d_test(da.nanstd, a, np.nanstd, x, False, False)
reduction_2d_test(da.nanmin, a, np.nanmin, x, False, False)
reduction_2d_test(da.nanmax, a, np.nanmax, x, False, False)
assert eq(da.argmax(a), np.argmax(x))
assert eq(da.argmin(a), np.argmin(x))
assert eq(da.nanargmax(a), np.nanargmax(x))
assert eq(da.nanargmin(a), np.nanargmin(x))
assert eq(da.argmax(a, axis=0), np.argmax(x, axis=0))
assert eq(da.argmin(a, axis=0), np.argmin(x, axis=0))
assert eq(da.nanargmax(a, axis=0), np.nanargmax(x, axis=0))
assert eq(da.nanargmin(a, axis=0), np.nanargmin(x, axis=0))
assert eq(da.argmax(a, axis=1), np.argmax(x, axis=1))
assert eq(da.argmin(a, axis=1), np.argmin(x, axis=1))
assert eq(da.nanargmax(a, axis=1), np.nanargmax(x, axis=1))
assert eq(da.nanargmin(a, axis=1), np.nanargmin(x, axis=1))
def test_nan():
x = np.array([[1, np.nan, 3, 4], [5, 6, 7, np.nan], [9, 10, 11, 12]])
d = da.from_array(x, chunks=(2, 2))
assert_eq(np.nansum(x), da.nansum(d))
assert_eq(np.nansum(x, axis=0), da.nansum(d, axis=0))
assert_eq(np.nanmean(x, axis=1), da.nanmean(d, axis=1))
assert_eq(np.nanmin(x, axis=1), da.nanmin(d, axis=1))
assert_eq(np.nanmax(x, axis=(0, 1)), da.nanmax(d, axis=(0, 1)))
assert_eq(np.nanvar(x), da.nanvar(d))
assert_eq(np.nanstd(x, axis=0), da.nanstd(d, axis=0))
assert_eq(np.nanargmin(x, axis=0), da.nanargmin(d, axis=0))
assert_eq(np.nanargmax(x, axis=0), da.nanargmax(d, axis=0))
assert_eq(np.nanprod(x), da.nanprod(d))
def test_nan():
x = np.array([[1, np.nan, 3, 4], [5, 6, 7, np.nan], [9, 10, 11, 12]])
d = da.from_array(x, blockshape=(2, 2))
assert eq(np.nansum(x), da.nansum(d))
assert eq(np.nansum(x, axis=0), da.nansum(d, axis=0))
assert eq(np.nanmean(x, axis=1), da.nanmean(d, axis=1))
assert eq(np.nanmin(x, axis=1), da.nanmin(d, axis=1))
assert eq(np.nanmax(x, axis=(0, 1)), da.nanmax(d, axis=(0, 1)))
assert eq(np.nanvar(x), da.nanvar(d))
assert eq(np.nanstd(x, axis=0), da.nanstd(d, axis=0))
assert eq(np.nanargmin(x, axis=0), da.nanargmin(d, axis=0))
assert eq(np.nanargmax(x, axis=0), da.nanargmax(d, axis=0))
with ignoring(AttributeError):
assert eq(np.nanprod(x), da.nanprod(d))
def test_nan():
x = np.array([[1, np.nan, 3, 4],
[5, 6, 7, np.nan],
[9, 10, 11, 12]])
d = da.from_array(x, chunks=(2, 2))
assert_eq(np.nansum(x), da.nansum(d))
assert_eq(np.nansum(x, axis=0), da.nansum(d, axis=0))
assert_eq(np.nanmean(x, axis=1), da.nanmean(d, axis=1))
assert_eq(np.nanmin(x, axis=1), da.nanmin(d, axis=1))
assert_eq(np.nanmax(x, axis=(0, 1)), da.nanmax(d, axis=(0, 1)))
assert_eq(np.nanvar(x), da.nanvar(d))
assert_eq(np.nanstd(x, axis=0), da.nanstd(d, axis=0))
assert_eq(np.nanargmin(x, axis=0), da.nanargmin(d, axis=0))
assert_eq(np.nanargmax(x, axis=0), da.nanargmax(d, axis=0))
assert_eq(nanprod(x), da.nanprod(d))
def test_nan():
x = np.array([[1, np.nan, 3, 4],
[5, 6, 7, np.nan],
[9, 10, 11, 12]])
d = da.from_array(x, blockshape=(2, 2))
assert eq(np.nansum(x), da.nansum(d))
assert eq(np.nansum(x, axis=0), da.nansum(d, axis=0))
assert eq(np.nanmean(x, axis=1), da.nanmean(d, axis=1))
assert eq(np.nanmin(x, axis=1), da.nanmin(d, axis=1))
assert eq(np.nanmax(x, axis=(0, 1)), da.nanmax(d, axis=(0, 1)))
assert eq(np.nanvar(x), da.nanvar(d))
assert eq(np.nanstd(x, axis=0), da.nanstd(d, axis=0))
assert eq(np.nanargmin(x, axis=0), da.nanargmin(d, axis=0))
assert eq(np.nanargmax(x, axis=0), da.nanargmax(d, axis=0))
with ignoring(AttributeError):
assert eq(np.nanprod(x), da.nanprod(d))
def test_reductions():
x = np.arange(5).astype('f4')
a = da.from_array(x, blockshape=(2, ))
assert eq(da.all(a), np.all(x))
assert eq(da.any(a), np.any(x))
assert eq(da.argmax(a, axis=0), np.argmax(x, axis=0))
assert eq(da.argmin(a, axis=0), np.argmin(x, axis=0))
assert eq(da.max(a), np.max(x))
assert eq(da.mean(a), np.mean(x))
assert eq(da.min(a), np.min(x))
assert eq(da.nanargmax(a, axis=0), np.nanargmax(x, axis=0))
assert eq(da.nanargmin(a, axis=0), np.nanargmin(x, axis=0))
assert eq(da.nanmax(a), np.nanmax(x))
assert eq(da.nanmin(a), np.nanmin(x))
assert eq(da.nansum(a), np.nansum(x))
assert eq(da.nanvar(a), np.nanvar(x))
assert eq(da.nanstd(a), np.nanstd(x))
def test_reductions():
x = np.arange(5).astype('f4')
a = da.from_array(x, chunks=(2,))
assert eq(da.all(a), np.all(x))
assert eq(da.any(a), np.any(x))
assert eq(da.argmax(a, axis=0), np.argmax(x, axis=0))
assert eq(da.argmin(a, axis=0), np.argmin(x, axis=0))
assert eq(da.max(a), np.max(x))
assert eq(da.mean(a), np.mean(x))
assert eq(da.min(a), np.min(x))
assert eq(da.nanargmax(a, axis=0), np.nanargmax(x, axis=0))
assert eq(da.nanargmin(a, axis=0), np.nanargmin(x, axis=0))
assert eq(da.nanmax(a), np.nanmax(x))
assert eq(da.nanmin(a), np.nanmin(x))
assert eq(da.nansum(a), np.nansum(x))
assert eq(da.nanvar(a), np.nanvar(x))
assert eq(da.nanstd(a), np.nanstd(x))
def new_grid_mapping_from_coords(
x_coords: xr.DataArray,
y_coords: xr.DataArray,
crs: Union[str, pyproj.crs.CRS],
*,
tile_size: Union[int, Tuple[int, int]] = None,
tolerance: float = DEFAULT_TOLERANCE,
) -> GridMapping:
crs = _normalize_crs(crs)
assert_instance(x_coords, xr.DataArray, name='x_coords')
assert_instance(y_coords, xr.DataArray, name='y_coords')
assert_true(x_coords.ndim in (1, 2),
'x_coords and y_coords must be either 1D or 2D arrays')
assert_instance(tolerance, float, name='tolerance')
assert_true(tolerance > 0.0, 'tolerance must be greater zero')
if x_coords.name and y_coords.name:
xy_var_names = str(x_coords.name), str(y_coords.name)
else:
xy_var_names = _default_xy_var_names(crs)
tile_size = _normalize_int_pair(tile_size, default=None)
is_lon_360 = None # None means "not yet known"
if crs.is_geographic:
is_lon_360 = bool(np.any(x_coords > 180))
x_res = 0
y_res = 0
if x_coords.ndim == 1:
# We have 1D x,y coordinates
cls = Coords1DGridMapping
assert_true(x_coords.size >= 2 and y_coords.size >= 2,
'sizes of x_coords and y_coords 1D arrays must be >= 2')
size = x_coords.size, y_coords.size
x_dim, y_dim = x_coords.dims[0], y_coords.dims[0]
x_diff = _abs_no_zero(x_coords.diff(dim=x_dim).values)
y_diff = _abs_no_zero(y_coords.diff(dim=y_dim).values)
if not is_lon_360 and crs.is_geographic:
is_anti_meridian_crossed = np.any(np.nanmax(x_diff) > 180)
if is_anti_meridian_crossed:
x_coords = to_lon_360(x_coords)
x_diff = _abs_no_zero(x_coords.diff(dim=x_dim))
is_lon_360 = True
x_res, y_res = x_diff[0], y_diff[0]
x_diff_equal = np.allclose(x_diff, x_res, atol=tolerance)
y_diff_equal = np.allclose(y_diff, y_res, atol=tolerance)
is_regular = x_diff_equal and y_diff_equal
if is_regular:
x_res = round_to_fraction(x_res, 5, 0.25)
y_res = round_to_fraction(y_res, 5, 0.25)
else:
x_res = round_to_fraction(float(np.nanmedian(x_diff)), 2, 0.5)
y_res = round_to_fraction(float(np.nanmedian(y_diff)), 2, 0.5)
if tile_size is None \
and x_coords.chunks is not None \
and y_coords.chunks is not None:
tile_size = (max(0,
*x_coords.chunks[0]), max(0, *y_coords.chunks[0]))
# Guess j axis direction
is_j_axis_up = bool(y_coords[0] < y_coords[-1])
else:
# We have 2D x,y coordinates
cls = Coords2DGridMapping
assert_true(
x_coords.shape == y_coords.shape, 'shapes of x_coords and y_coords'
' 2D arrays must be equal')
assert_true(
x_coords.dims == y_coords.dims,
'dimensions of x_coords and y_coords'
' 2D arrays must be equal')
y_dim, x_dim = x_coords.dims
height, width = x_coords.shape
size = width, height
x = da.asarray(x_coords)
y = da.asarray(y_coords)
x_x_diff = _abs_no_nan(da.diff(x, axis=1))
x_y_diff = _abs_no_nan(da.diff(x, axis=0))
y_x_diff = _abs_no_nan(da.diff(y, axis=1))
y_y_diff = _abs_no_nan(da.diff(y, axis=0))
if not is_lon_360 and crs.is_geographic:
is_anti_meridian_crossed = da.any(da.max(x_x_diff) > 180) \
or da.any(da.max(x_y_diff) > 180)
if is_anti_meridian_crossed:
x_coords = to_lon_360(x_coords)
x = da.asarray(x_coords)
x_x_diff = _abs_no_nan(da.diff(x, axis=1))
x_y_diff = _abs_no_nan(da.diff(x, axis=0))
is_lon_360 = True
is_regular = False
if da.all(x_y_diff == 0) and da.all(y_x_diff == 0):
x_res = x_x_diff[0, 0]
y_res = y_y_diff[0, 0]
is_regular = \
da.allclose(x_x_diff[0, :], x_res, atol=tolerance) \
and da.allclose(x_x_diff[-1, :], x_res, atol=tolerance) \
and da.allclose(y_y_diff[:, 0], y_res, atol=tolerance) \
and da.allclose(y_y_diff[:, -1], y_res, atol=tolerance)
if not is_regular:
# Let diff arrays have same shape as original by
# doubling last rows and columns.
x_x_diff_c = da.concatenate([x_x_diff, x_x_diff[:, -1:]], axis=1)
y_x_diff_c = da.concatenate([y_x_diff, y_x_diff[:, -1:]], axis=1)
x_y_diff_c = da.concatenate([x_y_diff, x_y_diff[-1:, :]], axis=0)
y_y_diff_c = da.concatenate([y_y_diff, y_y_diff[-1:, :]], axis=0)
# Find resolution via area
x_abs_diff = da.sqrt(da.square(x_x_diff_c) + da.square(x_y_diff_c))
y_abs_diff = da.sqrt(da.square(y_x_diff_c) + da.square(y_y_diff_c))
if crs.is_geographic:
# Convert degrees into meters
x_abs_diff_r = da.radians(x_abs_diff)
y_abs_diff_r = da.radians(y_abs_diff)
x_abs_diff = _ER * da.cos(x_abs_diff_r) * y_abs_diff_r
y_abs_diff = _ER * y_abs_diff_r
xy_areas = (x_abs_diff * y_abs_diff).flatten()
xy_areas = da.where(xy_areas > 0, xy_areas, np.nan)
# Get indices of min and max area
xy_area_index_min = da.nanargmin(xy_areas)
xy_area_index_max = da.nanargmax(xy_areas)
# Convert area to edge length
xy_res_min = math.sqrt(xy_areas[xy_area_index_min])
xy_res_max = math.sqrt(xy_areas[xy_area_index_max])
# Empirically weight min more than max
xy_res = 0.7 * xy_res_min + 0.3 * xy_res_max
if crs.is_geographic:
# Convert meters back into degrees
# print(f'xy_res in meters: {xy_res}')
xy_res = math.degrees(xy_res / _ER)
# print(f'xy_res in degrees: {xy_res}')
# Because this is an estimation, we can round to a nice number
xy_res = round_to_fraction(xy_res, digits=1, resolution=0.5)
x_res, y_res = float(xy_res), float(xy_res)
if tile_size is None and x_coords.chunks is not None:
j_chunks, i_chunks = x_coords.chunks
tile_size = max(0, *i_chunks), max(0, *j_chunks)
if tile_size is not None:
tile_width, tile_height = tile_size
x_coords = x_coords.chunk((tile_height, tile_width))
y_coords = y_coords.chunk((tile_height, tile_width))
# Guess j axis direction
is_j_axis_up = np.all(y_coords[0, :] < y_coords[-1, :]) or None
assert_true(x_res > 0 and y_res > 0,
'internal error: x_res and y_res could not be determined',
exception_type=RuntimeError)
x_res, y_res = _to_int_or_float(x_res), _to_int_or_float(y_res)
x_res_05, y_res_05 = x_res / 2, y_res / 2
x_min = _to_int_or_float(x_coords.min() - x_res_05)
y_min = _to_int_or_float(y_coords.min() - y_res_05)
x_max = _to_int_or_float(x_coords.max() + x_res_05)
y_max = _to_int_or_float(y_coords.max() + y_res_05)
return cls(x_coords=x_coords,
y_coords=y_coords,
crs=crs,
size=size,
tile_size=tile_size,
xy_bbox=(x_min, y_min, x_max, y_max),
xy_res=(x_res, y_res),
xy_var_names=xy_var_names,
xy_dim_names=(str(x_dim), str(y_dim)),
is_regular=is_regular,
is_lon_360=is_lon_360,
is_j_axis_up=is_j_axis_up)
