def test_maybe_promote(kind, expected):
# 'g': np.float128 is not tested : not available on all platforms
# 'G': np.complex256 is not tested : not available on all platforms
actual = dtypes.maybe_promote(np.dtype(kind))
assert actual[0] == expected[0]
assert str(actual[1]) == expected[1]
def construct_dataarray(dim_num, dtype, contains_nan, dask):
# dimnum <= 3
rng = np.random.RandomState(0)
shapes = [16, 8, 4][:dim_num]
dims = ('x', 'y', 'z')[:dim_num]
if np.issubdtype(dtype, np.floating):
array = rng.randn(*shapes).astype(dtype)
elif np.issubdtype(dtype, np.integer):
array = rng.randint(0, 10, size=shapes).astype(dtype)
elif np.issubdtype(dtype, np.bool_):
array = rng.randint(0, 1, size=shapes).astype(dtype)
elif dtype == str:
array = rng.choice(['a', 'b', 'c', 'd'], size=shapes)
else:
raise ValueError
if contains_nan:
inds = rng.choice(range(array.size), int(array.size * 0.2))
dtype, fill_value = dtypes.maybe_promote(array.dtype)
array = array.astype(dtype)
array.flat[inds] = fill_value
da = DataArray(array, dims=dims, coords={'x': np.arange(16)}, name='da')
if dask and has_dask:
chunks = {d: 4 for d in dims}
da = da.chunk(chunks)
return da
def construct_dataarray(dim_num, dtype, contains_nan, dask):
# dimnum <= 3
rng = np.random.RandomState(0)
shapes = [16, 8, 4][:dim_num]
dims = ('x', 'y', 'z')[:dim_num]
if np.issubdtype(dtype, np.floating):
array = rng.randn(*shapes).astype(dtype)
elif np.issubdtype(dtype, np.integer):
array = rng.randint(0, 10, size=shapes).astype(dtype)
elif np.issubdtype(dtype, np.bool_):
array = rng.randint(0, 1, size=shapes).astype(dtype)
elif dtype == str:
array = rng.choice(['a', 'b', 'c', 'd'], size=shapes)
else:
raise ValueError
if contains_nan:
inds = rng.choice(range(array.size), int(array.size * 0.2))
dtype, fill_value = dtypes.maybe_promote(array.dtype)
array = array.astype(dtype)
array.flat[inds] = fill_value
da = DataArray(array, dims=dims, coords={'x': np.arange(16)}, name='da')
if dask and has_dask:
chunks = {d: 4 for d in dims}
da = da.chunk(chunks)
return da
def test_maybe_promote(kind, expected):
# 'g': np.float128 is not tested : not available on all platforms
# 'G': np.complex256 is not tested : not available on all platforms
actual = dtypes.maybe_promote(np.dtype(kind))
assert actual[0] == expected[0]
assert str(actual[1]) == expected[1]
def __init__(
self,
manager,
lock,
name,
vrt_params=None,
masked=False,
mask_and_scale=False,
unsigned=False,
):
self.manager = manager
self.lock = lock
self.masked = masked or mask_and_scale
self.mask_and_scale = mask_and_scale
# cannot save riods as an attribute: this would break pickleability
riods = manager.acquire()
if vrt_params is not None:
riods = WarpedVRT(riods, **vrt_params)
self.vrt_params = vrt_params
self._shape = (riods.count, riods.height, riods.width)
self._dtype = None
dtypes = riods.dtypes
if not np.all(np.asarray(dtypes) == dtypes[0]):
raise ValueError("All bands should have the same dtype")
dtype = _rasterio_to_numpy_dtype(dtypes)
# handle unsigned case
if mask_and_scale and unsigned and dtype.kind == "i":
self._dtype = np.dtype(f"u{dtype.itemsize}")
elif mask_and_scale and unsigned:
warnings.warn(
f"variable {name!r} has _Unsigned attribute but is not "
"of integer type. Ignoring attribute.",
variables.SerializationWarning,
stacklevel=3,
)
self._fill_value = riods.nodata
if self._dtype is None:
if self.masked:
self._dtype, self._fill_value = maybe_promote(dtype)
else:
self._dtype = dtype
def as_sparse_xarray(obj): # pragma: no cover
"""Convert *obj* to an xr.DataArray with sparse.COO storage."""
import sparse
from xarray.core.dtypes import maybe_promote
if isinstance(obj, xr.DataArray) and isinstance(obj.data, numpy.ndarray):
return xr.DataArray(
data=sparse.COO.from_numpy(obj.data,
fill_value=maybe_promote(
obj.data.dtype)[1]),
coords=obj.coords,
dims=obj.dims,
name=obj.name,
attrs=obj.attrs,
)
elif isinstance(obj, pd.Series):
return xr.DataArray.from_series(obj, sparse=True)
else:
print(type(obj), type(obj.data))
return obj
def as_sparse_xarray(obj, units=None): # pragma: no cover
"""Convert *obj* to :class:`xarray.DataArray` with sparse.COO storage."""
import sparse
from xarray.core.dtypes import maybe_promote
if isinstance(obj, xr.DataArray) and isinstance(obj.data, numpy.ndarray):
result = xr.DataArray(
data=sparse.COO.from_numpy(obj.data,
fill_value=maybe_promote(
obj.data.dtype)[1]),
coords=obj.coords,
dims=obj.dims,
name=obj.name,
attrs=obj.attrs,
)
elif isinstance(obj, pd.Series):
result = xr.DataArray.from_series(obj, sparse=True)
else:
result = obj
if units:
result.attrs['_unit'] = pint.Unit(units)
return result
