def test_collapse_coords():
ds = generate_test_dataset()
for c in ds.coords:
expanded = xr.concat([ds.coords[c]] * 10, dim='new')
xr_assert_equal(
ds.coords[c], warp._collapse_coords(expanded)
)
def test_expand_collapse_coords():
ds = generate_test_dataset()
for c in ['x', 'y']:
expanded = warp._expand_var_to_xy(ds.coords[c], ds.coords)
xr_assert_equal(
ds.coords[c], warp._collapse_coords(expanded)
)
def test_alignment(tmpdir, extent, from_files):
datapath = tmpdir.mkdir('data')
path = tmpdir.mkdir('aligned')
bounds = [
(-10.0, 50.0, 0.0, 60.0),
(-12.0, 40.0, -2.0, 52.0),
(-13.0, 50.0, -3.0, 60.0),
(-9.0, 51.0, 1.0, 61.0)
]
datasets = [generate_test_dataset(extent=ext) for ext in bounds]
if extent is None:
common_bounds = warp.get_common_bounds(datasets)
else:
common_bounds = extent
files = [str(datapath.join('data_%d.nc' % i))
for i in range(len(datasets))]
if from_files:
for ds, f in zip(datasets, files):
to_netcdf(ds, f)
datasets = files
warp.Alignment(extent=extent).apply(datasets, path=str(path))
aligned = [open_dataset(str(f)) for f in path.listdir()]
for ds in aligned:
assert_equal(warp.get_bounds(ds), common_bounds)
assert_equal(
warp.get_transform(ds),
warp.get_transform(aligned[0])
)
xr_assert_equal(ds['x'], aligned[0]['x'])
xr_assert_equal(ds['y'], aligned[0]['y'])
def test_write_read_netcdf(tmpdir):
ds = generate_test_dataset()
ds = assemble_complex(ds)
path = str(tmpdir.join('test_dataset.nc'))
to_netcdf(ds, path)
ds_read = open_dataset(path)
xr_assert_equal(ds, ds_read)
def test_map_over_tiles(tmpdir, fn, buffer):
tile_path = tmpdir / 'tiles'
chunks = {'y': 10, 'x': 10}
tiling.tile(ds, str(tile_path), chunks=chunks, buffer=buffer)
files = str(tile_path / '*.nc')
mapped = tiling.map_over_tiles(files, fn)
xr_assert_equal(mapped, fn(ds))
def test_assemble_complex_dataset():
# Create real dataset with real and imag part
ds = generate_test_dataset(var=['a__im', 'a__re', 'b', 'c'])
# Check that assembling into complex works
ds_complex = assemble_complex(ds)
assert_equal(set(ds_complex.data_vars), {'a', 'b', 'c'})
xr_assert_equal(ds_complex['a'].real, ds['a__re'])
xr_assert_equal(ds_complex['a'].imag, ds['a__im'])
def test_accessor_nd_to_netcdf(tmpdir, generator):
ds = generator()
path_1 = str(tmpdir.join('ds1.nc'))
path_2 = str(tmpdir.join('ds2.nc'))
io.to_netcdf(ds, path_1)
ds.nd.to_netcdf(path_2)
xr_assert_equal(io.open_dataset(path_1), io.open_dataset(path_2))
def test_disassemble_complex_dataarray():
# Create complex dataset
da = generate_test_dataarray(name='data')
complex_data = np.random.rand(*da.shape) + 1j * np.random.rand(*da.shape)
da.values = complex_data
ds_real = disassemble_complex(da)
assert_equal(set(ds_real.data_vars), {'data__re', 'data__im'})
xr_assert_equal(da.real, ds_real['data__re'])
xr_assert_equal(da.imag, ds_real['data__im'])
def test_predict_proba():
dims = OrderedDict([('y', 50), ('x', 50), ('time', 10)])
ds, labels = create_mock_classes(dims)
c = classify.Classifier(LogisticRegression())
pred = c.fit(ds, labels).predict(ds)
proba = c.fit(ds, labels).predict(ds, func='predict_proba')
assert (proba >= 0).all()
assert (proba <= 1).all()
xr_assert_equal(pred, proba.argmax('label') + 1)
def test_accessor_nd_apply(generator):
ds = generator()
def func(arr):
"""Reduce a two dimensional array to its mean."""
return arr.mean()
signature = '(x,y)->()'
xr_assert_equal(ds.nd.apply(func, signature=signature),
utils.apply(ds, func, signature=signature))
def test_auto_merge_metadata():
ds_meta = generate_test_dataset(dims={'y': 20, 'x': 20, 'time': 10})
# Create copy of each chunk as otherwise the metadata cannot be altered
# individually.
chunks = [c.copy() for c in utils.xr_split(ds_meta, 'time', 5)]
for i, c in enumerate(chunks):
c.attrs['part_number'] = i
ds_meta['part_number'] = ('time', np.repeat(np.arange(5), 2))
xr_assert_equal(ds_meta,
tiling.auto_merge(chunks, meta_variables=['part_number']))
def test_disassemble_complex_dataset():
# Create complex dataset
ds = generate_test_dataset(var=['b', 'c'])
dims = tuple(ds.dims.keys())
shape = tuple(ds.dims.values())
complex_data = np.random.rand(*shape) + 1j * np.random.rand(*shape)
ds['a'] = (dims, complex_data)
# Check that disassembling into reals works as expected
ds_real = disassemble_complex(ds)
assert_equal(set(ds_real.data_vars), {'a__re', 'a__im', 'b', 'c'})
xr_assert_equal(ds['a'].real, ds_real['a__re'])
xr_assert_equal(ds['a'].imag, ds_real['a__im'])
def test_classifier(clf, array):
dims = OrderedDict([('y', 50), ('x', 50)])
ds, labels_true = create_mock_classes(dims)
if array:
ds = ds['C11']
# Select 10% for training
labels_train = labels_true.copy()
mask_train = (np.random.rand(dims['y'], dims['x']) < 0.1)
labels_train = labels_train.where(mask_train)
c = classify.Classifier(clf)
c.fit(ds, labels_train)
labels_predicted = c.predict(ds)
# Expect 100% accuracy for this trivial classification task.
xr_assert_equal(labels_predicted, labels_true)
def test_reproject(generator):
src_crs = epsg4326
dst_crs = sinusoidal
ds = generator(crs=src_crs)
src_bounds = get_bounds(ds)
dst_bounds_latlon = BoundingBox(
left=src_bounds.left - 1,
bottom=src_bounds.bottom - 1,
right=src_bounds.right + 1,
top=src_bounds.top + 1,
)
dst_bounds = BoundingBox(*rasterio.warp.transform_bounds(
src_crs, dst_crs, **dst_bounds_latlon._asdict()))
dst_width, dst_height = 35, 21
resx = (dst_bounds.right - dst_bounds.left) / (dst_width - 1)
resy = (dst_bounds.bottom - dst_bounds.top) / (dst_height - 1)
res = (abs(resx), abs(resy))
xoff = dst_bounds.left
yoff = dst_bounds.top
dst_transform = Affine(resx, 0, xoff, 0, resy, yoff)
projected = [
_reproject(ds,
dst_crs=dst_crs,
dst_transform=dst_transform,
width=dst_width,
height=dst_height),
_reproject(ds,
dst_crs=dst_crs,
dst_transform=dst_transform,
extent=dst_bounds),
_reproject(ds, dst_crs=dst_crs, extent=dst_bounds, res=res),
_reproject(ds,
dst_crs=dst_crs,
extent=dst_bounds,
width=dst_width,
height=dst_height)
]
for proj in projected[1:]:
xr_assert_equal(proj, projected[0])
assert_almost_equal(get_resolution(proj), res)
assert_almost_equal(get_bounds(proj), dst_bounds)
assert_almost_equal(get_transform(proj), dst_transform)
assert_equal_crs(get_crs(proj), dst_crs)
def test_wrap_algorithm(generator):
ds = generator()
args = (0.1, )
kwargs = {}
algo = DummyAlgorithm(*args, **kwargs)
wrapper = wrap_algorithm(DummyAlgorithm, 'wrapper_name')
# Make sure the result is the same
xr_assert_equal(algo.apply(ds), wrapper(ds, *args, **kwargs))
# Check name, docstring, signature
assert (wrapper.__name__ == 'wrapper_name')
assert_equal(
wrapper.__doc__,
'Wrapper for :class:`nd.tests.test_algorithm.DummyAlgorithm`.\n\n' +
DummyAlgorithm.__doc__)
assert_equal(
list(OrderedDict(inspect.signature(wrapper).parameters).keys()),
['ds', 'value', 'args', 'kwargs'])
def test_reprojection_with_target(generator):
src_crs = epsg4326
dst_crs = sinusoidal
ds = generator(crs=src_crs)
src_bounds = warp.get_bounds(ds)
dst_bounds_latlon = BoundingBox(
left=src_bounds.left - 1,
bottom=src_bounds.bottom - 1,
right=src_bounds.right + 1,
top=src_bounds.top + 1,
)
dst_bounds = BoundingBox(*rasterio.warp.transform_bounds(
src_crs, dst_crs, **dst_bounds_latlon._asdict()
))
dst_width, dst_height = 35, 21
resx = (dst_bounds.right - dst_bounds.left) / (dst_width - 1)
resy = (dst_bounds.bottom - dst_bounds.top) / (dst_height - 1)
res = (abs(resx), abs(resy))
xoff = dst_bounds.left
yoff = dst_bounds.top
dst_transform = Affine(resx, 0, xoff, 0, resy, yoff)
target = generator(
dims={'x': dst_width, 'y': dst_height, 'time': 1},
extent=dst_bounds, crs=dst_crs
)
projected = [
warp.Reprojection(crs=dst_crs, transform=dst_transform,
width=dst_width, height=dst_height).apply(ds),
warp.Reprojection(crs=dst_crs, extent=dst_bounds,
res=res).apply(ds),
warp.Reprojection(crs=dst_crs, extent=dst_bounds,
width=dst_width, height=dst_height).apply(ds),
warp.Reprojection(target=target).apply(ds),
]
for i, proj in enumerate(projected[1:]):
print(i)
xr_assert_equal(proj, projected[0])
assert_almost_equal(warp.get_resolution(proj), res)
assert_almost_equal(warp.get_bounds(proj), dst_bounds)
assert_almost_equal(warp.get_transform(proj), dst_transform)
assert_equal_crs(warp.get_crs(proj), dst_crs)
def test_classifier_feature_dims(dims, feature_dims):
ds, labels = create_mock_classes(dims)
c = classify.Classifier(RandomForestClassifier(n_estimators=20),
feature_dims=feature_dims)
# Expect 100% accuracy for this trivial classification task.
pred = c.fit(ds, labels).predict(ds)
xr_assert_equal(
pred, classify._broadcast_labels(labels, ds,
feature_dims=feature_dims))
# Check that the results are the same whether labels
# are passed as xr.DataArray or np.ndarray
pred_np = c.fit(ds, labels.values).predict(ds)
xr_assert_equal(pred, pred_np)
# Check that prediction result has correct dimensions
assert_equal(utils.get_dims(pred),
classify._get_data_dims(ds, feature_dims=feature_dims))
def test_accessor_nd_omnibus():
ds1 = generate_test_dataset(dims={
'y': 5,
'x': 5,
'time': 10
},
mean=[1, 0, 0, 1],
sigma=0.1).isel(time=slice(None, 5))
ds2 = generate_test_dataset(dims={
'y': 5,
'x': 5,
'time': 10
},
mean=[10, 0, 0, 10],
sigma=0.1).isel(time=slice(5, None))
ds = xr.concat([ds1, ds2], dim='time')
kwargs = dict(n=9, alpha=0.9)
xr_assert_equal(change.omnibus(ds, **kwargs),
ds.nd.change_omnibus(**kwargs))
def test_reprojection_with_src_crs():
src_crs = epsg4326
dst_crs = sinusoidal
# Set up test dataset with and without CRS information
ds = generate_test_dataset(crs=src_crs)
assert_equal_crs(src_crs, ds.nd.crs)
ds_nocrs = ds.copy()
del ds_nocrs.attrs['crs']
assert ds_nocrs.nd.crs is None
with assert_raises_regex(
CRSError,
"Could not infer projection from input data. "
"Please provide the parameter `src_crs`."):
warp.Reprojection(dst_crs=dst_crs).apply(ds_nocrs)
xr_assert_equal(
warp.Reprojection(dst_crs=dst_crs).apply(ds),
warp.Reprojection(src_crs=src_crs, dst_crs=dst_crs).apply(ds_nocrs)
)
def test_reproject_with_extra_dims(dims):
crs1 = warp._parse_crs('epsg:4326')
crs2 = warp._parse_crs('epsg:3395')
ds = generate_test_dataset(
dims=dims, crs=crs1
)
proj = warp.Reprojection(crs=crs2)
reprojected = proj.apply(ds)
# Check that a reprojected slice of the dataset is the same as
# the slice of the reprojection of the entire dataset.
slices = [
{'band': 3},
{'time': slice(1, 3)}
]
for s in slices:
xr_assert_equal(
proj.apply(ds.isel(**s)),
reprojected.isel(**s)
)
def test_broadcast(dims, feature_dims):
ds, labels = create_mock_classes(dims)
expected_shape = classify._get_data_shape(ds, feature_dims=feature_dims)
# Check broadcast for numpy array
blabels = classify._broadcast_labels(labels.values, ds, feature_dims)
assert blabels.shape == expected_shape
# Check broadcast for DataArray
blabels = classify._broadcast_labels(labels.transpose('x', 'y'), ds,
feature_dims)
blabels_t = classify._broadcast_labels(labels.transpose('y', 'x'), ds,
feature_dims)
xr_assert_equal(blabels, blabels_t)
assert blabels.shape == expected_shape
# Check values equal along broadcast dimensions
bc_dims = set(dims) - set(labels.dims) - set(feature_dims)
for d in bc_dims:
assert (blabels.std(d) == 0).all()
def test_reproject_one_dimensional_vars():
ds = generate_test_dataset(crs=epsg4326)
ds['xvar'] = (('x',), ds.x.values * 10)
ds['yvar'] = (('y',), ds.y.values * 10)
ds['timevar'] = (('time',), np.random.rand(ds.dims['time']))
warped = warp.Reprojection(
crs=sinusoidal, resampling=rasterio.warp.Resampling.bilinear
).apply(ds)
xr_assert_equal(
warped['timevar'], ds['timevar'])
# Check that xvar and yvar are still proportional to longitude
# and latitude
for v, coord in [
('xvar', 'lon'), ('yvar', 'lat')
]:
ratios = warped[v].values / warped[coord].values
print(v, coord)
print(np.nanmin(ratios), np.nanmax(ratios), np.nanstd(ratios))
# There is quite a bit of error, for now accept relatively
# high error.
assert np.nanstd(ratios) < 1.5
def test_expand_var_to_xy(extra, dims):
ref = generate_test_dataarray(
dims=OrderedDict([('y', 30), ('x', 20), ('time', 5)])
)
da = generate_test_dataarray(dims=dims)
expanded = warp._expand_var_to_xy(da, ref.coords)
# Check that the variance along added dimensions is zero
assert np.all(
expanded.var(extra) < 1e-16
)
# Check that the new DataArray contains x and y coordinates
xr_assert_equal(
expanded.coords['x'], ref.coords['x'])
xr_assert_equal(
expanded.coords['y'], ref.coords['y'])
# Check that the DataArray is unchanged if x and y were already
# dimensions
if 'x' in dims and 'y' in dims:
xr_assert_equal(da, expanded)
def test_auto_merge():
xr_assert_equal(ds, tiling.auto_merge(parts))
def test_parallelized_apply(generator, njobs):
ds = generator()
algo = ParallelDummyAlgorithm(3)
ref = algo.apply(ds)
result = algo.apply(ds, njobs=njobs)
xr_assert_equal(ref, result)
def test_accessor_nd_resample(generator):
ds = generator()
kwargs = dict(width=50)
xr_assert_equal(warp.resample(ds, **kwargs), ds.nd.resample(**kwargs))
def test_auto_merge_with_buffer():
xr_assert_equal(ds, tiling.auto_merge(buffered_parts))
def test_auto_merge_with_buffer(use_xarray):
xr_assert_equal(
ds, tiling.auto_merge(buffered_parts, use_xarray_combine=use_xarray))
def test_auto_merge(use_xarray):
xr_assert_equal(ds, tiling.auto_merge(parts,
use_xarray_combine=use_xarray))
def test_tile_and_merge(tmpdir, chunks, buffer):
tile_path = tmpdir / 'tiles'
tiling.tile(ds, str(tile_path), chunks=chunks, buffer=buffer)
merged = tiling.auto_merge(str(tile_path / '*.nc'))
xr_assert_equal(merged, ds)
