def test_collapse_coords(): ds = generate_test_dataset() for c in ds.coords: expanded = xr.concat([ds.coords[c]] * 10, dim='new') xr_assert_equal( ds.coords[c], warp._collapse_coords(expanded) )
def test_expand_collapse_coords(): ds = generate_test_dataset() for c in ['x', 'y']: expanded = warp._expand_var_to_xy(ds.coords[c], ds.coords) xr_assert_equal( ds.coords[c], warp._collapse_coords(expanded) )
def test_alignment(tmpdir, extent, from_files): datapath = tmpdir.mkdir('data') path = tmpdir.mkdir('aligned') bounds = [ (-10.0, 50.0, 0.0, 60.0), (-12.0, 40.0, -2.0, 52.0), (-13.0, 50.0, -3.0, 60.0), (-9.0, 51.0, 1.0, 61.0) ] datasets = [generate_test_dataset(extent=ext) for ext in bounds] if extent is None: common_bounds = warp.get_common_bounds(datasets) else: common_bounds = extent files = [str(datapath.join('data_%d.nc' % i)) for i in range(len(datasets))] if from_files: for ds, f in zip(datasets, files): to_netcdf(ds, f) datasets = files warp.Alignment(extent=extent).apply(datasets, path=str(path)) aligned = [open_dataset(str(f)) for f in path.listdir()] for ds in aligned: assert_equal(warp.get_bounds(ds), common_bounds) assert_equal( warp.get_transform(ds), warp.get_transform(aligned[0]) ) xr_assert_equal(ds['x'], aligned[0]['x']) xr_assert_equal(ds['y'], aligned[0]['y'])
def test_write_read_netcdf(tmpdir): ds = generate_test_dataset() ds = assemble_complex(ds) path = str(tmpdir.join('test_dataset.nc')) to_netcdf(ds, path) ds_read = open_dataset(path) xr_assert_equal(ds, ds_read)
def test_map_over_tiles(tmpdir, fn, buffer): tile_path = tmpdir / 'tiles' chunks = {'y': 10, 'x': 10} tiling.tile(ds, str(tile_path), chunks=chunks, buffer=buffer) files = str(tile_path / '*.nc') mapped = tiling.map_over_tiles(files, fn) xr_assert_equal(mapped, fn(ds))
def test_assemble_complex_dataset(): # Create real dataset with real and imag part ds = generate_test_dataset(var=['a__im', 'a__re', 'b', 'c']) # Check that assembling into complex works ds_complex = assemble_complex(ds) assert_equal(set(ds_complex.data_vars), {'a', 'b', 'c'}) xr_assert_equal(ds_complex['a'].real, ds['a__re']) xr_assert_equal(ds_complex['a'].imag, ds['a__im'])
def test_accessor_nd_to_netcdf(tmpdir, generator): ds = generator() path_1 = str(tmpdir.join('ds1.nc')) path_2 = str(tmpdir.join('ds2.nc')) io.to_netcdf(ds, path_1) ds.nd.to_netcdf(path_2) xr_assert_equal(io.open_dataset(path_1), io.open_dataset(path_2))
def test_disassemble_complex_dataarray(): # Create complex dataset da = generate_test_dataarray(name='data') complex_data = np.random.rand(*da.shape) + 1j * np.random.rand(*da.shape) da.values = complex_data ds_real = disassemble_complex(da) assert_equal(set(ds_real.data_vars), {'data__re', 'data__im'}) xr_assert_equal(da.real, ds_real['data__re']) xr_assert_equal(da.imag, ds_real['data__im'])
def test_predict_proba(): dims = OrderedDict([('y', 50), ('x', 50), ('time', 10)]) ds, labels = create_mock_classes(dims) c = classify.Classifier(LogisticRegression()) pred = c.fit(ds, labels).predict(ds) proba = c.fit(ds, labels).predict(ds, func='predict_proba') assert (proba >= 0).all() assert (proba <= 1).all() xr_assert_equal(pred, proba.argmax('label') + 1)
def test_accessor_nd_apply(generator): ds = generator() def func(arr): """Reduce a two dimensional array to its mean.""" return arr.mean() signature = '(x,y)->()' xr_assert_equal(ds.nd.apply(func, signature=signature), utils.apply(ds, func, signature=signature))
def test_auto_merge_metadata(): ds_meta = generate_test_dataset(dims={'y': 20, 'x': 20, 'time': 10}) # Create copy of each chunk as otherwise the metadata cannot be altered # individually. chunks = [c.copy() for c in utils.xr_split(ds_meta, 'time', 5)] for i, c in enumerate(chunks): c.attrs['part_number'] = i ds_meta['part_number'] = ('time', np.repeat(np.arange(5), 2)) xr_assert_equal(ds_meta, tiling.auto_merge(chunks, meta_variables=['part_number']))
def test_disassemble_complex_dataset(): # Create complex dataset ds = generate_test_dataset(var=['b', 'c']) dims = tuple(ds.dims.keys()) shape = tuple(ds.dims.values()) complex_data = np.random.rand(*shape) + 1j * np.random.rand(*shape) ds['a'] = (dims, complex_data) # Check that disassembling into reals works as expected ds_real = disassemble_complex(ds) assert_equal(set(ds_real.data_vars), {'a__re', 'a__im', 'b', 'c'}) xr_assert_equal(ds['a'].real, ds_real['a__re']) xr_assert_equal(ds['a'].imag, ds_real['a__im'])
def test_classifier(clf, array): dims = OrderedDict([('y', 50), ('x', 50)]) ds, labels_true = create_mock_classes(dims) if array: ds = ds['C11'] # Select 10% for training labels_train = labels_true.copy() mask_train = (np.random.rand(dims['y'], dims['x']) < 0.1) labels_train = labels_train.where(mask_train) c = classify.Classifier(clf) c.fit(ds, labels_train) labels_predicted = c.predict(ds) # Expect 100% accuracy for this trivial classification task. xr_assert_equal(labels_predicted, labels_true)
def test_reproject(generator): src_crs = epsg4326 dst_crs = sinusoidal ds = generator(crs=src_crs) src_bounds = get_bounds(ds) dst_bounds_latlon = BoundingBox( left=src_bounds.left - 1, bottom=src_bounds.bottom - 1, right=src_bounds.right + 1, top=src_bounds.top + 1, ) dst_bounds = BoundingBox(*rasterio.warp.transform_bounds( src_crs, dst_crs, **dst_bounds_latlon._asdict())) dst_width, dst_height = 35, 21 resx = (dst_bounds.right - dst_bounds.left) / (dst_width - 1) resy = (dst_bounds.bottom - dst_bounds.top) / (dst_height - 1) res = (abs(resx), abs(resy)) xoff = dst_bounds.left yoff = dst_bounds.top dst_transform = Affine(resx, 0, xoff, 0, resy, yoff) projected = [ _reproject(ds, dst_crs=dst_crs, dst_transform=dst_transform, width=dst_width, height=dst_height), _reproject(ds, dst_crs=dst_crs, dst_transform=dst_transform, extent=dst_bounds), _reproject(ds, dst_crs=dst_crs, extent=dst_bounds, res=res), _reproject(ds, dst_crs=dst_crs, extent=dst_bounds, width=dst_width, height=dst_height) ] for proj in projected[1:]: xr_assert_equal(proj, projected[0]) assert_almost_equal(get_resolution(proj), res) assert_almost_equal(get_bounds(proj), dst_bounds) assert_almost_equal(get_transform(proj), dst_transform) assert_equal_crs(get_crs(proj), dst_crs)
def test_wrap_algorithm(generator): ds = generator() args = (0.1, ) kwargs = {} algo = DummyAlgorithm(*args, **kwargs) wrapper = wrap_algorithm(DummyAlgorithm, 'wrapper_name') # Make sure the result is the same xr_assert_equal(algo.apply(ds), wrapper(ds, *args, **kwargs)) # Check name, docstring, signature assert (wrapper.__name__ == 'wrapper_name') assert_equal( wrapper.__doc__, 'Wrapper for :class:`nd.tests.test_algorithm.DummyAlgorithm`.\n\n' + DummyAlgorithm.__doc__) assert_equal( list(OrderedDict(inspect.signature(wrapper).parameters).keys()), ['ds', 'value', 'args', 'kwargs'])
def test_reprojection_with_target(generator): src_crs = epsg4326 dst_crs = sinusoidal ds = generator(crs=src_crs) src_bounds = warp.get_bounds(ds) dst_bounds_latlon = BoundingBox( left=src_bounds.left - 1, bottom=src_bounds.bottom - 1, right=src_bounds.right + 1, top=src_bounds.top + 1, ) dst_bounds = BoundingBox(*rasterio.warp.transform_bounds( src_crs, dst_crs, **dst_bounds_latlon._asdict() )) dst_width, dst_height = 35, 21 resx = (dst_bounds.right - dst_bounds.left) / (dst_width - 1) resy = (dst_bounds.bottom - dst_bounds.top) / (dst_height - 1) res = (abs(resx), abs(resy)) xoff = dst_bounds.left yoff = dst_bounds.top dst_transform = Affine(resx, 0, xoff, 0, resy, yoff) target = generator( dims={'x': dst_width, 'y': dst_height, 'time': 1}, extent=dst_bounds, crs=dst_crs ) projected = [ warp.Reprojection(crs=dst_crs, transform=dst_transform, width=dst_width, height=dst_height).apply(ds), warp.Reprojection(crs=dst_crs, extent=dst_bounds, res=res).apply(ds), warp.Reprojection(crs=dst_crs, extent=dst_bounds, width=dst_width, height=dst_height).apply(ds), warp.Reprojection(target=target).apply(ds), ] for i, proj in enumerate(projected[1:]): print(i) xr_assert_equal(proj, projected[0]) assert_almost_equal(warp.get_resolution(proj), res) assert_almost_equal(warp.get_bounds(proj), dst_bounds) assert_almost_equal(warp.get_transform(proj), dst_transform) assert_equal_crs(warp.get_crs(proj), dst_crs)
def test_classifier_feature_dims(dims, feature_dims): ds, labels = create_mock_classes(dims) c = classify.Classifier(RandomForestClassifier(n_estimators=20), feature_dims=feature_dims) # Expect 100% accuracy for this trivial classification task. pred = c.fit(ds, labels).predict(ds) xr_assert_equal( pred, classify._broadcast_labels(labels, ds, feature_dims=feature_dims)) # Check that the results are the same whether labels # are passed as xr.DataArray or np.ndarray pred_np = c.fit(ds, labels.values).predict(ds) xr_assert_equal(pred, pred_np) # Check that prediction result has correct dimensions assert_equal(utils.get_dims(pred), classify._get_data_dims(ds, feature_dims=feature_dims))
def test_accessor_nd_omnibus(): ds1 = generate_test_dataset(dims={ 'y': 5, 'x': 5, 'time': 10 }, mean=[1, 0, 0, 1], sigma=0.1).isel(time=slice(None, 5)) ds2 = generate_test_dataset(dims={ 'y': 5, 'x': 5, 'time': 10 }, mean=[10, 0, 0, 10], sigma=0.1).isel(time=slice(5, None)) ds = xr.concat([ds1, ds2], dim='time') kwargs = dict(n=9, alpha=0.9) xr_assert_equal(change.omnibus(ds, **kwargs), ds.nd.change_omnibus(**kwargs))
def test_reprojection_with_src_crs(): src_crs = epsg4326 dst_crs = sinusoidal # Set up test dataset with and without CRS information ds = generate_test_dataset(crs=src_crs) assert_equal_crs(src_crs, ds.nd.crs) ds_nocrs = ds.copy() del ds_nocrs.attrs['crs'] assert ds_nocrs.nd.crs is None with assert_raises_regex( CRSError, "Could not infer projection from input data. " "Please provide the parameter `src_crs`."): warp.Reprojection(dst_crs=dst_crs).apply(ds_nocrs) xr_assert_equal( warp.Reprojection(dst_crs=dst_crs).apply(ds), warp.Reprojection(src_crs=src_crs, dst_crs=dst_crs).apply(ds_nocrs) )
def test_reproject_with_extra_dims(dims): crs1 = warp._parse_crs('epsg:4326') crs2 = warp._parse_crs('epsg:3395') ds = generate_test_dataset( dims=dims, crs=crs1 ) proj = warp.Reprojection(crs=crs2) reprojected = proj.apply(ds) # Check that a reprojected slice of the dataset is the same as # the slice of the reprojection of the entire dataset. slices = [ {'band': 3}, {'time': slice(1, 3)} ] for s in slices: xr_assert_equal( proj.apply(ds.isel(**s)), reprojected.isel(**s) )
def test_broadcast(dims, feature_dims): ds, labels = create_mock_classes(dims) expected_shape = classify._get_data_shape(ds, feature_dims=feature_dims) # Check broadcast for numpy array blabels = classify._broadcast_labels(labels.values, ds, feature_dims) assert blabels.shape == expected_shape # Check broadcast for DataArray blabels = classify._broadcast_labels(labels.transpose('x', 'y'), ds, feature_dims) blabels_t = classify._broadcast_labels(labels.transpose('y', 'x'), ds, feature_dims) xr_assert_equal(blabels, blabels_t) assert blabels.shape == expected_shape # Check values equal along broadcast dimensions bc_dims = set(dims) - set(labels.dims) - set(feature_dims) for d in bc_dims: assert (blabels.std(d) == 0).all()
def test_reproject_one_dimensional_vars(): ds = generate_test_dataset(crs=epsg4326) ds['xvar'] = (('x',), ds.x.values * 10) ds['yvar'] = (('y',), ds.y.values * 10) ds['timevar'] = (('time',), np.random.rand(ds.dims['time'])) warped = warp.Reprojection( crs=sinusoidal, resampling=rasterio.warp.Resampling.bilinear ).apply(ds) xr_assert_equal( warped['timevar'], ds['timevar']) # Check that xvar and yvar are still proportional to longitude # and latitude for v, coord in [ ('xvar', 'lon'), ('yvar', 'lat') ]: ratios = warped[v].values / warped[coord].values print(v, coord) print(np.nanmin(ratios), np.nanmax(ratios), np.nanstd(ratios)) # There is quite a bit of error, for now accept relatively # high error. assert np.nanstd(ratios) < 1.5
def test_expand_var_to_xy(extra, dims): ref = generate_test_dataarray( dims=OrderedDict([('y', 30), ('x', 20), ('time', 5)]) ) da = generate_test_dataarray(dims=dims) expanded = warp._expand_var_to_xy(da, ref.coords) # Check that the variance along added dimensions is zero assert np.all( expanded.var(extra) < 1e-16 ) # Check that the new DataArray contains x and y coordinates xr_assert_equal( expanded.coords['x'], ref.coords['x']) xr_assert_equal( expanded.coords['y'], ref.coords['y']) # Check that the DataArray is unchanged if x and y were already # dimensions if 'x' in dims and 'y' in dims: xr_assert_equal(da, expanded)
def test_auto_merge(): xr_assert_equal(ds, tiling.auto_merge(parts))
def test_parallelized_apply(generator, njobs): ds = generator() algo = ParallelDummyAlgorithm(3) ref = algo.apply(ds) result = algo.apply(ds, njobs=njobs) xr_assert_equal(ref, result)
def test_accessor_nd_resample(generator): ds = generator() kwargs = dict(width=50) xr_assert_equal(warp.resample(ds, **kwargs), ds.nd.resample(**kwargs))
def test_auto_merge_with_buffer(): xr_assert_equal(ds, tiling.auto_merge(buffered_parts))
def test_auto_merge_with_buffer(use_xarray): xr_assert_equal( ds, tiling.auto_merge(buffered_parts, use_xarray_combine=use_xarray))
def test_auto_merge(use_xarray): xr_assert_equal(ds, tiling.auto_merge(parts, use_xarray_combine=use_xarray))
def test_tile_and_merge(tmpdir, chunks, buffer): tile_path = tmpdir / 'tiles' tiling.tile(ds, str(tile_path), chunks=chunks, buffer=buffer) merged = tiling.auto_merge(str(tile_path / '*.nc')) xr_assert_equal(merged, ds)