def test_fillnodata_mask_ones(): # when mask is all ones, image should be unmodified a = np.ones([3, 3]) * 42 a[1][1] = 0 mask = np.ones([3, 3]) result = fillnodata(a, mask) assert (np.all(a == result))
def interpolateRaster(dirpath, out_fp): search_criteria = "*.tif" q = os.path.join(dirpath, search_criteria) # glob function can be used to list files from a directory with specific criteria dem_fps = glob.glob(q) print(len(dem_fps)) for fp in tqdm(dem_fps): # print(fp) # print("interplate_dtm_%s"%os.path.basename(fp)) with rasterio.open(fp, 'r') as src: data = src.read(1, masked=True) msk = src.read_masks(1) #配置max_search_distance参数,或者多次执行插值,补全较大数据缺失区域 fill_raster = fillnodata( data, msk, max_search_distance=400.0, smoothing_iterations=0 ) #reference:https://rasterio.readthedocs.io/en/latest/api/rasterio.fill.html # scaler=MinMaxScaler() # plt.imshow(scaler.fit_transform(fill_raster)) # plt.imshow(scaler.fit_transform(data)) out_meta = src.meta.copy() arr = np.random.randint(5, size=(100, 100)).astype(np.float) with rasterio.open( os.path.join(out_fp, "interplate_dtm_%s" % os.path.basename(fp)), "w", **out_meta) as dest: dest.write(fill_raster, 1)
def fill_nodata(file_to_fill, mask_file=None, plot=False): raster_filepath = os.path.dirname(file_to_fill) + "/" raster_filename = os.path.basename(file_to_fill) if mask_file: with rasterio.open(mask_file) as src: masks = src.read_masks() count = src.count mask = masks[0] & masks[1] else: mask = None if plot: plt.imshow(mask, cmap='gray') plt.show() with rasterio.open(file_to_fill) as src: nodata = src.nodata kwargs = src.meta kwargs.update(dtype=rasterio.float32, count=1) ndwi = src.read(1) if plot: plt.imshow(ndwi, cmap='gray') plt.show() filled = fillnodata(ndwi, (ndwi != nodata), max_search_distance=300) if plot: plt.imshow(filled, cmap='gray') plt.show() out_filename = raster_filepath + raster_filename.split( sep=".")[0] + "_filled.tif" with rasterio.open(out_filename, 'w', **kwargs) as dst: dst.nodata = 0 dst.write_band(1, filled.astype(rasterio.float32))
def _fill_nodata(self): """Fill nodata values in elevation grid by interpolation. Wrapper around GDAL/rasterio's FillNoData, fillnodata methods """ if ~np.isnan(self.nodata_value): nodata_mask = self._griddata == self.nodata_value else: nodata_mask = np.isnan(self._griddata) self.nodata_mask = nodata_mask # XXX: GDAL (or rasterio) FillNoData takes mask with 0s at nodata num_nodata = np.sum(nodata_mask) prev_nodata = np.nan while num_nodata > 0 or num_nodata == prev_nodata: mask = np.isnan(self._griddata) col_nodata = np.sum(mask, axis=0).max() row_nodata = np.sum(mask, axis=1).max() dist = max(row_nodata, col_nodata) / 2 self._griddata = fillnodata(self._griddata, mask=~mask, max_search_distance=dist) prev_nodata = copy(num_nodata) num_nodata = np.sum(np.isnan(self._griddata)) self.is_interpolated = True
def _fill_tile(r, c, row_min, row_max): # rows = np.linspace(0, ids.shape[0], num_rows, dtype=int) rows = np.linspace(row_min, row_max, num_rows, dtype=int) cols = np.linspace(0, ids.shape[1], num_cols, dtype=int) r_buffer_b = kernel_size if r != 0 else 0 r_buffer_t = kernel_size if r != num_rows - 2 else 0 c_buffer_l = kernel_size if c != 0 else 0 c_buffer_r = kernel_size if c != num_cols - 2 else 0 window_write = ((rows[r], rows[r + 1]), (cols[c], cols[c + 1])) window_read = ((rows[r] - r_buffer_b, rows[r + 1] + r_buffer_t), (cols[c] - c_buffer_l, cols[c + 1] + c_buffer_r)) print('Write window {}'.format(window_write)) print('Read window due to patch {}'.format(window_read)) tile_data = ids.read(1, masked=True, window=window_read) print(tile_data.shape) if tile_data.count( ) == tile_data.size: # all unmasked pixels, nothing to do orig_data = ids.read(1, masked=True, window=window_write) return Tile(data=orig_data, window=window_write) elif tile_data.count() == 0: # all masked pixels, can't do filling orig_data = ids.read(1, masked=True, window=window_write) return Tile(data=orig_data, window=window_write) data_filled = fillnodata(tile_data, mask=None, max_search_distance=kernel_size) # return r_buffer_b, r_buffer_t, c_buffer_l, c_buffer_r, data_filled, window_write data_write = data_filled[r_buffer_b:data_filled.shape[0] - r_buffer_t, c_buffer_l:data_filled.shape[1] - c_buffer_r] return Tile(data=data_write, window=window_write)
def test_fillnodata_mask_ones(): # when mask is all ones, image should be unmodified a = numpy.ones([3, 3]) * 42 a[1][1] = 0 mask = numpy.ones([3, 3]) result = fillnodata(a, mask) assert(numpy.all(a == result))
def findPatch(heightmap, dbg=False): mask = heightmap > 0 filled = binary_fill_holes(mask) filled_pixels = filled.astype(np.float32) - mask.astype(np.float32) if dbg: img(filled_pixels, os.path.join(OUTPUT, '_pixels_to_interpolate.png')) img(heightmap, os.path.join(OUTPUT, '_heightmap.png')) interpolated_holes = fillnodata(heightmap, mask=filled_pixels == 0) if dbg: img(interpolated_holes, os.path.join(OUTPUT, '_interpolated_holes.png')) edt = distance_transform_edt(filled) if dbg: img(edt, os.path.join(OUTPUT, '_edt.png')) patch_size = edt.max() * np.sqrt(2) coord = np.unravel_index(edt.argmax(), edt.shape) return patch_size, coord
def interpolate(raster, max_search_distance=10): with rasterio.open(raster) as src: profile = src.profile arr = src.read(1) arr_filled = fillnodata(arr, mask=src.read_masks(1), max_search_distance=max_search_distance) with rasterio.open(raster, 'w', **profile) as dest: dest.write_band(1, arr_filled)
def test_fillnodata(): """Test filling nodata values in an ndarray""" # create a 5x5 array, with some missing data a = numpy.ones([3, 3]) * 42 a[1][1] = 0 # find the missing data mask = ~(a == 0) # fill the missing data using interpolation from the edges result = fillnodata(a, mask) assert(numpy.all((numpy.ones([3, 3]) * 42) == result))
def test_fillnodata(): """Test filling nodata values in an ndarray""" # create a 5x5 array, with some missing data a = np.ones([3, 3]) * 42 a[1][1] = 0 # find the missing data mask = ~(a == 0) # fill the missing data using interpolation from the edges result = fillnodata(a, mask) assert (np.all((np.ones([3, 3]) * 42) == result))
def fill_nodata(self): """ A parallelized version is presented here: https://github.com/basaks/rasterio/blob/master/examples/fill_large_raster.py """ tmpfile = tempfile.NamedTemporaryFile(prefix=tmpdir) with rasterio.open(tmpfile.name, 'w', **self._src.meta.copy()) as dst: for window in self.iter_windows(): dst.write(fillnodata(self._src.read(window=window, masked=True)), window=window) self._tmpfile = tmpfile
def thicket_agc_post_proc(image_mapper=ImageMapper()): """ Post process thicket AGC map - helper function for MsImageMapper Writes out a cleaned version of map generated by ImageMapper.map(...), ills nodata and places sensible limits on AGC values. Parameters ---------- image_mapper : ImageMapper instance of ImageMapper that has generated map Returns ------- post processed raster file name """ with rasterio.Env(): with rasterio.open(image_mapper.map_file_name, 'r') as in_ds: out_profile = in_ds.profile out_profile.update(count=1) split_ext = os.path.splitext(image_mapper.map_file_name) out_file_name = '{0}_postproc{1}'.format(split_ext[0], split_ext[1]) with rasterio.open(out_file_name, 'w', **out_profile) as out_ds: if (not out_profile['tiled']) or (np.prod(in_ds.shape) < 10e6): in_windows = enumerate( [Window(0, 0, in_ds.width, in_ds.height)]) # read whole raster at once else: in_windows = in_ds.block_windows(1) # read in blocks for ji, block_win in in_windows: in_block = in_ds.read(1, window=block_win, masked=True) in_block[in_block < 0] = 0 in_block.mask = (in_block.mask.astype(np.bool) | (in_block > 95) | (in_block < 0)).astype( rasterio.uint8) in_mask = in_block.mask.copy() sieved_msk = sieve(in_mask.astype(rasterio.uint8), size=2000) out_block = fill.fillnodata(in_block, mask=None, max_search_distance=20, smoothing_iterations=1) out_block[sieved_msk.astype(np.bool)] = image_mapper.nodata out_ds.write(out_block, indexes=1, window=block_win) return out_file_name
def mask_raster(raster_path, mask_path, crs): with fiona.open(mask_path, "r") as shapefile: shapes = [feature["geometry"] for feature in shapefile] with rasterio.open(raster_path) as src: out_image, out_transform = rasterio.mask.mask(src, shapes, crop=True) out_image[out_image < 0] = np.nan mask = (out_image != 0) out_image = fillnodata(out_image, mask) out_meta = src.meta out_meta.update({ "driver": "GTiff", "height": out_image.shape[1], "width": out_image.shape[2], "transform": out_transform, "crs": crs }) return out_image, out_meta
def fill_internal_nodata(img: str, out: str, aoi: str) -> None: """ Fills internal NoData gaps by interpolating across them. 'Interal' gaps are definied as those within aoi polygon. TODO: Areas outside of AOI polygon are masked regardless of if they are NoData are not, so this effectively only works if the img is clipped to the AOI already. Workaround would be to create the aoi on the fly from the edges of the valid data in the img. """ temp_filled = r'/vsimem/temp_filled.tif' with rio.open(img) as src: with rio.Env(): profile = src.profile with rio.open(temp_filled, 'w', **profile) as dst: for i in range(src.count): b = i + 1 arr = src.read(b) mask = src.read_masks(b) filled = fillnodata(arr, mask=mask) dst.write(filled.astype(src.dtypes[i]), b) # with fiona.open(aoi) as shapefile: # shapes = [feature['geometry'] for feature in shapefile] gdf = read_vec(aoi) shapes = gdf.geometry.values # Warn if not same CRS with rio.open(temp_filled) as src: if gdf.crs.to_wkt() != src.crs.to_wkt(): logger.warning('AOI and raster to-be-filled do not have matching' 'CRS:\nAOI:{}\nRaster:{}'.format(gdf.crs, src.crs)) out_img, out_trans = rasterio.mask.mask(src, shapes) with rio.open(out, 'w', **profile) as dst: dst.write(out_img)
dest_dir = '/media/rmsare/GALLIUMOS/data/ot_data/nsaf_6km/' files = os.listdir(proc_dir) files = sort_by_utm_northing(files) pbar = ProgressBar(widgets=[Percentage(), ' ', Bar(), ' ', ETA()], maxval=len(files)) pbar.start() for i, f in enumerate(files): padded_data = pad_with_neighboring_values(f, pad, data_dir=source_dir) nodata_mask = ~np.isnan(padded_data) num_nodata = np.sum(~nodata_mask) while num_nodata > 0: padded_data = fillnodata(padded_data, mask=nodata_mask) nodata_mask = ~np.isnan(padded_data) num_nodata = np.sum(~nodata_mask) nrows, ncols = padded_data.shape data_file = source_dir + f inraster = gdal.Open(data_file) transform = inraster.GetGeoTransform() driver = gdal.GetDriverByName('GTiff') outraster = driver.Create(dest_dir + f, ncols, nrows, 1, gdal.GDT_Float32) outraster.SetGeoTransform(transform) out_band = outraster.GetRasterBand(1) out_band.WriteArray(padded_data)
def calc(ctx, command, files, output, name, dtype, masked, overwrite, mem_limit, creation_options): """A raster data calculator Evaluates an expression using input datasets and writes the result to a new dataset. Command syntax is lisp-like. An expression consists of an operator or function name and one or more strings, numbers, or expressions enclosed in parentheses. Functions include ``read`` (gets a raster array) and ``asarray`` (makes a 3-D array from 2-D arrays). \b * (read i) evaluates to the i-th input dataset (a 3-D array). * (read i j) evaluates to the j-th band of the i-th dataset (a 2-D array). * (take foo j) evaluates to the j-th band of a dataset named foo (see help on the --name option above). * Standard numpy array operators (+, -, *, /) are available. * When the final result is a list of arrays, a multiple band output file is written. * When the final result is a single array, a single band output file is written. Example: \b $ rio calc "(+ 2 (* 0.95 (read 1)))" tests/data/RGB.byte.tif \\ > /tmp/out.tif The command above produces a 3-band GeoTIFF with all values scaled by 0.95 and incremented by 2. \b $ rio calc "(asarray (+ 125 (read 1)) (read 1) (read 1))" \\ > tests/data/shade.tif /tmp/out.tif The command above produces a 3-band RGB GeoTIFF, with red levels incremented by 125, from the single-band input. The maximum amount of memory used to perform caculations defaults to 64 MB. This number can be increased to improve speed of calculation. """ import numpy as np try: with ctx.obj['env']: output, files = resolve_inout(files=files, output=output, overwrite=overwrite) inputs = ([tuple(n.split('=')) for n in name] + [(None, n) for n in files]) sources = [rasterio.open(path) for name, path in inputs] first = sources[0] kwargs = first.profile kwargs.update(**creation_options) dtype = dtype or first.meta['dtype'] kwargs['dtype'] = dtype # Extend snuggs. snuggs.func_map['read'] = _read_array snuggs.func_map['band'] = lambda d, i: _get_bands(inputs, sources, d, i) snuggs.func_map['bands'] = lambda d: _get_bands(inputs, sources, d) snuggs.func_map['fillnodata'] = lambda *args: fillnodata(*args) snuggs.func_map['sieve'] = lambda *args: sieve(*args) dst = None # The windows iterator is initialized with a single sample. # The actual work windows will be added in the second # iteration of the loop. work_windows = [(None, Window(0, 0, 16, 16))] for ij, window in work_windows: ctxkwds = OrderedDict() for i, ((name, path), src) in enumerate(zip(inputs, sources)): # Using the class method instead of instance # method. Latter raises # # TypeError: astype() got an unexpected keyword # argument 'copy' # # possibly something to do with the instance being # a masked array. ctxkwds[name or '_i%d' % (i + 1)] = src.read(masked=masked, window=window) res = snuggs.eval(command, **ctxkwds) if (isinstance(res, np.ma.core.MaskedArray) and ( tuple(LooseVersion(np.__version__).version) < (1, 9) or tuple(LooseVersion(np.__version__).version) > (1, 10))): res = res.filled(kwargs['nodata']) if len(res.shape) == 3: results = np.ndarray.astype(res, dtype, copy=False) else: results = np.asanyarray( [np.ndarray.astype(res, dtype, copy=False)]) # The first iteration is only to get sample results and from them # compute some properties of the output dataset. if dst is None: kwargs['count'] = results.shape[0] dst = rasterio.open(output, 'w', **kwargs) work_windows.extend(_chunk_output(dst.width, dst.height, dst.count, np.dtype(dst.dtypes[0]).itemsize, mem_limit=mem_limit)) # In subsequent iterations we write results. else: dst.write(results, window=window) except snuggs.ExpressionError as err: click.echo("Expression Error:") click.echo(' %s' % err.text) click.echo(' ' + ' ' * err.offset + "^") click.echo(err) raise click.Abort() finally: if dst: dst.close() for src in sources: src.close()
def test_fillnodata_invalid_types(): a = numpy.ones([3, 3]) with pytest.raises(ValueError): fillnodata(None, a) with pytest.raises(ValueError): fillnodata(a, 42)
def calc(ctx, command, files, name, dtype): """A raster data calculator Evaluates an expression using input datasets and writes the result to a new dataset. Command syntax is lisp-like. An expression consists of an operator or function name and one or more strings, numbers, or expressions enclosed in parentheses. Functions include ``read`` (gets a raster array) and ``asarray`` (makes a 3-D array from 2-D arrays). \b * (read i) evaluates to the i-th input dataset (a 3-D array). * (read i j) evaluates to the j-th band of the i-th dataset (a 2-D array). * (take foo j) evaluates to the j-th band of a dataset named foo (see help on the --name option above). * Standard numpy array operators (+, -, *, /) are available. * When the final result is a list of arrays, a multi band output file is written. * When the final result is a single array, a single band output file is written. Example: \b $ rio calc "(+ 2 (* 0.95 (read 1)))" tests/data/RGB.byte.tif \\ > /tmp/out.tif Produces a 3-band GeoTIFF with all values scaled by 0.95 and incremented by 2. \b $ rio calc "(asarray (+ 125 (read 1)) (read 1) (read 1))" \\ > tests/data/shade.tif /tmp/out.tif Produces a 3-band RGB GeoTIFF, with red levels incremented by 125, from the single-band input. """ import numpy as np verbosity = (ctx.obj and ctx.obj.get('verbosity')) or 1 logger = logging.getLogger('rio') try: with rasterio.drivers(CPL_DEBUG=verbosity > 2): output = files[-1] inputs = ([tuple(n.split('=')) for n in name] + [(None, n) for n in files[:-1]]) with rasterio.open(inputs[0][1]) as first: kwargs = first.meta kwargs['transform'] = kwargs.pop('affine') dtype = dtype or first.meta['dtype'] kwargs['dtype'] = dtype ctxkwds = {} for i, (name, path) in enumerate(inputs): with rasterio.open(path) as src: # Using the class method instead of instance # method. Latter raises # # TypeError: astype() got an unexpected keyword # argument 'copy' # # possibly something to do with the instance being # a masked array. ctxkwds[name or '_i%d' % (i+1)] = src.read() # Extend snuggs. snuggs.func_map['read'] = read_array snuggs.func_map['band'] = lambda d, i: get_bands(inputs, d, i) snuggs.func_map['bands'] = lambda d: get_bands(inputs, d) snuggs.func_map['fillnodata'] = lambda *args: fillnodata(*args) snuggs.func_map['sieve'] = lambda *args: sieve(*args) res = snuggs.eval(command, **ctxkwds) if len(res.shape) == 3: results = np.ndarray.astype(res, dtype, copy=False) else: results = np.asanyarray( [np.ndarray.astype(res, dtype, copy=False)]) kwargs['count'] = results.shape[0] with rasterio.open(output, 'w', **kwargs) as dst: dst.write(results) sys.exit(0) except snuggs.ExpressionError as err: click.echo("Expression Error:") click.echo(' %s' % err.text) click.echo(' ' + ' ' * err.offset + "^") click.echo(err) sys.exit(1) except Exception as err: t, v, tb = sys.exc_info() for line in traceback.format_exception_only(t, v): click.echo(line, nl=False) sys.exit(1)
def calc(ctx, command, files, output, name, dtype, masked, overwrite, creation_options): """A raster data calculator Evaluates an expression using input datasets and writes the result to a new dataset. Command syntax is lisp-like. An expression consists of an operator or function name and one or more strings, numbers, or expressions enclosed in parentheses. Functions include ``read`` (gets a raster array) and ``asarray`` (makes a 3-D array from 2-D arrays). \b * (read i) evaluates to the i-th input dataset (a 3-D array). * (read i j) evaluates to the j-th band of the i-th dataset (a 2-D array). * (take foo j) evaluates to the j-th band of a dataset named foo (see help on the --name option above). * Standard numpy array operators (+, -, *, /) are available. * When the final result is a list of arrays, a multi band output file is written. * When the final result is a single array, a single band output file is written. Example: \b $ rio calc "(+ 2 (* 0.95 (read 1)))" tests/data/RGB.byte.tif \\ > /tmp/out.tif Produces a 3-band GeoTIFF with all values scaled by 0.95 and incremented by 2. \b $ rio calc "(asarray (+ 125 (read 1)) (read 1) (read 1))" \\ > tests/data/shade.tif /tmp/out.tif Produces a 3-band RGB GeoTIFF, with red levels incremented by 125, from the single-band input. """ import numpy as np try: with ctx.obj['env']: output, files = resolve_inout(files=files, output=output, overwrite=overwrite) inputs = ([tuple(n.split('=')) for n in name] + [(None, n) for n in files]) with rasterio.open(inputs[0][1]) as first: kwargs = first.meta kwargs.update(**creation_options) dtype = dtype or first.meta['dtype'] kwargs['dtype'] = dtype ctxkwds = OrderedDict() for i, (name, path) in enumerate(inputs): with rasterio.open(path) as src: # Using the class method instead of instance # method. Latter raises # # TypeError: astype() got an unexpected keyword # argument 'copy' # # possibly something to do with the instance being # a masked array. ctxkwds[name or '_i%d' % (i + 1)] = src.read(masked=masked) # Extend snuggs. snuggs.func_map['read'] = read_array snuggs.func_map['band'] = lambda d, i: get_bands(inputs, d, i) snuggs.func_map['bands'] = lambda d: get_bands(inputs, d) snuggs.func_map['fillnodata'] = lambda *args: fillnodata(*args) snuggs.func_map['sieve'] = lambda *args: sieve(*args) res = snuggs.eval(command, ctxkwds) if (isinstance(res, np.ma.core.MaskedArray) and ( tuple(LooseVersion(np.__version__).version) < (1, 9) or tuple(LooseVersion(np.__version__).version) > (1, 10))): res = res.filled(kwargs['nodata']) if len(res.shape) == 3: results = np.ndarray.astype(res, dtype, copy=False) else: results = np.asanyarray( [np.ndarray.astype(res, dtype, copy=False)]) kwargs['count'] = results.shape[0] with rasterio.open(output, 'w', **kwargs) as dst: dst.write(results) except snuggs.ExpressionError as err: click.echo("Expression Error:") click.echo(' %s' % err.text) click.echo(' ' + ' ' * err.offset + "^") click.echo(err) raise click.Abort()
#plt.imshow(img) ent = entropy(ndem, disk(2)) maske=ent<1.5 if plotear: plt.imshow(maske) #%% Etapa 1. Cierre: generación de la máscara como intersección de ambas máscaras mask=maske*mask3 if plotear: plt.imshow(mask) #%% Etapa 2. Interpolación # borro los datos fuera de la máscara # y completo los faltantes con un peso inverso a la distancia y radio de 100 mdt=fillnodata(dem*mask,mask*1, max_search_distance=100) if plotear: plt.imshow(mdt) #%% #diferentes cosas para ver if plotear: plt.imshow(mdt) plt.imshow(mask3) plt.imshow(maske) plt.imshow(mask) plt.figure() plt.imshow(mdt,vmin=hmin,vmax=30)
def get_aot(self, data, sza, meta, data_values='dn', angle_factor=0.01, dn_interp=None, interp_method='fast', aot_fallback=0.3, h2o=2.0, o3=0.3, altitude=0.0, w=None, n_jobs=1): """ Gets the aerosol optical thickness (AOT) from dark objects Args: data (DataArray): The digital numbers or top of atmosphere reflectance at a coarse resolution. sza (float | DataArray): The solar zenith angle. meta (Optional[namedtuple]): A metadata object with gain and bias coefficients. data_values (Optional[str]): The values of ``data``. Choices are ['dn', 'toar']. angle_factor (Optional[float]): The scale factor for angles. dn_interp (Optional[DataArray]): A source ``DataArray`` at the target resolution. interp_method (Optional[str]): The LUT interpolation method. Choices are ['fast', 'slow']. 'fast': Uses nearest neighbor lookup with ``scipy.interpolate.NearestNDInterpolator``. 'slow': Uses linear interpolation with ``scipy.interpolate.LinearNDInterpolator``. aot_fallback (Optional[float | DataArray]): The aerosol optical thickness fallback if no dark objects are found (unitless). [0,3]. h2o (Optional[float]): The water vapor (g/m^2). [0,8.5]. o3 (Optional[float]): The ozone (cm-atm). [0,8]. altitude (Optional[float]): The altitude over the sensor acquisition location (km above sea level). w (Optional[int]): The smoothing window size (in pixels). n_jobs (Optional[int]): The number of parallel jobs for ``moving_window`` and ``dask.compute``. Returns: ``xarray.DataArray``: Data range: 0-3 References: See :cite:`masek_etal_2006`, :cite:`kaufman_etal_1997`, and :cite:`ouaidrari_vermote_1999`. """ if data_values not in ['dn', 'toar']: logger.exception(" The data values should be 'dn' or 'toar'") raise NameError if isinstance(sza, xr.DataArray): sza = sza.squeeze().data.compute(num_workers=n_jobs) sza *= angle_factor band_names = data.band.values.tolist() doy = meta.date_acquired.timetuple().tm_yday if data_values == 'dn': m_p = coeffs_to_array(meta.m_p, band_names) a_p = coeffs_to_array(meta.a_p, band_names) m_l = coeffs_to_array(meta.m_l, band_names) a_l = coeffs_to_array(meta.a_l, band_names) toar = self.dn_to_toar(data, m_p, a_p, sun_angle=False) rad = self.dn_to_radiance(data, m_l, a_l) else: toar = data rad = self.toar_to_rad(data, meta) # Get the SWIR2 band TOAR swir2_toar = toar.sel(band='swir2') # Get the blue band Radiance blue_rad = rad.sel(band='blue') # Get SWIR2 TOAR dark pixels swir2_toar_dark = xr.where((swir2_toar >= 0.01) & (swir2_toar <= 0.15), swir2_toar, np.nan) blue_rad_dark = xr.where((swir2_toar >= 0.01) & (swir2_toar <= 0.15), blue_rad, np.nan) # Estimate the blue surface reflectance with # a simple linear transformation (Masek et al., 2006) blue_p = swir2_toar_dark * 0.33 # Get reflectance and radiance data as numpy arrays blue_p_data = blue_p.squeeze().data.compute(num_workers=n_jobs) blue_rad_dark_data = blue_rad_dark.squeeze().data.compute( num_workers=n_jobs) valid_idx = np.where(~np.isnan(blue_p_data)) if valid_idx[0].shape[0] > 0: aot = self.get_optimized_aot(blue_rad_dark_data, blue_p_data, meta.sensor, 'blue', interp_method, sza, doy, h2o, o3, altitude) mask = np.ones(aot.shape, dtype='uint8') mask[np.isnan(blue_p_data)] = 0 aot = fillnodata(aot, mask=mask, max_search_distance=100) if isinstance(dn_interp, xr.DataArray): aot = self._resize(aot, dn_interp, w, n_jobs) return ndarray_to_xarray(dn_interp, aot, ['aot']) else: return ndarray_to_xarray(data, aot, ['aot']) else: if isinstance(dn_interp, xr.DataArray): return ndarray_to_xarray( dn_interp, np.zeros((dn_interp.gw.nrows, dn_interp.gw.ncols), dtype='float64') + aot_fallback, ['aot']) else: return ndarray_to_xarray( data, np.zeros((data.gw.nrows, data.gw.ncols), dtype='float64') + aot_fallback, ['aot'])
def fillna_in_raster(dir: str, aqi_tif_name: str, na_val: float = 1.0, log: Logger = None) -> bool: """Fills nodata values in a raster by interpolating values from surrounding cells. Value 1.0 is considered as nodata. If no nodata is found with that value, a small offset will be applied, as sometimes the nodata value is slightly higher than 1.0 (assumably due to inaccuracy in netcdf to geotiff conversion). Args: aqi_tif_name: The name of a raster file to be processed (in aqi_cache directory). na_val: A value that represents nodata in the raster. """ # open AQI band from AQI raster file aqi_filepath = dir + aqi_tif_name aqi_raster = rasterio.open(aqi_filepath) aqi_band = aqi_raster.read(1) # create a nodata mask (map nodata values to 0) # nodata value may be slightly higher than 1.0, hence try different offsets na_offset = 0 for offset in [0.0, 0.01, 0.02, 0.04, 0.06, 0.08, 0.1, 0.12]: na_offset = na_val + offset nodata_count = np.sum(aqi_band <= na_offset) if log: log.info(f'Nodata offset: {offset} / nodata count: {nodata_count}') # check if nodata values can be mapped with the current offset if (nodata_count > 180000): break if (nodata_count < 180000): if log: log.info( f'Failed to set nodata values in the AQI tif, nodata count: {nodata_count}' ) aqi_nodata_mask = np.where(aqi_band <= na_offset, 0, aqi_band) # fill nodata in aqi_band using nodata mask aqi_band_fillna = fill.fillnodata(aqi_band, mask=aqi_nodata_mask) # validate AQI values after na fill invalid_count = np.sum(aqi_band_fillna < 1.0) if (invalid_count > 0): if log: log.warning( f'AQI band has {invalid_count} below 1 aqi values after na fill' ) # write raster with filled nodata aqi_raster_fillna = rasterio.open(aqi_filepath, 'w', driver='GTiff', height=aqi_raster.shape[0], width=aqi_raster.shape[1], count=1, dtype='float32', transform=aqi_raster.transform, crs=aqi_raster.crs) aqi_raster_fillna.write(aqi_band_fillna, 1) aqi_raster_fillna.close() return True
def mosaic_tiles(dir_tiles, roi_xmin, roi_ymin, roi_xmax, roi_ymax, roi_transform, roi_width, roi_length, roi_crs, tile_dtype, nodata_value, kbl_dir, kbl_scale=0, query_string="", resampling="nearest", fill_nodata=True, mosaic_filename=None): """ Combine KBL tiles into mosaic Inputs dir_tiles: str directory to image tiles roi_xmin, roi_ymin, roi_xmax, roi_ymax: float coordinates of ROI roi_transform: rasterio transform ROI transform roi_width: int ROI width (no. of columns) roi_length: int ROI length (no. of rows) roi_crs: str CRS in which ROI limits are expressed (format "EPSG:code") tile_dtype: numpy dtype mosaic datatype nodata_value: int/float value for no data kbl_scale: int KBL scale (0, 8, 16) query_string: str query string to search for tile files resampling: str resampling mode for warp_to_transform fill_nodata: bool whether to fill no data areas mosaic_filename: str path to store mosaic Outputs mos: nd array mosaic image """ tiles_id = find_tiles_kbl(roi_xmin, roi_ymin, roi_xmax, roi_ymax, roi_crs, kbl_dir, kbl_scale=kbl_scale) if kbl_scale == 0: # TODO: add for kbl_scale 4 tiles = [ glob( os.path.join(dir_tiles, "{}*{:02d}.tif".format(query_string, int(i)))) for i in tiles_id ] elif kbl_scale == 8: tiles = [ glob( os.path.join( dir_tiles, "{}*{:03d}.tif".format(query_string, int(i.replace("/", ""))))) for i in tiles_id ] elif kbl_scale == 16: tiles = [ glob( os.path.join( dir_tiles, "{}*{:03d}{}.tif".format(query_string, int(i[:-1].replace("/", "")), i[-1].lower()))) for i in tiles_id ] tiles = [val for sublist in tiles for val in sublist] if len(tiles) == 0: print("Error! No tiles covering ROI found.") return None mos = np.ones((roi_length, roi_width), dtype=tile_dtype) * nodata_value for tile in tiles: tile_arr, tile_nd = warp_to_transform(os.path.join(dir_tiles, tile), roi_transform, roi_width, roi_length, roi_crs, return_nodata=True, resampling=resampling) tile_arr[tile_arr == tile_nd] = nodata_value mos[(mos == nodata_value) & (tile_arr != tile_nd)] = tile_arr[(mos == nodata_value) & (tile_arr != tile_nd)] if fill_nodata & (np.sum(mos == nodata_value) > 0): mos = fill.fillnodata(mos, mos != nodata_value, max_search_distance=100.0, smoothing_iterations=0) if mosaic_filename is not None: ar2tif(mos, mosaic_filename, roi_crs, roi_transform, dtype=mos.dtype, band_names=["LC"], nodata_value=nodata_value) return mos
def test_fillnodata(hole_in_ones): """Test filling nodata values in an ndarray""" mask = hole_in_ones == 1 result = fillnodata(hole_in_ones, mask) assert (result == 1).all()
def calc(ctx, command, files, output, name, dtype, masked, creation_options): """A raster data calculator Evaluates an expression using input datasets and writes the result to a new dataset. Command syntax is lisp-like. An expression consists of an operator or function name and one or more strings, numbers, or expressions enclosed in parentheses. Functions include ``read`` (gets a raster array) and ``asarray`` (makes a 3-D array from 2-D arrays). \b * (read i) evaluates to the i-th input dataset (a 3-D array). * (read i j) evaluates to the j-th band of the i-th dataset (a 2-D array). * (take foo j) evaluates to the j-th band of a dataset named foo (see help on the --name option above). * Standard numpy array operators (+, -, *, /) are available. * When the final result is a list of arrays, a multi band output file is written. * When the final result is a single array, a single band output file is written. Example: \b $ rio calc "(+ 2 (* 0.95 (read 1)))" tests/data/RGB.byte.tif \\ > /tmp/out.tif Produces a 3-band GeoTIFF with all values scaled by 0.95 and incremented by 2. \b $ rio calc "(asarray (+ 125 (read 1)) (read 1) (read 1))" \\ > tests/data/shade.tif /tmp/out.tif Produces a 3-band RGB GeoTIFF, with red levels incremented by 125, from the single-band input. """ import numpy as np verbosity = (ctx.obj and ctx.obj.get('verbosity')) or 1 logger = logging.getLogger('rio') try: with rasterio.drivers(CPL_DEBUG=verbosity > 2): output, files = resolve_inout(files=files, output=output) inputs = ([tuple(n.split('=')) for n in name] + [(None, n) for n in files]) with rasterio.open(inputs[0][1]) as first: kwargs = first.meta kwargs.update(**creation_options) kwargs['transform'] = kwargs.pop('affine') dtype = dtype or first.meta['dtype'] kwargs['dtype'] = dtype ctxkwds = {} for i, (name, path) in enumerate(inputs): with rasterio.open(path) as src: # Using the class method instead of instance # method. Latter raises # # TypeError: astype() got an unexpected keyword # argument 'copy' # # possibly something to do with the instance being # a masked array. ctxkwds[name or '_i%d' % (i + 1)] = src.read(masked=masked) # Extend snuggs. snuggs.func_map['read'] = read_array snuggs.func_map['band'] = lambda d, i: get_bands(inputs, d, i) snuggs.func_map['bands'] = lambda d: get_bands(inputs, d) snuggs.func_map['fillnodata'] = lambda *args: fillnodata(*args) snuggs.func_map['sieve'] = lambda *args: sieve(*args) res = snuggs.eval(command, **ctxkwds) if (isinstance(res, np.ma.core.MaskedArray) and tuple(LooseVersion(np.__version__).version) < (1, 9, 0)): res = res.filled(kwargs['nodata']) if len(res.shape) == 3: results = np.ndarray.astype(res, dtype, copy=False) else: results = np.asanyarray( [np.ndarray.astype(res, dtype, copy=False)]) kwargs['count'] = results.shape[0] with rasterio.open(output, 'w', **kwargs) as dst: dst.write(results) except snuggs.ExpressionError as err: click.echo("Expression Error:") click.echo(' %s' % err.text) click.echo(' ' + ' ' * err.offset + "^") click.echo(err) raise click.Abort()
def test_fillnodata_masked_array(hole_in_ones): """Test filling nodata values in a masked ndarray""" ma = np.ma.masked_array(hole_in_ones, (hole_in_ones == 0)) result = fillnodata(ma) assert (result == 1).all()
def fill_nodata(img, mask, fillBands, maxSearchDistance): for b in fillBands: img[b - 1] = fillnodata(img[b - 1], mask, maxSearchDistance) return img
def test_fillnodata_invalid_types(): a = np.ones([3, 3]) with pytest.raises(ValueError): fillnodata(None, a) with pytest.raises(ValueError): fillnodata(a, 42)
def calc(ctx, command, files, output, name, dtype, masked, overwrite, mem_limit, creation_options): """A raster data calculator Evaluates an expression using input datasets and writes the result to a new dataset. Command syntax is lisp-like. An expression consists of an operator or function name and one or more strings, numbers, or expressions enclosed in parentheses. Functions include ``read`` (gets a raster array) and ``asarray`` (makes a 3-D array from 2-D arrays). \b * (read i) evaluates to the i-th input dataset (a 3-D array). * (read i j) evaluates to the j-th band of the i-th dataset (a 2-D array). * (take foo j) evaluates to the j-th band of a dataset named foo (see help on the --name option above). * Standard numpy array operators (+, -, *, /) are available. * When the final result is a list of arrays, a multiple band output file is written. * When the final result is a single array, a single band output file is written. Example: \b $ rio calc "(+ 2 (* 0.95 (read 1)))" tests/data/RGB.byte.tif \\ > /tmp/out.tif The command above produces a 3-band GeoTIFF with all values scaled by 0.95 and incremented by 2. \b $ rio calc "(asarray (+ 125 (read 1)) (read 1) (read 1))" \\ > tests/data/shade.tif /tmp/out.tif The command above produces a 3-band RGB GeoTIFF, with red levels incremented by 125, from the single-band input. The maximum amount of memory used to perform caculations defaults to 64 MB. This number can be increased to improve speed of calculation. """ import numpy as np dst = None sources = [] try: with ctx.obj['env']: output, files = resolve_inout(files=files, output=output, overwrite=overwrite) inputs = ([tuple(n.split('=')) for n in name] + [(None, n) for n in files]) sources = [rasterio.open(path) for name, path in inputs] first = sources[0] kwargs = first.profile kwargs.update(**creation_options) dtype = dtype or first.meta['dtype'] kwargs['dtype'] = dtype # Extend snuggs. snuggs.func_map['read'] = _read_array snuggs.func_map['band'] = lambda d, i: _get_bands( inputs, sources, d, i) snuggs.func_map['bands'] = lambda d: _get_bands(inputs, sources, d) snuggs.func_map['fillnodata'] = lambda *args: fillnodata(*args) snuggs.func_map['sieve'] = lambda *args: sieve(*args) # The windows iterator is initialized with a single sample. # The actual work windows will be added in the second # iteration of the loop. work_windows = [(None, Window(0, 0, 16, 16))] for ij, window in work_windows: ctxkwds = OrderedDict() for i, ((name, path), src) in enumerate(zip(inputs, sources)): # Using the class method instead of instance # method. Latter raises # # TypeError: astype() got an unexpected keyword # argument 'copy' # # possibly something to do with the instance being # a masked array. ctxkwds[name or '_i%d' % (i + 1)] = src.read(masked=masked, window=window) res = snuggs.eval(command, **ctxkwds) if (isinstance(res, np.ma.core.MaskedArray) and (tuple(LooseVersion(np.__version__).version) < (1, 9) or tuple(LooseVersion(np.__version__).version) > (1, 10))): res = res.filled(kwargs['nodata']) if len(res.shape) == 3: results = np.ndarray.astype(res, dtype, copy=False) else: results = np.asanyarray( [np.ndarray.astype(res, dtype, copy=False)]) # The first iteration is only to get sample results and from them # compute some properties of the output dataset. if dst is None: kwargs['count'] = results.shape[0] dst = rasterio.open(output, 'w', **kwargs) work_windows.extend( _chunk_output(dst.width, dst.height, dst.count, np.dtype(dst.dtypes[0]).itemsize, mem_limit=mem_limit)) # In subsequent iterations we write results. else: dst.write(results, window=window) except snuggs.ExpressionError as err: click.echo("Expression Error:") click.echo(' %s' % err.text) click.echo(' ' + ' ' * err.offset + "^") click.echo(err) raise click.Abort() finally: if dst: dst.close() for src in sources: src.close()
def test_fillnodata_mask_ones(hole_in_ones): """when mask is all ones, image should be unmodified""" mask = np.ones((5, 5)) result = fillnodata(hole_in_ones, mask) assert (hole_in_ones == result).all()