def get_tiles(datasource, width=64, height=64, fold=1, extension=16): """Function to create the product matrix for the tiles from the big picture. Width & Height corresponds to the tile dimensions, whereas fold refers to the overlap of the tiles. 1 = no overlap, 2 = overlapping by a half tile, and so on.""" ncols, nrows = datasource.meta['width'], datasource.meta['height'] offsets = product(range(0, ncols, width//fold), range(0, nrows, height//fold)) # control the level of overlap big_window = windows.Window(col_off=0, row_off=0, width=ncols, height=nrows) # defining window and transfor functions (original and extended) for col_off, row_off in offsets: # creating the original tile window_original = windows.Window(col_off=col_off, row_off=row_off, width=width, height=height).intersection(big_window) transform_original = windows.transform(window_original, datasource.transform) # creating the extended boundary tile col_off_extended = col_off - extension row_off_extended = row_off - extension width_extended = width + extension*2 height_extended = height + extension*2 # windowing with the extended tile window_extended = windows.Window(col_off = col_off_extended, row_off = row_off_extended, width = width_extended, height = height_extended).intersection(big_window) transform_extended = windows.transform(window_extended, datasource.transform) yield window_original, transform_original, window_extended, transform_extended
def get_tiles(src, nr_tiles): if nr_tiles < 36: scale = int(np.ceil(np.sqrt(nr_tiles))) size = int(src.meta['width'] / scale) #get original image dimension ncols, nrows = src.meta['width'], src.meta['height'] #define offsets (iterator) based on specified tile size offsets = product(range(0, ncols, size), range(0, nrows, size)) #create a large tile from original image dimension big_window = windows.Window(col_off=0, row_off=0, width=ncols, height=nrows) #find intersection with the large tile and get the tile dimension and affine tiles = [] for col_off, row_off in offsets: tile = windows.Window(col_off=col_off, row_off=row_off, width=size, height=size).intersection(big_window) tile_affine = windows.transform(tile, src.transform) tiles.append((tile, tile_affine)) return tiles else: print('Try number of tiles < 36')
def get_tile(self, level, tile_y, tile_x): """ Returns (id_x, id_y), Window, Transformation :param tile_x: int :param level: int :param tile_y: int :return tuple """ level = int(level) tile_y = int(tile_y) tile_x = int(tile_x) ds = self.get_dataset(level) width = ds.width height = ds.height geotransform = self.get_transform(level) y_offset = tile_y * self.TileHeight x_offset = tile_x * self.TileWidth y_size = min(self.TileWidth, height - y_offset) x_size = min(self.TileHeight, width - x_offset) w = Window(x_offset, y_offset, x_size, y_size) t = transform(w, geotransform) return (tile_x + 1, tile_y + 1), w, t
def get_tiles(dataset, tile_a, tile_b): """Creates rasterio.windows for a given Band with the size of tile_a x tile_b :parameter: Open Source, tile-width in m, tile-height in m, :returns: rasterio window """ # calculate Width and Height as Distance from Origin tile_x, tile_y = (dataset.bounds.left + tile_a, dataset.bounds.top - tile_b) height, width = dataset.index(tile_x, tile_y) # get max rows and cols of dataset nols, nrows = dataset.meta['width'], dataset.meta['height'] # create offset for the window processing offsets = product(range(0, nols, width), range(0, nrows, height)) # create big_window around the whole dataset big_window = windows.Window(col_off=0, row_off=0, width=nols, height=nrows) # create custom set blocks for col_off, row_off in offsets: # get windows with the custom parameters # until it intersects with # the boundaries of the source dataset window = windows.Window(col_off=col_off, row_off=row_off, width=width, height=height).intersection(big_window) transform = windows.transform(window, dataset.transform) yield window, transform
def __init__(self, filepath, resolution=1, min_lat=None, max_lat=None, min_long=None, max_long=None): self.src = rasterio.open(filepath) self.min_lat = min_lat self.max_lat = max_lat self.min_long = min_long self.max_long = max_long self.resolution = resolution self.window = self._calc_window() if min_lat is None: self.transform_window = self.src.transform self.band = self.src.read(1) else: logging.warn('there are bugs in windowed transformation!') self.transform_window = windows.transform(self.window, self.src.transform) self.band = self.src.read(1, window=self.window) self.transform, self.width, self.height = self._calc_transform() self.array = self._reproject()
def get_tiles(ds, width, height): """ Takes as input the a tile and returns chips. ========================================== :width: Desired width of each chip. :height: Desired height of each chip. :out_path: Desired output file storage folder. :in_path: Folder where the input tile is stored. :input_filename: Name of the input tile :output_filename: Desired output file pattern =========================================== """ ncols = ds.meta["width"] nrows = ds.meta["height"] offsets = product(range(0, ncols, width), range(0, nrows, height)) big_window = windows.Window(col_off=0, row_off=0, width=ncols, height=nrows) for col_off, row_off in offsets: window = windows.Window( col_off=col_off, row_off=row_off, width=width, height=height, ).intersection(big_window) transform = windows.transform(window, ds.transform) yield window, transform
def split(infile, width, height=None): dataset = rasterio.open(infile) height = height or width raw_width = dataset.width raw_height = dataset.height raw_window = Window(0, 0, raw_width, raw_height) filename = dataset.name outfiles = [] for col_off in range(0, raw_width, width): for row_off in range(0, raw_height, height): window = Window(col_off, row_off, width, height).intersection(raw_window) meta = dataset.meta.copy() meta.update({ 'width': window.width, 'height': window.height, 'transform': transform(window, dataset.transform) }) outfile = os.path.splitext(filename)[0] + "_{}_{}_{}_{}".format(col_off, row_off, window.width, window.height) + os.path.splitext(filename)[1] with rasterio.open(outfile, 'w', **meta) as dst: dst.write(dataset.read(window=window)) # Copy existing band descriptions to the splitted images descriptions = rasterio.open(infile).descriptions for band, desc in enumerate(descriptions, start=1): if desc is not None: dst.set_band_description(band, desc) outfiles.append(outfile) return outfiles
def get_rasters_merged(raster_file1, raster_file2, output_file, tmp_dir, grid_blocks=5): p = Path(tmp_dir) tmp_files = [str(f) for f in list(p.glob('tmp*.tif'))] for f in tmp_files: os.remove(f) windows = make_windows(raster_file1, grid_blocks=grid_blocks) pbar = tqdm(enumerate(windows), total=len(windows)) for idx, window in pbar: raster1 = rio.open(raster_file1).read(1, window=window) raster2 = rio.open(raster_file2).read(1, window=window) result = np.maximum(raster1, raster2) # Save image_src = raster_file1 tmp_file = tmp_dir + 'tmp{}.tif'.format(idx) tfm = transform(window, transform=rio.open(image_src).transform) save_predictions_window(result, image_src, tmp_file, window, tfm) stitch(output_file, tmp_dir)
def get_tiles(ds, width, height): nols, nrows = ds.meta['width'], ds.meta['height'] offsets = product(range(0, nols, width), range(0, nrows, height)) big_window = windows.Window(col_off=0, row_off=0, width=nols, height=nrows) for col_off, row_off in offsets: window =windows.Window(col_off=col_off, row_off=row_off, width=width, height=height).intersection(big_window) transform = windows.transform(window, ds.transform) yield window, transform
def generate_tiles(input_tif, width=500, height=500): n_cols, n_rows = input_tif.meta['width'], input_tif.meta['height'] offsets = product(range(0, n_cols, width), range(0, n_rows, height)) full_window = windows.Window(col_off=0, row_off=0, width=n_cols, height=n_rows) for col_off, row_off in offsets: window=windows.Window(col_off=col_off, row_off=row_off, width=width, height=height).intersection(full_window) transform = windows.transform(window, input_tif.transform) yield window, transform
def _compute_image_stats_chunked(dataset: 'DatasetReader') -> Optional[Dict[str, Any]]: """Compute statistics for the given rasterio dataset by looping over chunks.""" from rasterio import features, warp, windows from shapely import geometry total_count = valid_data_count = 0 tdigest = TDigest() sstats = SummaryStats() convex_hull = geometry.Polygon() block_windows = [w for _, w in dataset.block_windows(1)] for w in block_windows: with warnings.catch_warnings(): warnings.filterwarnings('ignore', message='invalid value encountered.*') block_data = dataset.read(1, window=w, masked=True) # handle NaNs for float rasters block_data = np.ma.masked_invalid(block_data, copy=False) total_count += int(block_data.size) valid_data = block_data.compressed() if valid_data.size == 0: continue valid_data_count += int(valid_data.size) if np.any(block_data.mask): hull_candidates = RasterDriver._hull_candidate_mask(~block_data.mask) hull_shapes = [geometry.shape(s) for s, _ in features.shapes( np.ones(hull_candidates.shape, 'uint8'), mask=hull_candidates, transform=windows.transform(w, dataset.transform) )] else: w, s, e, n = windows.bounds(w, dataset.transform) hull_shapes = [geometry.Polygon([(w, s), (e, s), (e, n), (w, n)])] convex_hull = geometry.MultiPolygon([convex_hull, *hull_shapes]).convex_hull tdigest.update(valid_data) sstats.update(valid_data) if sstats.count() == 0: return None convex_hull_wgs = warp.transform_geom( dataset.crs, 'epsg:4326', geometry.mapping(convex_hull) ) return { 'valid_percentage': valid_data_count / total_count * 100, 'range': (sstats.min(), sstats.max()), 'mean': sstats.mean(), 'stdev': sstats.std(), 'percentiles': tdigest.quantile(np.arange(0.01, 1, 0.01)), 'convex_hull': convex_hull_wgs }
def _get_save_profile(self, dataset: DatasetReader, window: Window): kwargs = dataset.meta.copy() kwargs.update({ "driver": self.driver, "height": window.height, "width": window.width, "transform": transform(window, dataset.transform), "compress": "lzw", }) return kwargs
def get_preds_windowing(area, area_dict, model, tmp_dir, best_features, output, grid_blocks=5, threshold=0): # Delete tmp files from previous run if Path(output).is_file(): os.remove(output) p = Path(tmp_dir) tmp_files = [str(f) for f in list(p.glob('tmp*.tif'))] for f in tmp_files: os.remove(f) # Read bands src_file = area_dict[area]['images'][0] windows = make_windows(src_file, grid_blocks=grid_blocks) pbar = tqdm(enumerate(windows), total=len(windows)) for idx, window in pbar: pbar.set_description('Processing {}...'.format(area)) df_bands = read_bands_window(area_dict, area, window=window) df_inds = read_inds_window(area_dict, area, window=window) df_test = pd.concat((df_bands, df_inds), axis=1) df_test = rename_ind_cols(df_test) df_test = df_test.replace([np.inf, -np.inf], 0) # Prediction X_test = df_test[best_features].fillna(0) all_zeroes = (X_test.iloc[:, :-1].sum(axis=1) == 0) data = X_test features = best_features # Prettify Tiff preds = model.predict_proba(data)[:, 1] if threshold > 0: preds[(preds < threshold)] = 0 preds[all_zeroes] = -1 # Save image_src = src_file output_file = tmp_dir + 'tmp{}.tif'.format(idx) tfm = transform(window, transform=rio.open(src_file).transform) save_predictions_window(preds, image_src, output_file, window, tfm) #print('Saving to {}...'.format(output)) stitch(output, tmp_dir)
def get_tiles(ds, width=256, height=256): nols, nrows = ds.meta['width'], ds.meta['height'] offsets = product(range(0, nols, int(width / 2)), range(0, nrows, int(height / 2))) big_window = windows.Window(col_off=0, row_off=0, width=nols, height=nrows) for col_off, row_off in offsets: window = windows.Window(col_off=col_off, row_off=row_off, width=width, height=height).intersection(big_window) if window.width == width and window.height == height: transform = windows.transform(window, ds.transform) yield window, transform
def _get_window_transform(width, height): num_rows, num_cols = src.meta['height'], src.meta['width'] offsets = itertools.product(range(0, num_cols, width), range(0, num_rows, height)) big_window = windows.Window(col_off=0, row_off=0, width=num_cols, height=num_rows) for col_off, row_off in offsets: window = windows.Window(col_off=col_off, row_off=row_off, width=width, height=height).intersection(big_window) transform = windows.transform(window, src.transform) yield window, transform
def build_window(in_src, in_xy_ul, in_chip_size, in_chip_stride): out_window = windows.Window(col_off=in_xy_ul[0], row_off=in_xy_ul[1], width=in_chip_size, height=in_chip_size) out_win_transform = windows.transform(out_window, in_src.transform) col_id = in_xy_ul[1] // in_chip_stride row_id = in_xy_ul[0] // in_chip_stride out_win_id = f'{col_id}_{row_id}' out_win_bounds = windows.bounds(out_window, out_win_transform) return out_window, out_win_transform, out_win_bounds, out_win_id
def window_transform(self, window): """Get the affine transform for a dataset window. Parameters ---------- window: tuple Dataset window tuple Returns ------- transform: Affine The affine transform matrix for the given window """ transform = guard_transform(self.transform) return windows.transform(window, transform)
def _get_window_transform(width, height, transform, num_tile_subdivisions): dst_width = width // num_tile_subdivisions dst_height = height // num_tile_subdivisions offsets = itertools.product(range(0, width, dst_width), range(0, height, dst_height)) big_window = windows.Window(col_off=0, row_off=0, width=width, height=height) for col_off, row_off in offsets: dst_window = windows.Window(col_off=col_off, row_off=row_off, width=dst_width, height=dst_height).intersection(big_window) dst_transform = windows.transform(dst_window, transform) yield dst_window, dst_transform
def test_window_transform_function(path_rgb_byte_tif): with rasterio.open(path_rgb_byte_tif) as src: assert transform(((0, None), (0, None)), src.transform) == src.transform assert transform(((None, None), (None, None)), src.transform) == src.transform assert transform( ((1, None), (1, None)), src.transform).c == src.bounds.left + src.res[0] assert transform( ((1, None), (1, None)), src.transform).f == src.bounds.top - src.res[1] assert transform( ((-1, None), (-1, None)), src.transform).c == src.bounds.left - src.res[0] assert transform( ((-1, None), (-1, None)), src.transform).f == src.bounds.top + src.res[1] assert transform( Window(-1, -1, src.width + 1, src.height + 1), src.transform).f == src.bounds.top + src.res[1]
def extract_raster_window(in_ds, upperleft, lowerright): meta = in_ds.meta.copy() start_row, start_col = in_ds.index(upperleft.x, upperleft.y) stop_row, stop_col = in_ds.index(lowerright.x, lowerright.y) width = stop_col - start_col height = stop_row - start_row zone = Window(start_col, start_row, width, height) data = in_ds.read(1, window=zone) meta.update({ "driver": "GTiff", "height": zone.height, "width": zone.width, "transform": transform(zone, in_ds.transform), }) return data, meta, zone
def get_tiles(ds, width=256, height=256): out_window = [] out_transform = [] ncols, nrows = ds.meta['width'], ds.meta['height'] offsets = product(range(0, ncols, width), range(0, nrows, height)) big_window = windows.Window(col_off=0, row_off=0, width=ncols, height=nrows) for col_off, row_off in offsets: window = windows.Window(col_off=col_off, row_off=row_off, width=width, height=height).intersection(big_window) out_window.append(window) out_transform.append(windows.transform(window, ds.transform)) return out_window, out_transform
def get_tiles(ds, width, height): """ Create chip tiles generator :param ds: rasterio dataset :param width: tile width :param height: tile height :return: generator of rasterio windows and transforms for each tile to be created """ intersect_window = get_intersect_win(ds) offsets = product(range(intersect_window.col_off, intersect_window.width + intersect_window.col_off, width), range(intersect_window.row_off, intersect_window.height + intersect_window.row_off, height)) for col_off, row_off in offsets: window = windows.Window(col_off=col_off, row_off=row_off, width=width, height=height).intersection(intersect_window) transform = windows.transform(window, ds.transform) yield window, transform
def test_window_transform_function(): with rasterio.open('tests/data/RGB.byte.tif') as src: assert transform(((0, None), (0, None)), src.transform) == src.transform assert transform(((None, None), (None, None)), src.transform) == src.transform assert transform( ((1, None), (1, None)), src.transform).c == src.bounds.left + src.res[0] assert transform( ((1, None), (1, None)), src.transform).f == src.bounds.top - src.res[1] assert transform( ((-1, None), (-1, None)), src.transform).c == src.bounds.left - src.res[0] assert transform( ((-1, None), (-1, None)), src.transform).f == src.bounds.top + src.res[1]
def get_windows(self, raster, width, height, overlap=None): """ Produces regularly spaced tiles (instances of rasterio.window.Window) of size [width, height] pixels from rasterio DataSetReader (image file opened with rasterio.open) with overlap width: width of tiles (pixels) height: height of tiles (pixels) overlap: overlap of tiles in each dimension (relative units, range [0 1[) """ # check overlap if overlap is None: overlap = 0.0 else: assert (overlap >= 0.0) and (overlap < 1.0), "overlap must be in [0 1[" width_masterImg, height_masterImg = raster.meta['width'], raster.meta[ 'height'] # check that tiles are within master image assert (width <= width_masterImg) and ( height <= height_masterImg), "tiles are too large for image" # produce a list of regularly spaced horizontal offsets such that all tiles produced # from it fit into the master image, plus one offset which ensures that the right edge # of the last tile is identical to the right edge of the master window (the implication # is that the last tile has a different degree of overlap with its neighbor than the other # images) offsets_horiz = list( range(0, width_masterImg, round(width * (1.0 - overlap)))) offsets_horiz[-1] = width_masterImg - width # same for vertical offsets offsets_vert = list( range(0, height_masterImg, round(height * (1.0 - overlap)))) offsets_vert[-1] = height_masterImg - height # construct iterator offsets = product(offsets_horiz, offsets_vert) for col_off, row_off in offsets: window = windows.Window(col_off=col_off, row_off=row_off, width=width, height=height) transform = windows.transform(window, raster.transform) yield window, transform
def get_tile(self, file, width=512, height=512, col_off=0, row_off=0): with rio.open(file, dtype='float32') as data: meta = data.meta.copy() return data.read().transpose(1, 2, 0).astype(rio.float32), meta ncols, nrows = meta['width'], meta['height'] offsets = product(range(0, ncols, width), range(0, nrows, height)) big_window = wnd.Window(col_off=0, row_off=0, width=ncols, height=nrows) window = wnd.Window(col_off=col_off * width, row_off=row_off * height, width=width, height=height).intersection(big_window) transform = wnd.transform(window, data.transform) meta['transform'] = transform meta['width'], meta['height'] = window.width, window.height return data.read(window=window).transpose(1, 2, 0).astype(rio.float32), meta
def get_window(file_path, width=512, height=512, col_off=0, row_off=0): with rio.open(file_path) as data: meta = data.meta.copy() ncols, nrows = meta['width'], meta['height'] offsets = product(range(0, ncols, width), range(0, nrows, height)) full_image = wnd.Window(col_off=0, row_off=0, width=ncols, height=nrows) window = wnd.Window(col_off=col_off * width, row_off=row_off * height, width=width, height=height).intersection(full_image) transform = wnd.transform(window, meta['transform']) meta['transform'] = transform meta['width'], meta['height'] = window.width, window.height return data.read(window=window).transpose(1, 2, 0), meta
def get_mask(mask_type=mask_types[0]): assert mask_type in mask_types, "mask type must be OPAQUE or CIRRUS" fill_value = {m: i + 1 for i, m in enumerate(mask_types)} n_polys = np.sum([ poly["attributes"]["maskType"] == mask_type for poly in poly_list ]) msk = np.zeros(shape=(nrows, ncols), dtype=np.float32) if n_polys > 0: multi_polygon = MultiPolygon([ poly["geometry"] for poly in poly_list if poly["attributes"]["maskType"] == mask_type ]).buffer(0) bounds = multi_polygon.bounds bbox2read = coords.BoundingBox(*bounds) window_read = windows.from_bounds(*bbox2read, transform_) slice_read = tuple( slice(int(round(s.start)), int(round(s.stop))) for s in window_read.toslices()) out_shape = tuple([s.stop - s.start for s in slice_read]) transform_slice = windows.transform(window_read, transform_) shapes = [({ "type": "Polygon", "coordinates": [ np.stack([ p_elem["geometry"].exterior.xy[0], p_elem["geometry"].exterior.xy[1] ], axis=1).tolist() ] }, fill_value[mask_type]) for p_elem in poly_list if p_elem["attributes"]['maskType'] == mask_type] sub_msk = features.rasterize(shapes=shapes, fill=0, out_shape=out_shape, dtype=np.float32, transform=transform_slice) msk[slice_read] = sub_msk return msk
def get_tiles(ds, tile_a, tile_b): """Tiles a band into blocks with the dimension of tile_a x tile_b :parameter: Band, tile-width and tile-height :returns: rasterio window """ # calculate Width and Height as Distance from Origin x, y = (ds.bounds.left + tile_a, ds.bounds.top - tile_b) height, width = ds.index(x, y) nols, nrows = ds.meta['width'], ds.meta['height'] offsets = product(range(0, nols, width), range(0, nrows, height)) big_window = windows.Window(col_off=0, row_off=0, width=nols, height=nrows) # create custom set blocks for col_off, row_off in offsets: window = windows.Window(col_off=col_off, row_off=row_off, width=width, height=height).intersection(big_window) transform = windows.transform(window, ds.transform) yield window, transform
def get_patch_windows(img_size: ImageSize, patch_size: PatchSize, patch_residue: PatchResidue = None, img_transform=None) -> Iterator: img_w, img_h = img_size big_window = windows.Window( col_off=0, row_off=0, width=img_w, height=img_h) offsets = get_patch_offsets((img_w, img_h), patch_size, patch_residue) patch_w, patch_h = normalize_patch_size(patch_size) for col_off, row_off in offsets: window = windows.Window( col_off=col_off, row_off=row_off, width=patch_w, height=patch_h ).intersection(big_window) if img_transform: transform = windows.transform(window, img_transform) yield window, transform else: yield window
def get_tiles(dataset, width=utils.constants.width, height=utils.constants.height): """ Implementation from https://github.com/jesseniagonzalezv/App_segmentation_water_bodies :param dataset: data reader :param width: patch expected width :param height: patch expected height :return: crop windows and Affine transform """ nols, nrows = dataset.meta['width'], dataset.meta['height'] offsets = product(range(0, nols, width), range(0, nrows, height)) big_window = windows.Window(col_off=0, row_off=0, width=nols, height=nrows) for col_off, row_off in offsets: window = windows.Window(col_off=col_off, row_off=row_off, width=width, height=height).intersection(big_window) transform = windows.transform(window, dataset.transform) yield window, transform
def test_reproject_view(): """Source views are reprojected properly""" with rasterio.open("tests/data/RGB.byte.tif") as src: source = src.read(1) window = windows.Window(100, 100, 500, 500) # window = windows.get_data_window(source) reduced_array = source[window.toslices()] reduced_transform = windows.transform(window, src.transform) # Assert that we're working with a view. assert reduced_array.base is source dst_crs = dict( proj="merc", a=6378137, b=6378137, lat_ts=0.0, lon_0=0.0, x_0=0.0, y_0=0, k=1.0, units="m", nadgrids="@null", wktext=True, no_defs=True, ) out = np.empty(src.shape, dtype=np.uint8) reproject( reduced_array, out, src_transform=reduced_transform, src_crs=src.crs, dst_transform=DST_TRANSFORM, dst_crs=dst_crs, resampling=Resampling.nearest, ) assert (out > 0).sum() == 299199