def _recompress_image(
        input_image: rasterio.DatasetReader,
        output_fp: rasterio.MemoryFile,
        zlevel=9,
        block_size=(512, 512),
):
    """
    Read an image from given file pointer, and write as a compressed GeoTIFF.
    """
    # noinspection PyUnusedLocal

    block_size_y, block_size_x = block_size

    if len(input_image.indexes) != 1:
        raise ValueError(
            f"Expecting one-band-per-tif input (USGS packages). "
            f"Input has multiple layers {repr(input_image.indexes)}")

    array: numpy.ndarray = input_image.read(1)
    profile = input_image.profile
    profile.update(
        driver="GTiff",
        predictor=_PREDICTOR_TABLE[array.dtype.name],
        compress="deflate",
        zlevel=zlevel,
        blockxsize=block_size_x,
        blockysize=block_size_y,
        tiled=True,
    )

    with output_fp.open(**profile) as output_dataset:
        output_dataset.write(array, 1)
        # Copy gdal metadata
        output_dataset.update_tags(**input_image.tags())
        output_dataset.update_tags(1, **input_image.tags(1))
Exemple #2
0
 def _write_array(self, dataset: rio.DatasetReader, window: Box,
                  arr: np.ndarray) -> None:
     """Write array out to a rasterio dataset. Array must be of shape
     (C, H, W).
     """
     window = window.rasterio_format()
     if len(arr.shape) == 2:
         dataset.write_band(1, arr, window=window)
     else:
         for i, band in enumerate(arr, start=1):
             dataset.write_band(i, band, window=window)
Exemple #3
0
    def get_landsat8_ndvi(
            tile_reader: rasterio.DatasetReader,
            window: Union[rasterio.windows.Window,
                          Tuple] = None) -> np.ndarray:
        """Computes the NDVI (Normalized Difference Vegetation Index) over a tile or a section
        on the tile specified by the window, for Landsat 8 tiles.

        NDVI values are between -1.0 and 1.0 (hence "normalized"), mostly representing greenness, where any
        negative values are mainly generated from clouds, water, and snow, and values
        near zero are mainly generated from rock and bare soil. Very low values (0.1 and below)
        of NDVI correspond to barren areas of rock, sand, or snow. Moderate values (0.2 to 0.3)
        represent shrub and grassland, while high values (0.6 to 0.8) indicate temperate
        and tropical rainforests. Source: https://desktop.arcgis.com/

        For surface reflectance products, there are some out of range values in the red and near infrared bands
        for areas around water/clouds. We should get rid of these before NDVI calculation,
        otherwise the NDVI will also be out of range.
        Source: https://www.researchgate.net/post/Why_Landsat_8_NDVI_Values_are_out_of_Range_Not_in_between-1_to_1

        If we decide not to get rid of invalid values in the red and NIR bands and see some out-of-range NDVI values,
        we can check to see if these out-of-range NDVI values are a small minority in all pixels in the dataset.

        Args:
            tile_reader: a rasterio.io.DatasetReader object returned by rasterio.open()
            window: a tuple of four (col_off x, row_off y, width delta_x, height delta_y)
                to specify the section of the tile to compute NDVI over, or a rasterio Window object

        Returns:
            2D numpy array of dtype float32 of the NDVI values at each pixel
            Pixel value is set to 0 if the sum of the red and NIR value there is 0 (empty).
        """
        if window and isinstance(window, Tuple):
            window = rasterio.windows.Window(window[0], window[1], window[2],
                                             window[3])

        band_red = tile_reader.read(4,
                                    window=window,
                                    boundless=True,
                                    fill_value=0).squeeze()
        band_nir = tile_reader.read(5,
                                    window=window,
                                    boundless=True,
                                    fill_value=0).squeeze()

        sum_red_nir = band_nir + band_red

        # sum of the NIR and red bands being zero is most likely because this section is empty
        # this workaround means that the final NDVI at such pixels are 0.
        sum_red_nir[sum_red_nir == 0.0] = 1

        ndvi = (band_nir - band_red) / sum_red_nir
        return ndvi
Exemple #4
0
    def write_measurement_rio(
        self,
        name: str,
        ds: DatasetReader,
        overviews=images.DEFAULT_OVERVIEWS,
        overview_resampling=Resampling.average,
        expand_valid_data=True,
        file_id=None,
    ):
        """
        Write a measurement by reading it an open rasterio dataset

        :param ds: An open rasterio dataset

        See :func:`write_measurement` for other parameters.
        """
        if len(ds.indexes) != 1:
            raise NotImplementedError(
                f"TODO: Multi-band images not currently implemented (have {len(ds.indexes)})"
            )

        self._write_measurement(
            name,
            ds.read(1),
            images.GridSpec.from_rio(ds),
            self.names.measurement_file_path(
                self._work_path, name, "tif", file_id=file_id
            ),
            expand_valid_data=expand_valid_data,
            nodata=ds.nodata,
            overview_resampling=overview_resampling,
            overviews=overviews,
        )
def get_window_and_affine(geom: BasePolygon,
                          raster_src: DatasetReader) -> Tuple[Window, Affine]:
    """
    Get a rasterio window block from the bounding box of a vector feature and
    calculates the affine transformation needed to map the coordinates of the
    geometry onto a resulting array defined by the shape of the window.

    Args:
        geom (Shapely geometry): A geometry in the spatial reference system
            of the raster to be read.

        raster_src (rasterio file-like object): A rasterio raster source which
            will have the window operation performed and contains the base
            affine transformation.

    Returns:
        A pair of tuples which define a rectangular range that can be provided
        to rasterio for a windowed read
        See: https://mapbox.github.io/rasterio/windowed-rw.html#windowrw

        An Affine object used to transform geometry coordinates to cell values
    """

    # Create a window range from the bounds
    window: Window = raster_src.window(*geom.bounds).round_lengths(
        pixel_precision=5).round_offsets(pixel_precision=5)

    # Create a transform relative to this window
    affine = rasterio.windows.transform(window, raster_src.transform)

    return window, affine
Exemple #6
0
def _crop_img_to_shp(img: rasterio.DatasetReader, shape: shapefile.Shape,
                     out_path: _OutPath) -> bool:
    # Get the bbox to crop to
    shp_bbox = _BBox(*[round(v) for v in shape.bbox])
    img_bbox = _BBox(*list(img.bounds))
    bbox = shp_bbox.intersect(img_bbox)

    if not bbox.is_valid:
        return False

    # Crop the image
    window = bbox.to_window(img)
    data = img.read(window=window)

    # Write to the output directory
    out_path = out_path.crop_path(shp_bbox.left, shp_bbox.bottom)
    x_res, y_res = img.res
    transform = Affine.translation(bbox.left, bbox.top) * Affine.scale(
        x_res, -y_res)

    profile = img.profile
    profile.update(transform=transform,
                   height=window.height,
                   width=window.width)

    with rasterio.open(out_path, 'w', **profile) as writer:
        writer.write(data)

        # Fixes band 4 being labelled as alpha channel
        writer.colorinterp = img.colorinterp

    print(f'Created: {out_path}')
    return True
Exemple #7
0
def read_raster_from_polygon(
        src: rio.DatasetReader,
        poly: Union[Polygon, MultiPolygon]) -> np.ma.MaskedArray:
    """Read valid pixel values from all locations inside a polygon
        Uses the polygon as a mask in addition to the existing raster mask

    Args:
        src: an open rasterio dataset to read from
        poly: a shapely Polygon or MultiPolygon

    Returns:
        masked array of shape (nbands, nrows, ncols)
    """
    # get the read parameters
    window = rio.windows.from_bounds(*poly.bounds, src.transform)
    transform = rio.windows.transform(window, src.transform)

    # get the data
    data = src.read(window=window, masked=True, boundless=True)
    bands, rows, cols = data.shape
    poly_mask = geometry_mask([poly],
                              transform=transform,
                              out_shape=(rows, cols))

    # update the mask
    data[:, poly_mask] = np.ma.masked

    return data
Exemple #8
0
    def show_patch(
            tile_reader: rasterio.DatasetReader,
            bands: Union[Iterable, int],
            window: Union[rasterio.windows.Window, Tuple] = None,
            band_min: Numeric = 0,
            band_max: Numeric = 7000,
            gamma: Numeric = 1.0,
            size: Tuple[Numeric, Numeric] = (256, 256),
            return_array: bool = False) -> Union[np.ndarray, Image.Image]:
        """Show a patch of imagery.

        Args:
            tile_reader: a rasterio.io.DatasetReader object returned by rasterio.open()
            bands: list or tuple of ints, or a single int, indicating which band(s) to read. See notes
                below regarding the order of band numbers to pass in
            window: a tuple of four (col_off x, row_off y, width delta_x, height delta_y)
                to specify the section of the tile to return, or a rasterio Window object
            band_min: minimum value the pixels are clipped tp
            band_max: maximum value the pixels are clipped to
            gamma: the gamma value to use in gamma correction. Output is darker for gamma > 1, lighter if gamma < 1.
                This is not the same as GEE visParams' gamma!
            size: Used when this function is called to produce a PIL Image, i.e. only when
                `return_array` is False.
                None if do not resize, otherwise a (w, h) tuple in pixel unit.
                Default is (256, 256). (500, 500) looks better in notebooks
            return_array: True will cause this function to return a numpy array, with dtype the same as
                the original data;
                False (default) to get a PIL Image object (values scaled to be uint8 values)

        Returns:
            a PIL Image object, resized to `size`, or the (not resized, original data type) numpy array
            if `return_array` is true. The dims start with height and width, optionally
            with the channel dim at the end if greater than 1.

            rasterio read() does not pad with 0. The array returned may be smaller than the window specified

        Notes:
            - PIL renders the bands in RGB order; keep that in mind when passing in `bands` as a list or tuple
              so that the bands are mapped to Red, Green and Blue in the desired order.
            - Band index starts with 1
        """
        if isinstance(bands, int):
            bands = [
                bands
            ]  # otherwise rasterio read will return a 2D array instead of 3D

        if window and isinstance(window, Tuple):
            window = rasterio.windows.Window(window[0], window[1], window[2],
                                             window[3])

        # read as (bands, rows, columns) or (c, h, w)
        bands = tile_reader.read(bands,
                                 window=window,
                                 boundless=True,
                                 fill_value=0)

        bands = ImageryVisualizer.norm_band(bands,
                                            band_min=band_min,
                                            band_max=band_max,
                                            gamma=gamma)

        # need to rearrange to (h, w, channel/bands)
        bands = np.transpose(bands, axes=[1, 2, 0])

        bands = bands.squeeze(
        )  # PIL accepts (h, w, 3) or (h, w), not (h, w, 1)

        if return_array:
            return bands

        # skimage.img_as_ubyte: negative input values will be clipped. Positive values are scaled between 0 and 255
        # fine to use here as we already got rid of negative values by normalizing above
        bands = img_as_ubyte(bands)

        im = Image.fromarray(bands)
        if size:
            im = im.resize(size)
        return im
Exemple #9
0
def __make_rastertiles_Z__(src_dataset: rio.DatasetReader, world_size: float,
                           tile_size: int, zoom: int) -> list():

    # get bands
    src_bands = src_dataset.read()

    # structure for store tiles
    tiles = []

    # get bounds
    src_bbox = src_dataset.bounds
    src_bbox = [src_bbox.left, src_bbox.top, src_bbox.right, src_bbox.bottom]

    # get pixel size
    pixel_size = __pixel_size__(world_size, tile_size, zoom)

    # get all quadrant
    quadrants = __make_quadrants__(src_bbox, zoom, world_size, 1)

    for xmin, ymin, xmax, ymax in quadrants:

        # get bbox of quadrant
        Xmin, Ymin, Xmax, Ymax = list(
            __tile_world_bbox__(xmin, ymin, zoom, world_size, tile_size))

        # get pixel size
        pixel_size = __pixel_size__(world_size, tile_size, zoom)

        # make dst shape (3, tsize, tsize), 3 is fix because it's an image RGB
        dst_shape = (3, tile_size, tile_size)

        # make transform with orig (Xmin, Ymin) and scale (psize, -psize)
        dst_transform = A.translation(Xmin, Ymin) * A.scale(
            pixel_size, -pixel_size)

        dtype = src_dataset.dtypes[0]

        if dtype == rio.uint8:
            datatype = 1
        elif dtype == rio.uint16:
            datatype = 2
        elif dtype == rio.int16:
            datatype = 3
        elif dtype == rio.uint32:
            datatype = 4
        elif dtype == rio.int32:
            datatype = 5
        elif dtype == rio.float32:
            datatype = 6
        elif dtype == rio.float64:
            datatype = 7
        else:
            assert False

        # init dst bands
        dst_bands = np.zeros(dst_shape, dtype=dtype)

        count = dst_bands.shape[0]
        nodata = 0 if src_dataset.nodata is None else src_dataset.nodata

        # make reprojection for each bands
        for i in range(count):

            try:

                reproject(source=src_bands[i],
                          destination=dst_bands[i],
                          src_transform=src_dataset.transform,
                          src_crs=src_dataset.crs,
                          src_nodata=nodata,
                          dst_transform=dst_transform,
                          dst_crs=src_dataset.crs)

            except IndexError:
                continue

        gdal_bands = [{
            'data': dst_bands[x],
            'nodata_value': nodata
        } for x in range(count)]

        gdal_raster = GDALRaster({
            'srid': WEB_MERCATOR_SRID,
            'width': tile_size,
            'height': tile_size,
            'datatype': datatype,
            'nr_of_bands': count,
            'origin': [Xmin, Ymin],
            'scale': [pixel_size, -pixel_size],
            'bands': gdal_bands
        })

        tiles.append((zoom, xmin, ymin, gdal_raster))

    del src_bands

    # return structure
    return tiles
Exemple #10
0
def __make_imagetiles_Z__(src_dataset: rio.DatasetReader, world_size: float,
                          tile_size: int, zoom: int) -> list():

    # structure for store tiles
    tiles = []

    # get bounding box
    src_bbox = src_dataset.bounds
    src_bbox = [src_bbox.left, src_bbox.top, src_bbox.right, src_bbox.bottom]

    # get pixel size
    pixel_size = __pixel_size__(world_size, tile_size, zoom)

    # get all quadrant
    quadrants = __make_quadrants__(src_bbox, zoom, world_size, 1)

    for xmin, ymin, xmax, ymax in quadrants:

        # get bbox of quadrant
        Xmin, Ymin, Xmax, Ymax = list(
            __tile_world_bbox__(xmin, ymin, zoom, world_size, tile_size))

        # get pixel size
        pixel_size = __pixel_size__(world_size, tile_size, zoom)

        # make dst shape (3, tsize, tsize), 3 is fix because it's an image RGB
        dst_shape = (3, tile_size, tile_size)

        # make transform with orig (Xmin, Ymin) and scale (psize, -psize)
        dst_transform = A.translation(Xmin, Ymin) * A.scale(
            pixel_size, -pixel_size)

        # init dst bands
        dst_bands = np.zeros(dst_shape, dtype=np.uint8)

        # make reprojection for each bands
        for i in range(3):

            reproject(source=src_dataset.read(i + 1),
                      destination=dst_bands[i],
                      src_transform=src_dataset.transform,
                      src_crs=src_dataset.crs,
                      dst_transform=dst_transform,
                      dst_crs=src_dataset.crs)

        # switch channel fst to channel last
        dst_bands = np.rollaxis(dst_bands, 0, 3)

        # make alpha band for no data
        dst_sum = np.sum(dst_bands, axis=2)
        alpha = np.zeros((tile_size, tile_size, 3))
        alpha[dst_sum > 0] = np.array([255, 255, 255])

        # convert alpha as pilimage
        pil_alpha = Image.fromarray(alpha.astype(dtype=np.uint8)).convert('L')

        # convert dst_bands as pilimage & put alpha
        pil_tile = Image.fromarray(dst_bands)
        pil_tile.putalpha(pil_alpha)

        # write in a buffer as bytes
        buffer = BytesIO()
        pil_tile.save(fp=buffer, format="PNG")

        # push all in ret structure
        tiles.append((zoom, xmin, ymin, buffer))

    # return structure
    return tiles
Exemple #11
0
 def _read_chip(self, ds: rio.DatasetReader, kwargs) -> np.ndarray:
     """Reads from the given dataset with the parameters in kwargs. Created
     mostly to utilize retry functionality"""
     return ds.read(**kwargs)
    def _tile_segments(self, dataset_directory: str,
                       ds: rasterio.DatasetReader,
                       bbox_filtering_function) -> List[List[Dict[str, any]]]:
        """

        :param dataset_directory:
        :param ds:
        :return: annotations: List[List[Dict]]
        """
        # Use rasterio to compute geometric transformation parameters
        ulx, xres, _, uly, _, yres = ds.get_transform()

        # Size of Ortho in geometric scale
        img_h, img_w = ds.height, ds.width
        orthox = xres * img_w
        orthoy = yres * img_h

        # Locate ShapeFile in datasets/dataset/Segments/ directory
        shapefile_name = glob(os.path.join(dataset_directory, 'Segments',
                                           '*'))[0][:-4]

        print(f'Importing Shapefile {shapefile_name}')
        shpf = shapefile.Reader(shapefile_name)

        shape_recs = shpf.shapeRecords()
        bbox = shpf.bbox

        minx = bbox[0]
        maxy = bbox[3]

        # Offset of Shapefile from top left corner of Ortho [0, width/height] in geometric scale
        offsetx = minx - ulx
        offsety = maxy - uly

        # Scale is ratio of image pixel width/height to geometric width/height in raster
        scalex = img_w / orthox
        scaley = img_h / orthoy
        rescale_x = lambda x: (x - minx + offsetx) * scalex
        rescale_y = lambda y: (y - maxy + offsety) * scaley

        # Compute the number of image tiles and initialize annotation
        x_tiles = ceil(img_w / self.dx)
        num_tiles = x_tiles * ceil(img_h / self.dy)
        annotations = [[] for _ in range(num_tiles)]
        bad_segments = 0
        print(f'Tiling Shapes for {os.path.basename(dataset_directory)}')
        for shape_rec in tqdm(shape_recs):
            shp = shape_rec.shape
            if shp.shapeType == 5:  # 5 - polygon

                class_name = shape_rec.record.segClass
                if class_name[-1] == '2':
                    class_name = class_name[:-1]
                # Update class dict
                if class_name not in self.classes:
                    self.classes[class_name] = len(self.classes)

                # transform polygon and bounding into image coordinate system
                rescaled_poly = [[rescale_x(x), rescale_y(y)]
                                 for x, y in fix_polygon_tail(shp.points)]
                if len(rescaled_poly) < 3:
                    bad_segments += 1
                    continue

                shape_bbox = [
                    rescale_x(shp.bbox[0]),
                    rescale_y(shp.bbox[3]),
                    rescale_x(shp.bbox[2]),
                    rescale_y(shp.bbox[1])
                ]

                if not bbox_filtering_function(shape_bbox):
                    bad_segments += 1
                    continue

                x_min, y_min, x_max, y_max = shape_bbox
                x_pos, y_pos = (x_min // self.dx), (y_min // self.dy)

                # Compute tiles that shape belongs to
                for y_shift in (0, self.dy):
                    for x_shift in (0, self.dx):
                        x, y = self.dx * x_pos - x_shift, self.dy * y_pos - y_shift
                        x_c = x + self.tile_width if x + self.tile_width < img_w else img_w
                        y_c = y + self.tile_height if y + self.tile_height < img_h else img_h

                        if box_in_box(shape_bbox, [x, y, x_c, y_c]):
                            x_pos_c, y_pos_c = x_pos - if_non_zero(
                                x_shift), y_pos - if_non_zero(y_shift)
                            annotations[int(
                                x_tiles * y_pos_c + x_pos_c)].append(
                                    create_annotation(
                                        poly=rescaled_poly,
                                        bbox=shape_bbox,
                                        rescale_corner=(x, y),
                                        is_crowd=0,
                                        category_id=self.classes[class_name]))
        print(f'{bad_segments} of {len(shape_recs)} segments filtered')
        return annotations
Exemple #13
0
 def img_coords(self, img: rasterio.DatasetReader) -> Dict[str, int]:
     (top, left), (bottom, right) = img.index(self.left,
                                              self.top), img.index(
                                                  self.right, self.bottom)
     return {'left': left, 'bottom': bottom, 'right': right, 'top': top}