def test_create_corner(self): scale = 0.00025 shape = (3, 2) origin = (150.0, -34.0) corner = (shape[1] * scale + origin[0], origin[1] - shape[0] * scale) ggb = GriddedGeoBox(shape, origin) self.assertEqual(corner, ggb.corner)
def test_utm_to_lonlat(self): """ Test that coordinates in one CRS are correctly transformed from utm into lonlat This came about with GDAL3 and Proj6, where the native axis mapping of a CRS is respected, meaning that an x,y input and output is dependent on the CRS axis. We'll instead enforce the x,y axis mapping strategy. """ shape = (3, 2) origin = (669700, 6111700) ggb = GriddedGeoBox(shape, origin, crs="EPSG:32755") lon = 148.862561 lat = -35.123064 easting = 669717.105361586 northing = 6111722.038508673 to_crs = osr.SpatialReference() to_crs.ImportFromEPSG(4326) lon, lat = ggb.transform_coordinates((easting, northing), to_crs) self.assertAlmostEqual(lon, 148.862561) self.assertAlmostEqual(lat, -35.123064)
def test_y_size(self): scale = 0.00025 shape = (3, 2) origin = (150.0, -34.0) corner = (shape[1] * scale + origin[0], origin[1] - shape[0] * scale) ggb = GriddedGeoBox(shape, origin) self.assertEqual(shape[0], ggb.y_size())
def test_get_shape_xy(self): scale = 0.00025 shape = (3, 2) shape_xy = (2, 3) origin = (150.0, -34.0) corner = (shape[1] * scale + origin[0], origin[1] - shape[0] * scale) ggb = GriddedGeoBox(shape, origin) self.assertEqual(shape_xy, ggb.get_shape_xy())
def convert_format(self, dataset_name, group, attrs=None, compression=H5CompressionFilter.LZF, filter_opts=None): """ Convert the HDF file to a HDF5 dataset. """ if attrs is None: attrs = {} # Get the UL corner of the UL pixel co-ordinate ul_lon = self.ul[0] ul_lat = self.ul[1] # pixel size x & y pixsz_x = self.delta_lon pixsz_y = self.delta_lat # Setup the projection; assuming Geographics WGS84 # (Tests have shown that this appears to be the case) # (unfortunately it is not expicitly defined in the HDF file) sr = osr.SpatialReference() sr.SetWellKnownGeogCS("WGS84") prj = sr.ExportToWkt() # Setup the geobox dims = self.data[0].shape res = (abs(pixsz_x), abs(pixsz_y)) geobox = GriddedGeoBox(shape=dims, origin=(ul_lon, ul_lat), pixelsize=res, crs=prj) # Write the dataset attrs['description'] = 'Converted BRDF data from H4 to H5.' attrs['crs_wkt'] = prj attrs['geotransform'] = geobox.transform.to_gdal() write_h5_image(self.data[0], dataset_name, group, compression, attrs, filter_opts)
def __init__(self, band_data): self.no_data = 0 self.acquisition_datetime = dateutil.parser.parse( '2018-05-20 02:16:38') self.brdf_datasets = band_data['brdf_datasets'] bbox = geopandas.GeoDataFrame({ 'geometry': [box(115.155105, -30.585143333333335, 115.15636, -30.58394)] }) bbox.crs = {'init': 'EPSG:4326'} albers = bbox.to_crs(epsg=3577) buff = albers.buffer(1000) lonlat = buff.to_crs(epsg=4326) minx, miny, maxx, maxy = lonlat.total_bounds self.geobox = GriddedGeoBox(shape=(1, 1), origin=(minx, miny), pixelsize=(maxx - minx, maxy - miny), crs='EPSG:4326')
def test_lonlat_to_utm(self): """ Test that coordinates in one CRS are correctly transformed from lonlat into utm This came about with GDAL3 and Proj6, where the native axis mapping of a CRS is respected, meaning that an x,y input and output is dependent on the CRS axis. We'll instead enforce the x,y axis mapping strategy. """ shape = (3, 2) origin = (150.0, -34.0) ggb = GriddedGeoBox(shape, origin) lon = 148.862561 lat = -35.123064 to_crs = osr.SpatialReference() to_crs.ImportFromEPSG(32755) easting, northing = ggb.transform_coordinates((lon, lat), to_crs) self.assertAlmostEqual(easting, 669717.105361586) self.assertAlmostEqual(northing, 6111722.038508673)
def read_subset(fname, ul_xy, ur_xy, lr_xy, ll_xy, bands=1): """ Return a 2D or 3D NumPy array subsetted to the given bounding extents. :param fname: A string containing the full file pathname to an image on disk. :param ul_xy: A tuple containing the Upper Left (x,y) co-ordinate pair in real world (map) co-ordinates. Co-ordinate pairs can be (longitude, latitude) or (eastings, northings), but they must be of the same reference as the image of interest. :param ur_xy: A tuple containing the Upper Right (x,y) co-ordinate pair in real world (map) co-ordinates. Co-ordinate pairs can be (longitude, latitude) or (eastings, northings), but they must be of the same reference as the image of interest. :param lr_xy: A tuple containing the Lower Right (x,y) co-ordinate pair in real world (map) co-ordinates. Co-ordinate pairs can be (longitude, latitude) or (eastings, northings), but they must be of the same reference as the image of interest. :param ll_xy: A tuple containing the Lower Left (x,y) co-ordinate pair in real world (map) co-ordinates. Co-ordinate pairs can be (longitude, latitude) or (eastings, northings), but they must be of the same reference as the image of interest. :param bands: Can be an integer of list of integers representing the band(s) to be read from disk. If bands is a list, then the returned subset will be 3D, otherwise the subset will be strictly 2D. :return: A tuple of 3 elements: * 1. 2D or 3D NumPy array containing the image subset. * 2. A list of length 6 containing the GDAL geotransform. * 3. A WKT formatted string representing the co-ordinate reference system (projection). :additional notes: The ending array co-ordinates are increased by +1, i.e. xend = 270 + 1 to account for Python's [inclusive, exclusive) index notation. """ if isinstance(fname, h5py.Dataset): geobox = GriddedGeoBox.from_dataset(fname) prj = fname.attrs['crs_wkt'] else: # Open the file with rasterio.open(fname) as src: # Get the inverse transform of the affine co-ordinate reference geobox = GriddedGeoBox.from_dataset(src) prj = src.crs.wkt # rasterio returns a unicode inv = ~geobox.transform rows, cols = geobox.shape # Convert each map co-ordinate to image/array co-ordinates img_ul_x, img_ul_y = [int(v) for v in inv * ul_xy] img_ur_x, img_ur_y = [int(v) for v in inv * ur_xy] img_lr_x, img_lr_y = [int(v) for v in inv * lr_xy] img_ll_x, img_ll_y = [int(v) for v in inv * ll_xy] # Calculate the min and max array extents # The ending array extents have +1 to account for Python's # [inclusive, exclusive) index notation. xstart = min(img_ul_x, img_ll_x) ystart = min(img_ul_y, img_ur_y) xend = max(img_ur_x, img_lr_x) + 1 yend = max(img_ll_y, img_lr_y) + 1 # Check for out of bounds if (((xstart < 0) or (ystart < 0)) or ((xend -1 > cols) or (yend -1 > rows))): msg = ("Error! Attempt to read a subset that is outside of the" "image domain. Index: ({ys}, {ye}), ({xs}, {xe}))") msg = msg.format(ys=ystart, ye=yend, xs=xstart, xe=xend) raise IndexError(msg) if isinstance(fname, h5py.Dataset): subs = fname[ystart:yend, xstart:xend] else: with rasterio.open(fname) as src: subs = src.read(bands, window=((ystart, yend), (xstart, xend))) # Get the new UL co-ordinates of the array ul_x, ul_y = geobox.transform * (xstart, ystart) geobox_subs = GriddedGeoBox(shape=subs.shape, origin=(ul_x, ul_y), pixelsize=geobox.pixelsize, crs=prj) return (subs, geobox_subs)
def get_dsm( acquisition, pathname, buffer_distance=8000, out_group=None, compression=H5CompressionFilter.LZF, filter_opts=None, ): """ Given an acquisition and a national Digitial Surface Model, extract a subset from the DSM based on the acquisition extents plus an x & y margins. The subset is then smoothed with a 3x3 gaussian filter. A square margins is applied to the extents. :param acquisition: An instance of an acquisition object. :param pathname: A string pathname of the DSM with a ':' to seperate the filename from the import HDF5 dataset name. :param buffer_distance: A number representing the desired distance (in the same units as the acquisition) in which to calculate the extra number of pixels required to buffer an image. Default is 8000. :param out_group: If set to None (default) then the results will be returned as an in-memory hdf5 file, i.e. the `core` driver. Otherwise, a writeable HDF5 `Group` object. The dataset name will be as follows: * DatasetName.DSM_SMOOTHED :param compression: The compression filter to use. Default is H5CompressionFilter.LZF :filter_opts: A dict of key value pairs available to the given configuration instance of H5CompressionFilter. For example H5CompressionFilter.LZF has the keywords *chunks* and *shuffle* available. Default is None, which will use the default settings for the chosen H5CompressionFilter instance. :return: An opened `h5py.File` object, that is either in-memory using the `core` driver, or on disk. """ # Use the 1st acquisition to setup the geobox geobox = acquisition.gridded_geo_box() shape = geobox.get_shape_yx() # buffered image extents/margins margins = pixel_buffer(acquisition, buffer_distance) # Get the dimensions and geobox of the new image dem_cols = shape[1] + margins.left + margins.right dem_rows = shape[0] + margins.top + margins.bottom dem_shape = (dem_rows, dem_cols) dem_origin = geobox.convert_coordinates( (0 - margins.left, 0 - margins.top)) dem_geobox = GriddedGeoBox( dem_shape, origin=dem_origin, pixelsize=geobox.pixelsize, crs=geobox.crs.ExportToWkt(), ) # split the DSM filename, dataset name, and load fname, dname = pathname.split(":") with h5py.File(fname, "r") as dsm_fid: dsm_ds = dsm_fid[dname] dsm_geobox = GriddedGeoBox.from_dataset(dsm_ds) # calculate full border extents into CRS of DSM extents = dem_geobox.project_extents(dsm_geobox.crs) ul_xy = (extents[0], extents[3]) ur_xy = (extents[2], extents[3]) lr_xy = (extents[2], extents[1]) ll_xy = (extents[0], extents[1]) # load the subset and corresponding geobox subs, subs_geobox = read_subset(dsm_ds, ul_xy, ur_xy, lr_xy, ll_xy, edge_buffer=1) # ancillary metadata tracking metadata = current_h5_metadata(dsm_fid, dataset_path=dname) # Retrive the DSM data dsm_data = reproject_array_to_array(subs, subs_geobox, dem_geobox, resampling=Resampling.bilinear) # free memory subs = None # Output the reprojected result # Initialise the output files if out_group is None: fid = h5py.File("dsm-subset.h5", "w", driver="core", backing_store=False) else: fid = out_group if filter_opts is None: filter_opts = {} else: filter_opts = filter_opts.copy() if acquisition.tile_size[0] == 1: filter_opts["chunks"] = (1, dem_cols) else: # TODO: rework the tiling regime for larger dsm # for non single row based tiles, we won't have ideal # matching reads for tiled processing between the acquisition # and the DEM filter_opts["chunks"] = acquisition.tile_size kwargs = compression.config(**filter_opts).dataset_compression_kwargs() group = fid.create_group(GroupName.ELEVATION_GROUP.value) param_grp = group.create_group("PARAMETERS") param_grp.attrs["left_buffer"] = margins.left param_grp.attrs["right_buffer"] = margins.right param_grp.attrs["top_buffer"] = margins.top param_grp.attrs["bottom_buffer"] = margins.bottom # dataset attributes attrs = { "crs_wkt": geobox.crs.ExportToWkt(), "geotransform": dem_geobox.transform.to_gdal(), } # Smooth the DSM dsm_data = filter_dsm(dsm_data) dname = DatasetName.DSM_SMOOTHED.value out_sm_dset = group.create_dataset(dname, data=dsm_data, **kwargs) desc = "A subset of a Digital Surface Model smoothed with a gaussian " "kernel." attrs["description"] = desc attrs["id"] = numpy.array([metadata["id"]], VLEN_STRING) attach_image_attributes(out_sm_dset, attrs) if out_group is None: return fid
def read_subset(fname, ul_xy, ur_xy, lr_xy, ll_xy, edge_buffer=0, bands=1): """ Return a 2D or 3D NumPy array subsetted to the given bounding extents. The function will allow a user to ask for a region outside of the requested domain. Those elements that fall outside of the requested domain will be populated with the datasets' fillvalue or 0 if the fillvalue is None. :param fname: A string containing the full file pathname to an image on disk. OR an HDF5 Dataset (h5py.Dataset). :param ul_xy: A tuple containing the Upper Left (x,y) co-ordinate pair in real world (map) co-ordinates. Co-ordinate pairs can be (longitude, latitude) or (eastings, northings), but they must be of the same reference as the image of interest. :param ur_xy: A tuple containing the Upper Right (x,y) co-ordinate pair in real world (map) co-ordinates. Co-ordinate pairs can be (longitude, latitude) or (eastings, northings), but they must be of the same reference as the image of interest. :param lr_xy: A tuple containing the Lower Right (x,y) co-ordinate pair in real world (map) co-ordinates. Co-ordinate pairs can be (longitude, latitude) or (eastings, northings), but they must be of the same reference as the image of interest. :param ll_xy: A tuple containing the Lower Left (x,y) co-ordinate pair in real world (map) co-ordinates. Co-ordinate pairs can be (longitude, latitude) or (eastings, northings), but they must be of the same reference as the image of interest. :param edge_buffer: An integer indicating the additional number of pixels to read along each edge of the subset. Useful for when additional data might be required, such as for reprojection. Default is 0 pixels on each edge. :param bands: Can be an integer of list of integers representing the band(s) to be read from disk. If bands is a list, then the returned subset will be 3D, otherwise the subset will be strictly 2D. :return: A tuple of 2 elements: * 1. 2D or 3D NumPy array containing the requested region * 2. An instance of a GriddedGeoBox covering the requested region :additional notes: The array dimensions are determined via the supplied ROI. As such, the returned array will use a fill value for the pixels falling outside of the dataset we're reading from. """ if isinstance(fname, h5py.Dataset): geobox = GriddedGeoBox.from_dataset(fname) prj = fname.attrs['crs_wkt'] dtype = fname.dtype fillv = fname.attrs.get('fillvalue') elif isinstance(fname, rasterio.io.DatasetReader): # Get the inverse transform of the affine co-ordinate reference geobox = GriddedGeoBox.from_dataset(fname) prj = fname.crs.wkt # rasterio returns a unicode dtype = fname.dtypes[0] fillv = fname.nodata elif isinstance(fname, str): # Open the file with rasterio.open(fname) as src: # Get the inverse transform of the affine co-ordinate reference geobox = GriddedGeoBox.from_dataset(src) prj = src.crs.wkt # rasterio returns a unicode dtype = src.dtypes[0] fillv = src.nodata else: raise ValueError('Unexpected file description of type {}'.format(type(fname))) inv = ~geobox.transform rows, cols = geobox.shape # fillvalue will default to zero if None fillv = 0 if fillv is None else fillv # Convert each map co-ordinate to image/array co-ordinates img_ul_x, img_ul_y = [int(round(v)) for v in inv * ul_xy] img_ur_x, img_ur_y = [int(round(v)) for v in inv * ur_xy] img_lr_x, img_lr_y = [int(round(v)) for v in inv * lr_xy] img_ll_x, img_ll_y = [int(round(v)) for v in inv * ll_xy] # Calculate the min and max array extents including edge_buffer xstart = min(img_ul_x, img_ll_x) - edge_buffer ystart = min(img_ul_y, img_ur_y) - edge_buffer xend = max(img_ur_x, img_lr_x) + edge_buffer yend = max(img_ll_y, img_lr_y) + edge_buffer # intialise the output array dims = (yend - ystart, xend - xstart) subs = np.full(dims, fillv, dtype=dtype) # Get the new UL co-ordinates of the array ul_x, ul_y = geobox.transform * (xstart, ystart) geobox_subs = GriddedGeoBox(shape=subs.shape, origin=(ul_x, ul_y), pixelsize=geobox.pixelsize, crs=prj) # test for intersection if not geobox_subs.intersects(geobox): raise IndexError("Requested Subset Does Not Intersect With Array") # intersected region (source index xy start and end coords) source_xs = max(0, xstart) source_ys = max(0, ystart) source_xe = min(cols, xend) source_ye = min(rows, yend) # source indices/slice source_idx = np.s_[source_ys:source_ye, source_xs:source_xe] # destination origin/start index (UL) coords -> abs(min(0, ul)) dest_xs = abs(min(0, xstart)) dest_ys = abs(min(0, ystart)) # destination end (LR) -> (source_end - source_start) + dest_start dest_xe = (source_xe - source_xs) + dest_xs dest_ye = (source_ye - source_ys) + dest_ys # destination indices/slice dest_idx = np.s_[dest_ys:dest_ye, dest_xs:dest_xe] if isinstance(fname, h5py.Dataset): fname.read_direct(subs, source_idx, dest_idx) elif isinstance(fname, rasterio.io.DatasetReader): window = ((source_ys, source_ye), (source_xs, source_xe)) fname.read(bands, window=window, out=subs[dest_idx]) elif isinstance(fname, str): with rasterio.open(fname) as src: window = ((source_ys, source_ye), (source_xs, source_xe)) src.read(bands, window=window, out=subs[dest_idx]) else: raise ValueError('Unexpected file description of type {}'.format(type(fname))) return (subs, geobox_subs)
def create_test_image( dimensions=(1000, 1000), geotransform=None, projection=None, resolution=(25.0, 25.0), dtype="uint8", ): """ Creates an image with geo-location information. :param dimensions: A tuple containing the (y, x) dimensions of the 2D image to be generated. :param geotransform: A list or tuple containing the upper left co-ordinate of the image. This info can be retrieved from gdal. Otherwise create your own using the following as a guide. Must have 6 elements. geoT = (635000.0, 25.0, 0.0, 6277000.0, 0.0, 25.0) geoT[0] is top left x co-ordinate. geoT[1] is west to east pixel size. geoT[2] is image rotation (0 if image is north up). geoT[3] is to left y co-ordinate. geoT[4] is image rotation (0 if image is north up). geoT[5] is north to south pixel size. If either the geotransform or projection keywords are None, then geotransform will be set to: (635000.0, 25.0, 0.0, 6277000.0, 0.0, 25.0) and the projection will be set to EPSG:28355. :param projection: An osr compliant projection input such as WKT. If either the projection or geotransform keywords are None, then the projection will be set to EPSG:28355 and the geotransform will be set to: (635000.0, 25.0, 0.0, 6277000.0, 0.0, 25.0) :param resolution: A tuple containing the (x, y) pixel resolution/size. Default is (25.0, 25.0). :return: A tuple of two elements. The 1st element contains a random 8bit Unsigned Integer of dimensions (y, x), containing values in the range [0,256). The 2nd element contains an instance of a GriddedGeoBox. """ img = numpy.random.randint(0, 256, dimensions).astype(dtype) if (geotransform is None) or (projection is None): geotransform = (635000.0, 25.0, 0.0, 6277000.0, 0.0, 25.0) sr = osr.SpatialReference() # GDA94/ MGA Zone 55 sr.SetFromUserInput(CRS) projection = sr.ExportToWkt() resolution = (geotransform[1], geotransform[5]) UL = (geotransform[0], geotransform[3]) geobox = GriddedGeoBox(shape=dimensions, origin=UL, pixelsize=resolution, crs=projection) return (img, geobox)
flindersCorner = (150.931697, -34.457915) flindersGGB = GriddedGeoBox.from_corners(flindersOrigin, flindersCorner) mask = get_land_sea_mask(flindersGGB) print("geobox_shape=%s" % str(flindersGGB.shape)) print("mask_shape=%s" % str(mask.shape)) print(mask) # same test for AGDC cell around Darwin area scale = 0.00025 shape = (4000, 4000) origin = (130.0, -12.0) corner = (shape[1] * scale + origin[0], origin[1] - shape[0] * scale) ggb = GriddedGeoBox(shape, origin, pixelsize=(scale, scale)) print(ggb) # now get UTM equilavent geo_box = ggb.copy(crs="EPSG:32752") print(geo_box) # and get the mask mask = get_land_sea_mask(geo_box) total_pixels = geo_box.shape[1] * geo_box.shape[0] land_pixels = sum(sum(mask.astype('uint32'))) sea_pixels = total_pixels - land_pixels sea_pct = 100.0 * sea_pixels / total_pixels
def get_dsm(acquisition, national_dsm, buffer_distance=8000, out_group=None, compression=H5CompressionFilter.LZF, filter_opts=None): """ Given an acquisition and a national Digitial Surface Model, extract a subset from the DSM based on the acquisition extents plus an x & y margins. The subset is then smoothed with a 3x3 gaussian filter. A square margins is applied to the extents. :param acquisition: An instance of an acquisition object. :param national_dsm: A string containing the full filepath name to an image on disk containing national digital surface model. :param buffer_distance: A number representing the desired distance (in the same units as the acquisition) in which to calculate the extra number of pixels required to buffer an image. Default is 8000. :param out_group: If set to None (default) then the results will be returned as an in-memory hdf5 file, i.e. the `core` driver. Otherwise, a writeable HDF5 `Group` object. The dataset name will be as follows: * DatasetName.DSM_SMOOTHED :param compression: The compression filter to use. Default is H5CompressionFilter.LZF :filter_opts: A dict of key value pairs available to the given configuration instance of H5CompressionFilter. For example H5CompressionFilter.LZF has the keywords *chunks* and *shuffle* available. Default is None, which will use the default settings for the chosen H5CompressionFilter instance. :return: An opened `h5py.File` object, that is either in-memory using the `core` driver, or on disk. """ # Use the 1st acquisition to setup the geobox geobox = acquisition.gridded_geo_box() shape = geobox.get_shape_yx() # buffered image extents/margins margins = pixel_buffer(acquisition, buffer_distance) # Get the dimensions and geobox of the new image dem_cols = shape[1] + margins.left + margins.right dem_rows = shape[0] + margins.top + margins.bottom dem_shape = (dem_rows, dem_cols) dem_origin = geobox.convert_coordinates((0 - margins.left, 0 - margins.top)) dem_geobox = GriddedGeoBox(dem_shape, origin=dem_origin, pixelsize=geobox.pixelsize, crs=geobox.crs.ExportToWkt()) # Retrive the DSM data dsm_data = reproject_file_to_array(national_dsm, dst_geobox=dem_geobox, resampling=Resampling.bilinear) # Output the reprojected result # Initialise the output files if out_group is None: fid = h5py.File('dsm-subset.h5', driver='core', backing_store=False) else: fid = out_group if filter_opts is None: filter_opts = {} else: filter_opts = filter_opts.copy() if acquisition.tile_size[0] == 1: filter_opts['chunks'] = (1, dem_cols) else: # TODO: rework the tiling regime for larger dsm # for non single row based tiles, we won't have ideal # matching reads for tiled processing between the acquisition # and the DEM filter_opts['chunks'] = acquisition.tile_size kwargs = compression.config(**filter_opts).dataset_compression_kwargs() group = fid.create_group(GroupName.ELEVATION_GROUP.value) param_grp = group.create_group('PARAMETERS') param_grp.attrs['left_buffer'] = margins.left param_grp.attrs['right_buffer'] = margins.right param_grp.attrs['top_buffer'] = margins.top param_grp.attrs['bottom_buffer'] = margins.bottom # dataset attributes attrs = {'crs_wkt': geobox.crs.ExportToWkt(), 'geotransform': dem_geobox.transform.to_gdal()} # Smooth the DSM dsm_data = filter_dsm(dsm_data) dname = DatasetName.DSM_SMOOTHED.value out_sm_dset = group.create_dataset(dname, data=dsm_data, **kwargs) desc = ("A subset of a Digital Surface Model smoothed with a gaussian " "kernel.") attrs['description'] = desc attach_image_attributes(out_sm_dset, attrs) if out_group is None: return fid
def test_create_origin(self): shape = (3, 2) origin = (150.0, -34.0) ggb = GriddedGeoBox(shape, origin) self.assertEqual(origin, ggb.origin)
def test_y_size(self): shape = (3, 2) origin = (150.0, -34.0) ggb = GriddedGeoBox(shape, origin) self.assertEqual(shape[0], ggb.y_size())
def test_get_shape_yx(self): shape = (3, 2) origin = (150.0, -34.0) ggb = GriddedGeoBox(shape, origin) self.assertEqual(shape, ggb.get_shape_yx())