def make_storage_unit(su, is_diskless=False, include_lineage=False): """convert search result into StorageUnit object :param su: database index storage unit :param is_diskless: Use a cached object for the source of data, rather than the file :param include_lineage: Include an 'extra_metadata' variable containing detailed lineage information. Note: This can cause the query to be slow for large datasets, as it is not lazy-loaded. """ crs = { dim: su.descriptor['coordinates'][dim].get('units', None) for dim in su.storage_type.dimensions } for dim in crs.keys(): if dim in su.storage_type.spatial_dimensions: crs[dim] = su.storage_type.crs coordinates = su.coordinates variables = { varname: Variable(dtype=numpy.dtype(attributes['dtype']), nodata=attributes.get('nodata', None), dimensions=su.storage_type.dimensions, units=attributes.get('units', None)) for varname, attributes in su.storage_type.measurements.items() } if 'extra_metadata' not in variables.keys() and include_lineage: variables['extra_metadata'] = Variable(numpy.dtype('S30000'), None, ('time', ), None) attributes = { 'storage_type': su.storage_type, 'dataset_ids': su.dataset_ids } if is_diskless: return make_in_memory_storage_unit(su, coordinates=coordinates, variables=variables, attributes=attributes, crs=crs) if su.storage_type.driver == 'NetCDF CF': return NetCDF4StorageUnit(su.local_path, coordinates=coordinates, variables=variables, attributes=attributes, crs=crs) if su.storage_type.driver == 'GeoTiff': result = GeoTifStorageUnit(su.local_path, coordinates=coordinates, variables=variables, attributes=attributes) time = datetime.datetime.strptime(su.descriptor['extents']['time_min'], '%Y-%m-%dT%H:%M:%S.%f') return StorageUnitDimensionProxy(result, time_coordinate_value(time)) raise RuntimeError('unsupported storage unit access driver %s' % su.storage_type.driver)
def test_chunksizes(tmpnetcdf_filename): nco = create_netcdf(tmpnetcdf_filename) coord1 = create_coordinate(nco, 'greg', numpy.array([1.0, 2.0, 3.0]), 'cubic gregs') coord2 = create_coordinate(nco, 'bleh', numpy.array([1.0, 2.0, 3.0, 4.0, 5.0]), 'metric blehs') no_chunks = create_variable(nco, 'no_chunks', Variable(numpy.dtype('int16'), None, ('greg', 'bleh'), None)) min_max_chunks = create_variable(nco, 'min_max_chunks', Variable(numpy.dtype('int16'), None, ('greg', 'bleh'), None), chunksizes=[2, 50]) nco.close() with netCDF4.Dataset(tmpnetcdf_filename) as nco: assert nco['no_chunks'].chunking() == 'contiguous' assert nco['min_max_chunks'].chunking() == [2, 5]
def make_sample_netcdf(tmpdir): """Make a test Geospatial NetCDF file, 4000x4000 int16 random data, in a variable named `sample`. Return the GDAL access string.""" sample_nc = str(tmpdir.mkdir('netcdfs').join('sample.nc')) geobox = GeoBox(4000, 4000, affine=Affine(25.0, 0.0, 1200000, 0.0, -25.0, -4200000), crs=CRS('EPSG:3577')) sample_data = np.random.randint(10000, size=(4000, 4000), dtype=np.int16) variables = { 'sample': Variable(sample_data.dtype, nodata=-999, dims=geobox.dimensions, units=1) } nco = create_netcdf_storage_unit(sample_nc, geobox.crs, geobox.coordinates, variables=variables, variable_params={}) nco['sample'][:] = sample_data nco.close() return "NetCDF:%s:sample" % sample_nc, geobox, sample_data
def from_file(cls, file_path): coordinates = {} variables = {} grid_mappings = {} standard_names = {} with _GLOBAL_LOCK, contextlib.closing(_open_dataset(file_path)) as ncds: attributes = {k: getattr(ncds, k) for k in ncds.ncattrs()} for name, var in ncds.variables.items(): dims = var.dimensions units = getattr(var, 'units', None) if hasattr(var, 'grid_mapping_name') and hasattr(var, 'spatial_ref'): grid_mappings[getattr(var, 'grid_mapping_name', None)] = getattr(var, 'spatial_ref', None) elif len(dims) == 1 and name == dims[0]: coordinates[name] = Coordinate(dtype=numpy.dtype(var.dtype), begin=var[0].item(), end=var[var.size-1].item(), length=var.shape[0], units=units) standard_name = getattr(var, 'standard_name', None) if standard_name: standard_names[standard_name] = name else: dims, dtype = _get_dims_and_dtype(var) ndv = (getattr(var, '_FillValue', None) or getattr(var, 'missing_value', None) or getattr(var, 'fill_value', None)) ndv = ndv.item() if ndv else None variables[name] = Variable(dtype, ndv, dims, units) crs = _make_crs(grid_mappings, standard_names) return cls(file_path, variables=variables, coordinates=coordinates, attributes=attributes, crs=crs)
def make_fake_netcdf_dataset(nc_name, yaml_doc): from datacube.model import Variable from datacube.storage.netcdf_writer import create_variable, netcdfy_data from netCDF4 import Dataset import numpy as np content = yaml_doc.read_text() npdata = np.array([content], dtype=bytes) with Dataset(nc_name, 'w') as nco: var = Variable(npdata.dtype, None, ('time', ), None) nco.createDimension('time', size=1) create_variable(nco, 'dataset', var) nco['dataset'][:] = netcdfy_data(npdata)
def test_chunksizes(tmpnetcdf_filename): nco = create_netcdf(tmpnetcdf_filename) x = numpy.arange(3, dtype='float32') y = numpy.arange(5, dtype='float32') coord1 = create_coordinate(nco, 'x', x, 'm') coord2 = create_coordinate(nco, 'y', y, 'm') assert coord1 is not None and coord2 is not None no_chunks = create_variable( nco, 'no_chunks', Variable(numpy.dtype('int16'), None, ('x', 'y'), None)) min_max_chunks = create_variable(nco, 'min_max_chunks', Variable(numpy.dtype('int16'), None, ('x', 'y'), None), chunksizes=(2, 50)) assert no_chunks is not None assert min_max_chunks is not None strings = numpy.array(["AAa", 'bbb', 'CcC'], dtype='S') strings = xr.DataArray(strings, dims=['x'], coords={'x': x}) create_variable(nco, 'strings_unchunked', strings) create_variable(nco, 'strings_chunked', strings, chunksizes=(1, )) nco.close() with netCDF4.Dataset(tmpnetcdf_filename) as nco: assert nco['no_chunks'].chunking() == 'contiguous' assert nco['min_max_chunks'].chunking() == [2, 5] assert nco['strings_unchunked'].chunking() == 'contiguous' assert nco['strings_chunked'].chunking() == [1, 3]
def test_create_string_variable(tmpnetcdf_filename): nco = create_netcdf(tmpnetcdf_filename) coord = create_coordinate(nco, 'greg', numpy.array([1.0, 3.0, 9.0]), 'cubic gregs') dtype = numpy.dtype('S100') data = numpy.array(["test-str1", "test-str2", "test-str3"], dtype=dtype) var = create_variable(nco, 'str_var', Variable(dtype, None, ('greg', ), None)) var[:] = netcdfy_data(data) nco.close() with netCDF4.Dataset(tmpnetcdf_filename) as nco: assert 'str_var' in nco.variables assert netCDF4.chartostring(nco['str_var'][0]) == data[0]
def saveNC(output,filename, history): nco=netcdf_writer.create_netcdf(filename) nco.history = (history.decode('utf-8').encode('ascii','replace')) coords=output.coords cnames=() for x in coords: netcdf_writer.create_coordinate(nco, x, coords[x].values, coords[x].units) cnames=cnames+(x,) netcdf_writer.create_grid_mapping_variable(nco, output.crs) for band in output.data_vars: output.data_vars[band].values[np.isnan(output.data_vars[band].values)]=nodata var= netcdf_writer.create_variable(nco, band, Variable(output.data_vars[band].dtype, nodata, cnames, None) ,set_crs=True) var[:] = netcdf_writer.netcdfy_data(output.data_vars[band].values) nco.close()
def nco_from_sources(sources, geobox, measurements, variable_params, filename): coordinates = { name: Coordinate(coord.values, coord.units) for name, coord in sources.coords.items() } coordinates.update(geobox.coordinates) variables = { variable['name']: Variable(dtype=numpy.dtype(variable['dtype']), nodata=variable['nodata'], dims=sources.dims + geobox.dimensions, units=variable['units']) for variable in measurements } return create_netcdf_storage_unit(filename, geobox.crs, coordinates, variables, variable_params)
def test_create_string_variable(tmpdir, s1, s2, s3): tmpnetcdf_filename = get_tmpnetcdf_filename(tmpdir) str_var = 'str_var' nco = create_netcdf(tmpnetcdf_filename) coord = create_coordinate(nco, 'greg', numpy.array([1.0, 3.0, 9.0]), 'cubic gregs') dtype = numpy.dtype('S100') data = numpy.array([s1, s2, s3], dtype=dtype) var = create_variable(nco, str_var, Variable(dtype, None, ('greg',), None)) var[:] = netcdfy_data(data) nco.close() with netCDF4.Dataset(tmpnetcdf_filename) as nco: assert str_var in nco.variables for returned, expected in zip(read_strings_from_netcdf(tmpnetcdf_filename, variable=str_var), (s1, s2, s3)): assert returned == expected
def write_dataset_to_netcdf(access_unit, global_attributes, variable_params, filename): if filename.exists(): raise RuntimeError('Storage Unit already exists: %s' % filename) try: filename.parent.mkdir(parents=True) except OSError: pass # _LOG.info("Writing storage unit: %s", filename) nco = netcdf_writer.create_netcdf(str(filename)) for name, coord in access_unit.coords.items(): netcdf_writer.create_coordinate(nco, name, coord.values, coord.units) netcdf_writer.create_grid_mapping_variable(nco, access_unit.crs) for name, variable in access_unit.data_vars.items(): # Create variable var_params = variable_params.get(name, {}) data_var = netcdf_writer.create_variable( nco, name, Variable(variable.dtype, getattr(variable, 'nodata', None), variable.dims, getattr(variable, 'units', '1')), **var_params) # Write data data_var[:] = netcdf_writer.netcdfy_data(variable.values) # TODO: 'flags_definition', 'spectral_definition'? for key, value in variable_params.get(name, {}).get('attrs', {}).items(): setattr(data_var, key, value) # write global atrributes for key, value in global_attributes.items(): setattr(nco, key, value) nco.close()
def _nco_from_sources(self, sources, geobox, measurements, variable_params, filename): coordinates = OrderedDict( (name, geometry.Coordinate(coord.values, coord.units)) for name, coord in sources.coords.items()) coordinates.update(geobox.coordinates) variables = OrderedDict( (variable['name'], Variable(dtype=numpy.dtype(variable['dtype']), nodata=variable['nodata'], dims=sources.dims + geobox.dimensions, units=variable['units'])) for variable in measurements) return create_netcdf_storage_unit( filename, crs=geobox.crs, coordinates=coordinates, variables=variables, variable_params=variable_params, global_attributes=self.global_attributes)
def compute_and_write(self): """ Computes the wofs confidence and filtered summary bands and write to the corresponding NetCDF file. The file template and location etc are read from the configs. """ geo_box = self.grid_spec.tile_geobox(self.tile_index) # Compute metadata env = self.cfg.get_env_of_product('wofs_filtered_summary') with Datacube(app='wofs-confidence', env=env) as dc: product = dc.index.products.get_by_name('wofs_filtered_summary') extent = self.grid_spec.tile_geobox(self.tile_index).extent center_time = datetime.now() uri = self.get_filtered_uri() dts = make_dataset(product=product, sources=self.factor_sources, extent=extent, center_time=center_time, uri=uri) metadata = yaml.dump(dts.metadata_doc, Dumper=SafeDumper, encoding='utf-8') # Compute dataset coords coords = dict() coords['time'] = Coordinate( netcdf_writer.netcdfy_coord( np.array([datetime_to_seconds_since_1970(center_time)])), ['seconds since 1970-01-01 00:00:00']) for dim in geo_box.dimensions: coords[dim] = Coordinate( netcdf_writer.netcdfy_coord(geo_box.coordinates[dim].values), geo_box.coordinates[dim].units) # Compute dataset variables spatial_var = Variable(dtype=np.dtype(DEFAULT_TYPE), nodata=DEFAULT_FLOAT_NODATA, dims=('time', ) + geo_box.dimensions, units=('seconds since 1970-01-01 00:00:00', ) + geo_box.crs.units) band1 = self.cfg.cfg['wofs_filtered_summary']['confidence'] band2 = self.cfg.cfg['wofs_filtered_summary']['confidence_filtered'] vars = {band1: spatial_var, band2: spatial_var} vars_params = {band1: {}, band2: {}} global_atts = self.cfg.cfg['global_attributes'] # Get crs string crs = self.cfg.cfg['storage']['crs'] if self.cfg.cfg['storage'].get( 'crs') else DEFAULT_CRS # Create a dataset container filename = self.get_filtered_uri() logger.info('creating', file=filename.name) netcdf_unit = create_netcdf_storage_unit(filename=filename, crs=CRS(crs), coordinates=coords, variables=vars, global_attributes=global_atts, variable_params=vars_params) # Confidence layer: Fill variable data and set attributes confidence = self.compute_confidence() netcdf_unit[band1][:] = netcdf_writer.netcdfy_data(confidence) netcdf_unit[band1].units = '1' netcdf_unit[band1].valid_range = [0, 1.0] netcdf_unit[band1].coverage_content_type = 'modelResult' netcdf_unit[band1].long_name = \ 'Wofs Confidence Layer predicted by {}'.format(self.confidence_model.factors.__str__()) # Confidence filtered wofs-stats frequency layer: Fill variable data and set attributes confidence_filtered = self.compute_confidence_filtered() netcdf_unit[band2][:] = netcdf_writer.netcdfy_data(confidence_filtered) netcdf_unit[band2].units = '1' netcdf_unit[band2].valid_range = [0, 1.0] netcdf_unit[band2].coverage_content_type = 'modelResult' netcdf_unit[ band2].long_name = 'WOfS-Stats frequency confidence filtered layer' # Metadata dataset_data = DataArray(data=[metadata], dims=('time', )) netcdf_writer.create_variable(netcdf_unit, 'dataset', dataset_data, zlib=True) netcdf_unit['dataset'][:] = netcdf_writer.netcdfy_data( dataset_data.values) netcdf_unit.close() logger.info('completed', file=filename.name)
def main(argv=None): if argv is None: argv = sys.argv print(sys.argv) # If no user arguments provided if len(argv) < 2: str_usage = "You must specify a polygon ID" print(str_usage) sys.exit() # Set ITEM polygon for analysis polygon_id = int(argv[1]) # polygon_id = 33 # Import configuration details from NIDEM_configuration.ini config = configparser.ConfigParser() config.read('NIDEM_configuration.ini') # Set paths to ITEM relative, confidence and offset products item_offset_path = config['ITEM inputs']['item_offset_path'] item_relative_path = config['ITEM inputs']['item_relative_path'] item_conf_path = config['ITEM inputs']['item_conf_path'] item_polygon_path = config['ITEM inputs']['item_polygon_path'] # Set paths to elevation, bathymetry and shapefile datasets used to create NIDEM mask srtm30_raster = config['Masking inputs']['srtm30_raster'] ausbath09_raster = config['Masking inputs']['ausbath09_raster'] gbr30_raster = config['Masking inputs']['gbr30_raster'] nthaus30_raster = config['Masking inputs']['nthaus30_raster'] # Print run details print('Processing polygon {} from {}'.format(polygon_id, item_offset_path)) ################################## # Import and prepare ITEM raster # ################################## # Contours generated by `skimage.measure.find_contours` stop before the edge of nodata pixels. To prevent gaps # from occurring between adjacent NIDEM tiles, the following steps 'fill' pixels directly on the boundary of # two NIDEM tiles with the value of the nearest pixel with data. # Import raster item_filename = glob.glob('{}/ITEM_REL_{}_*.tif'.format(item_relative_path, polygon_id))[0] item_ds = gdal.Open(item_filename) item_array = item_ds.GetRasterBand(1).ReadAsArray() # Get coord string of polygon from ITEM array name to use for output names coord_str = item_filename[-17:-4] # Extract shape, projection info and geotransform data yrows, xcols = item_array.shape prj = item_ds.GetProjection() geotrans = item_ds.GetGeoTransform() upleft_x, x_size, x_rotation, upleft_y, y_rotation, y_size = geotrans bottomright_x = upleft_x + (x_size * xcols) bottomright_y = upleft_y + (y_size * yrows) # Identify valid intertidal area by selecting pixels between the lowest and highest ITEM intervals. This is # subsequently used to restrict the extent of interpolated elevation data to match the input ITEM polygons. valid_intertidal_extent = np.where((item_array > 0) & (item_array < 9), 1, 0) # Convert datatype to float to allow assigning nodata -6666 values to NaN item_array = item_array.astype('float32') item_array[item_array == -6666] = np.nan # First, identify areas to be filled by dilating non-NaN pixels by two pixels (i.e. ensuring vertical, horizontal # and diagonally adjacent pixels are filled): dilated_mask = nd.morphology.binary_dilation(~np.isnan(item_array), iterations=2) # For every pixel, identify the indices of the nearest pixel with data (i.e. data pixels will return their own # indices; nodata pixels will return the indices of the nearest data pixel). This output can be used to index # back into the original array, returning a new array where data pixels remain the same, but every nodata pixel # is filled with the value of the nearest data pixel: nearest_inds = nd.distance_transform_edt(input=np.isnan(item_array), return_distances=False, return_indices=True) item_array = item_array[tuple(nearest_inds)] # As we only want to fill pixels on the boundary of NIDEM tiles, set pixels outside the dilated area back to NaN: item_array[~dilated_mask] = np.nan ########################################## # Median and SD tide height per interval # ########################################## # Each ITEM v2.0 tidal interval boundary was produced from a composite of multiple Landsat images that cover a # range of tidal heights. To obtain an elevation relative to modelled mean sea level for each interval boundary, # we import a precomputed file containing the median tidal height for all Landsat images that were used to # generate the interval (Sagar et al. 2017, https://doi.org/10.1016/j.rse.2017.04.009). # Import ITEM offset values for each ITEM tidal interval, dividing by 1000 to give metre units item_offsets = np.loadtxt('{}/elevation.txt'.format(item_offset_path), delimiter=',', dtype='str') item_offsets = {int(key): [float(val) / 1000.0 for val in value.split(' ')] for (key, value) in item_offsets} contour_offsets = item_offsets[polygon_id] # The range of tide heights used to compute the above median tide height can vary significantly between tidal # modelling polygons. To quantify this range, we take the standard deviation of tide heights for all Landsat # images used to produce each ITEM interval. This represents a measure of the 'uncertainty' (not to be confused # with accuracy) of NIDEM elevations in m units for each contour. These values are subsequently interpolated to # return an estimate of uncertainty for each individual pixel in the NIDEM datasets: larger values indicate the # ITEM interval was produced from a composite of images with a larger range of tide heights. # Compute uncertainties for each interval, and create a lookup dict to link uncertainties to each NIDEM contour uncertainty_array = interval_uncertainty(polygon_id=polygon_id, item_polygon_path=item_polygon_path) uncertainty_dict = dict(zip(contour_offsets, uncertainty_array)) #################### # Extract contours # #################### # Here, we use `skimage.measure.find_contours` to extract contours along the boundary of each ITEM tidal interval # (e.g. 0.5 is the boundary between ITEM interval 0 and interval 1; 5.5 is the boundary between interval 5 and # interval 6). This function outputs a dictionary with ITEM interval boundaries as keys and lists of xy point # arrays as values. Contours are also exported as a shapefile with elevation and uncertainty attributes in metres. contour_dict = contour_extract(z_values=np.arange(0.5, 9.5, 1.0), ds_array=item_array, ds_crs='EPSG:3577', ds_affine=geotrans, output_shp=f'output_data/shapefile/nidem_contours/' f'NIDEM_contours_{polygon_id}_{coord_str}.shp', attribute_data={'elev_m': contour_offsets, 'uncert_m': uncertainty_array}, attribute_dtypes={'elev_m': 'float:9.2', 'uncert_m': 'float:9.2'}) ####################################################################### # Interpolate contours using TIN/Delaunay triangulation interpolation # ####################################################################### # Here we assign each previously generated contour with its modelled height relative to MSL, producing a set of # tidally tagged xyz points that can be used to interpolate elevations across the intertidal zone. We use the # linear interpolation method from `scipy.interpolate.griddata`, which computes a TIN/Delaunay triangulation of # the input data using Qhull before performing linear barycentric interpolation on each triangle. # If contours include valid data, proceed with interpolation try: # Combine all individual contours for each contour height, and insert a height above MSL column into array elev_contours = [np.insert(np.concatenate(v), 2, contour_offsets[i], axis=1) for i, v in enumerate(contour_dict.values())] # Combine all contour heights into a single array, and then extract xy points and z-values all_contours = np.concatenate(elev_contours) points_xy = all_contours[:, [1, 0]] values_elev = all_contours[:, 2] # Create a matching list of uncertainty values for each xy point values_uncert = np.array([np.round(uncertainty_dict[i], 2) for i in values_elev]) # Calculate bounds of ITEM layer to create interpolation grid (from-to-by values in metre units) grid_y, grid_x = np.mgrid[upleft_y:bottomright_y:1j * yrows, upleft_x:bottomright_x:1j * xcols] # Interpolate between points onto grid. This uses the 'linear' method from # scipy.interpolate.griddata, which computes a TIN/Delaunay triangulation of the input # data with Qhull and performs linear barycentric interpolation on each triangle print('Interpolating data for polygon {}'.format(polygon_id)) interp_elev_array = scipy.interpolate.griddata(points_xy, values_elev, (grid_y, grid_x), method='linear') interp_uncert_array = scipy.interpolate.griddata(points_xy, values_uncert, (grid_y, grid_x), method='linear') except ValueError: # If contours contain no valid data, create empty arrays interp_elev_array = np.full((yrows, xcols), -9999) interp_uncert_array = np.full((yrows, xcols), -9999) ######################################################### # Create ITEM confidence and elevation/bathymetry masks # ######################################################### # The following code applies a range of masks to remove pixels where elevation values are likely to be invalid: # # 1. Non-coastal terrestrial pixels with elevations greater than 25 m above MSL. This mask is computed using # SRTM-derived 1 Second Digital Elevation Model data (http://pid.geoscience.gov.au/dataset/ga/69769). # 2. Sub-tidal pixels with bathymetry values deeper than -25 m below MSL. This mask is computed by identifying # any pixels that are < -25 m in all of the national Australian Bathymetry and Topography Grid # (http://pid.geoscience.gov.au/dataset/ga/67703), gbr30 High-resolution depth model for the Great # Barrier Reef (http://pid.geoscience.gov.au/dataset/ga/115066) and nthaus30 High-resolution depth model # for Northern Australia (http://pid.geoscience.gov.au/dataset/ga/121620). # 3. Pixels with high ITEM confidence NDWI standard deviation (i.e. areas where inundation patterns are not driven # by tidal influences). This mask is computed using ITEM v2.0 confidence layer data from DEA. # Import ITEM confidence NDWI standard deviation array for polygon conf_filename = glob.glob('{}/ITEM_STD_{}_*.tif'.format(item_conf_path, polygon_id))[0] conf_ds = gdal.Open(conf_filename) # Reproject SRTM-derived 1 Second DEM to cell size and projection of NIDEM srtm30_reproj = reproject_to_template(input_raster=srtm30_raster, template_raster=item_filename, output_raster='scratch/temp.tif', nodata_val=-9999) # Reproject Australian Bathymetry and Topography Grid to cell size and projection of NIDEM ausbath09_reproj = reproject_to_template(input_raster=ausbath09_raster, template_raster=item_filename, output_raster='scratch/temp.tif', nodata_val=-9999) # Reproject gbr30 bathymetry to cell size and projection of NIDEM gbr30_reproj = reproject_to_template(input_raster=gbr30_raster, template_raster=item_filename, output_raster='scratch/temp.tif', nodata_val=-9999) # Reproject nthaus30 bathymetry to cell size and projection of NIDEM nthaus30_reproj = reproject_to_template(input_raster=nthaus30_raster, template_raster=item_filename, output_raster='scratch/temp.tif', nodata_val=-9999) # Convert raster datasets to arrays conf_array = conf_ds.GetRasterBand(1).ReadAsArray() srtm30_array = srtm30_reproj.GetRasterBand(1).ReadAsArray() ausbath09_array = ausbath09_reproj.GetRasterBand(1).ReadAsArray() gbr30_array = gbr30_reproj.GetRasterBand(1).ReadAsArray() nthaus30_array = nthaus30_reproj.GetRasterBand(1).ReadAsArray() # Convert arrays to boolean masks: # For elevation: any elevations > 25 m in SRTM 30m DEM # For bathymetry: any depths < -25 m in GBR30 AND nthaus30 AND Ausbath09 bathymetry # For ITEM confidence: any cells with NDWI STD > 0.25 elev_mask = srtm30_array > 25 bathy_mask = (ausbath09_array < -25) & (gbr30_array < -25) & (nthaus30_array < -25) conf_mask = conf_array > 0.25 # Create a combined mask with -9999 nodata in unmasked areas and where: # 1 = elevation mask # 2 = bathymetry mask # 3 = ITEM confidence mask nidem_mask = np.full(item_array.shape, -9999) nidem_mask[elev_mask] = 1 nidem_mask[bathy_mask] = 2 nidem_mask[conf_mask] = 3 ################################ # Export output NIDEM geoTIFFs # ################################ # Because the lowest and highest ITEM intervals (0 and 9) cannot be correctly interpolated as they have no lower # or upper bounds, the NIDEM layers are constrained to valid intertidal terrain (ITEM intervals 1-8). nidem_uncertainty = np.where(valid_intertidal_extent, interp_uncert_array, -9999).astype(np.float32) nidem_unfiltered = np.where(valid_intertidal_extent, interp_elev_array, -9999).astype(np.float32) # NIDEM is exported as two DEMs: an unfiltered layer, and a layer that is filtered to remove terrestrial (> 25 m) # and sub-tidal terrain (< -25 m) and pixels with high ITEM confidence NDWI standard deviation. Here we mask # the unfiltered layer by NIDEM mask to produce a filtered NIDEM layer: nidem_filtered = np.where(nidem_mask > 0, -9999, nidem_unfiltered).astype(np.float32) # Export filtered NIDEM as a GeoTIFF print(f'Exporting filtered NIDEM for polygon {polygon_id}') array_to_geotiff(fname=f'output_data/geotiff/nidem/NIDEM_{polygon_id}_{coord_str}.tif', data=nidem_filtered, geo_transform=geotrans, projection=prj, nodata_val=-9999) # Export unfiltered NIDEM as a GeoTIFF print(f'Exporting unfiltered NIDEM for polygon {polygon_id}') array_to_geotiff(fname=f'output_data/geotiff/nidem_unfiltered/NIDEM_unfiltered_{polygon_id}_{coord_str}.tif', data=nidem_unfiltered, geo_transform=geotrans, projection=prj, nodata_val=-9999) # Export NIDEM uncertainty layer as a GeoTIFF print(f'Exporting NIDEM uncertainty for polygon {polygon_id}') array_to_geotiff(fname=f'output_data/geotiff/nidem_uncertainty/NIDEM_uncertainty_{polygon_id}_{coord_str}.tif', data=nidem_uncertainty, geo_transform=geotrans, projection=prj, nodata_val=-9999) # Export NIDEM mask as a GeoTIFF print(f'Exporting NIDEM mask for polygon {polygon_id}') array_to_geotiff(fname=f'output_data/geotiff/nidem_mask/NIDEM_mask_{polygon_id}_{coord_str}.tif', data=nidem_mask.astype(int), geo_transform=geotrans, projection=prj, dtype=gdal.GDT_Int16, nodata_val=-9999) ###################### # Export NetCDF data # ###################### # If netcdf file already exists, delete it filename_netcdf = f'output_data/netcdf/NIDEM_{polygon_id}_{coord_str}.nc' if os.path.exists(filename_netcdf): os.remove(filename_netcdf) # Compute coords x_coords = netcdf_writer.netcdfy_coord(np.linspace(upleft_x + 12.5, bottomright_x - 12.5, num=xcols)) y_coords = netcdf_writer.netcdfy_coord(np.linspace(upleft_y - 12.5, bottomright_y + 12.5, num=yrows)) # Define output compression parameters comp_params = dict(zlib=True, complevel=9, shuffle=True, fletcher32=True) # Create new dataset output_netcdf = create_netcdf_storage_unit(filename=filename_netcdf, crs=CRS('EPSG:3577'), coordinates={'x': Coordinate(x_coords, 'metres'), 'y': Coordinate(y_coords, 'metres')}, variables={'nidem': Variable(dtype=np.dtype('float32'), nodata=-9999, dims=('y', 'x'), units='metres'), 'nidem_unfiltered': Variable(dtype=np.dtype('float32'), nodata=-9999, dims=('y', 'x'), units='metres'), 'nidem_uncertainty': Variable(dtype=np.dtype('float32'), nodata=-9999, dims=('y', 'x'), units='metres'), 'nidem_mask': Variable(dtype=np.dtype('int16'), nodata=-9999, dims=('y', 'x'), units='1')}, variable_params={'nidem': comp_params, 'nidem_unfiltered': comp_params, 'nidem_uncertainty': comp_params, 'nidem_mask': comp_params}) # dem: assign data and set variable attributes output_netcdf['nidem'][:] = netcdf_writer.netcdfy_data(nidem_filtered) output_netcdf['nidem'].valid_range = [-25.0, 25.0] output_netcdf['nidem'].standard_name = 'height_above_mean_sea_level' output_netcdf['nidem'].coverage_content_type = 'modelResult' output_netcdf['nidem'].long_name = 'National Intertidal Digital Elevation Model (NIDEM): elevation data in metre ' \ 'units relative to mean sea level for each pixel of intertidal terrain across ' \ 'the Australian coastline. Cleaned by masking out non-intertidal pixels' \ 'and pixels where tidal processes poorly explain patterns of inundation.' # dem_unfiltered: assign data and set variable attributes output_netcdf['nidem_unfiltered'][:] = netcdf_writer.netcdfy_data(nidem_unfiltered) output_netcdf['nidem_unfiltered'].standard_name = 'height_above_mean_sea_level' output_netcdf['nidem_unfiltered'].coverage_content_type = 'modelResult' output_netcdf['nidem_unfiltered'].long_name = 'NIDEM unfiltered: uncleaned elevation data in metre units ' \ 'relative to mean sea level for each pixel of intertidal terrain ' \ 'across the Australian coastline. Compared to the default NIDEM ' \ 'product, these layers have not been filtered to remove noise, ' \ 'artifacts or invalid elevation values.' # uncertainty: assign data and set variable attributes output_netcdf['nidem_uncertainty'][:] = netcdf_writer.netcdfy_data(nidem_uncertainty) output_netcdf['nidem_uncertainty'].standard_name = 'height_above_mean_sea_level' output_netcdf['nidem_uncertainty'].coverage_content_type = 'modelResult' output_netcdf['nidem_uncertainty'].long_name = 'NIDEM uncertainty: provides a measure of the uncertainty (not ' \ 'accuracy) of NIDEM elevations in metre units for each pixel. ' \ 'Represents the standard deviation of tide heights of all Landsat ' \ 'observations used to produce each ITEM 2.0 ten percent tidal ' \ 'interval.' # mask: assign data and set variable attributes output_netcdf['nidem_mask'][:] = netcdf_writer.netcdfy_data(nidem_mask) output_netcdf['nidem_mask'].valid_range = [1, 3] output_netcdf['nidem_mask'].coverage_content_type = 'qualityInformation' output_netcdf['nidem_mask'].long_name = 'NIDEM mask: flags non-intertidal terrestrial pixels with elevations ' \ 'greater than 25 m (value = 1), sub-tidal pixels with depths greater ' \ 'than -25 m (value = 2), and pixels where tidal processes poorly ' \ 'explain patterns of inundation (value = 3).' # Add global attributes output_netcdf.title = 'National Intertidal Digital Elevation Model 25m 1.0.0' output_netcdf.institution = 'Commonwealth of Australia (Geoscience Australia)' output_netcdf.product_version = '1.0.0' output_netcdf.license = 'CC BY Attribution 4.0 International License' output_netcdf.time_coverage_start = '1986-01-01' output_netcdf.time_coverage_end = '2016-12-31' output_netcdf.cdm_data_type = 'Grid' output_netcdf.contact = '*****@*****.**' output_netcdf.publisher_email = '*****@*****.**' output_netcdf.source = 'ITEM v2.0' output_netcdf.keywords = 'Tidal, Topography, Landsat, Elevation, Intertidal, MSL, ITEM, NIDEM, DEM, Coastal' output_netcdf.summary = "The National Intertidal Digital Elevation Model (NIDEM) product is a continental-scale " \ "dataset providing continuous elevation data for Australia's exposed intertidal zone. " \ "NIDEM provides the first three-dimensional representation of Australia's intertidal " \ "zone (excluding off-shore Territories and intertidal mangroves) at 25 m spatial " \ "resolution, addressing a key gap between the availability of sub-tidal bathymetry and " \ "terrestrial elevation data. NIDEM was generated by combining global tidal modelling " \ "with a 30-year time series archive of spatially and spectrally calibrated Landsat " \ "satellite data managed within the Digital Earth Australia (DEA) platform. NIDEM " \ "complements existing intertidal extent products, and provides data to support a new " \ "suite of use cases that require a more detailed understanding of the three-dimensional " \ "topography of the intertidal zone, such as hydrodynamic modelling, coastal risk " \ "management and ecological habitat mapping." # Close dataset output_netcdf.close()
def band2var(i): return Variable(dataset.dtypes[i], dataset.nodatavals[i], ('y', 'x'), '1')
def expand_var(var): return Variable(var.dtype, var.nodata, self._dimensions + var.dimensions, var.units)