def write_data(self, prod_name, measurement_name, tile_index, values): self._output_file_handles[prod_name][measurement_name][ (0, ) + tile_index[1:]] = netcdf_writer.netcdfy_data(values) self._output_file_handles[prod_name].sync() _LOG.debug( "Updated %s %s", measurement_name, "({})".format(", ".join(prettier_slice(x) for x in tile_index[1:])))
def write_data_variables(data_vars, nco): for name, variable in data_vars.items(): try: with dask.set_options(get=dask.async.get_sync): da.store(variable.data, nco[name], lock=True) except ValueError: nco[name][:] = netcdf_writer.netcdfy_data(variable.values) nco.sync()
def make_fake_netcdf_dataset(nc_name, yaml_doc): from datacube.model import Variable from datacube.storage.netcdf_writer import create_variable, netcdfy_data from netCDF4 import Dataset import numpy as np content = yaml_doc.read_text() npdata = np.array([content], dtype=bytes) with Dataset(nc_name, 'w') as nco: var = Variable(npdata.dtype, None, ('time', ), None) nco.createDimension('time', size=1) create_variable(nco, 'dataset', var) nco['dataset'][:] = netcdfy_data(npdata)
def do_stack_task(config, task): global_attributes = config['global_attributes'] global_attributes['history'] = get_history_attribute(config, task) variable_params = config['variable_params'] output_filename = Path(task['output_filename']) tile = task['tile'] data = datacube.api.GridWorkflow.load( tile, dask_chunks=config['storage']['chunking']) unwrapped_datasets = xr_apply(tile.sources, _unwrap_dataset_list, dtype='O') data['dataset'] = datasets_to_doc(unwrapped_datasets) nco = create_netcdf_storage_unit(output_filename, data.crs, data.coords, data.data_vars, variable_params, global_attributes) for name, variable in data.data_vars.items(): try: with dask.set_options(get=dask. async .get_sync): da.store(variable.data, nco[name], lock=True) except ValueError: nco[name][:] = netcdf_writer.netcdfy_data(variable.values) nco.sync() nco.close() def update_dataset_location(labels, dataset): new_dataset = copy.copy(dataset) new_dataset.local_uri = output_filename.absolute().as_uri() return [dataset] updated_datasets = xr_apply(unwrapped_datasets, update_dataset_location, dtype='O') new_tile = datacube.api.Tile(sources=updated_datasets, geobox=tile.geobox) new_data = datacube.api.GridWorkflow.load( new_tile, dask_chunks=config['storage']['chunking']) if not data.identical(new_data): _LOG.error("Mismatch found for %s, not indexing", output_filename) raise ValueError("Mismatch found for %s, not indexing" % output_filename) return unwrapped_datasets, output_filename.absolute().as_uri()
def test_create_string_variable(tmpnetcdf_filename): nco = create_netcdf(tmpnetcdf_filename) coord = create_coordinate(nco, 'greg', numpy.array([1.0, 3.0, 9.0]), 'cubic gregs') dtype = numpy.dtype('S100') data = numpy.array(["test-str1", "test-str2", "test-str3"], dtype=dtype) var = create_variable(nco, 'str_var', Variable(dtype, None, ('greg',), None)) var[:] = netcdfy_data(data) nco.close() with netCDF4.Dataset(tmpnetcdf_filename) as nco: assert 'str_var' in nco.variables assert netCDF4.chartostring(nco['str_var'][0]) == data[0]
def _create_storage_unit(self, stat: OutputProduct, output_filename: Path): all_measurement_defns = list(stat.product.measurements.values()) datasets = self._find_source_datasets(stat, uri=output_filename.as_uri()) variable_params = self._create_netcdf_var_params(stat) nco = self._nco_from_sources(datasets, self._geobox, all_measurement_defns, variable_params, output_filename) netcdf_writer.create_variable(nco, 'dataset', datasets, zlib=True) nco['dataset'][:] = netcdf_writer.netcdfy_data(datasets.values) return nco
def write_dataset_to_netcdf(access_unit, global_attributes, variable_params, filename, netcdfparams=None): nco = create_netcdf_storage_unit(filename, access_unit.crs, access_unit.coords, access_unit.data_vars, variable_params, global_attributes, netcdfparams) for name, variable in access_unit.data_vars.items(): nco[name][:] = netcdf_writer.netcdfy_data(variable.values) nco.close()
def test_create_string_variable(tmpnetcdf_filename): nco = create_netcdf(tmpnetcdf_filename) coord = create_coordinate(nco, 'greg', numpy.array([1.0, 3.0, 9.0]), 'cubic gregs') dtype = numpy.dtype('S100') data = numpy.array(["test-str1", "test-str2", "test-str3"], dtype=dtype) var = create_variable(nco, 'str_var', Variable(dtype, None, ('greg', ), None)) var[:] = netcdfy_data(data) nco.close() with netCDF4.Dataset(tmpnetcdf_filename) as nco: assert 'str_var' in nco.variables assert netCDF4.chartostring(nco['str_var'][0]) == data[0]
def saveNC(output,filename, history): nco=netcdf_writer.create_netcdf(filename) nco.history = (history.decode('utf-8').encode('ascii','replace')) coords=output.coords cnames=() for x in coords: netcdf_writer.create_coordinate(nco, x, coords[x].values, coords[x].units) cnames=cnames+(x,) netcdf_writer.create_grid_mapping_variable(nco, output.crs) for band in output.data_vars: output.data_vars[band].values[np.isnan(output.data_vars[band].values)]=nodata var= netcdf_writer.create_variable(nco, band, Variable(output.data_vars[band].dtype, nodata, cnames, None) ,set_crs=True) var[:] = netcdf_writer.netcdfy_data(output.data_vars[band].values) nco.close()
def test_create_string_variable(tmpdir, s1, s2, s3): tmpnetcdf_filename = get_tmpnetcdf_filename(tmpdir) str_var = 'str_var' nco = create_netcdf(tmpnetcdf_filename) coord = create_coordinate(nco, 'greg', numpy.array([1.0, 3.0, 9.0]), 'cubic gregs') dtype = numpy.dtype('S100') data = numpy.array([s1, s2, s3], dtype=dtype) var = create_variable(nco, str_var, Variable(dtype, None, ('greg',), None)) var[:] = netcdfy_data(data) nco.close() with netCDF4.Dataset(tmpnetcdf_filename) as nco: assert str_var in nco.variables for returned, expected in zip(read_strings_from_netcdf(tmpnetcdf_filename, variable=str_var), (s1, s2, s3)): assert returned == expected
def do_stack_task(task): datasets_to_add = None datasets_to_update = None datasets_to_archive = None global_attributes = task['global_attributes'] variable_params = task['variable_params'] output_filename = Path(task['output_filename']) tile = task['tile'] if task.get('make_new_datasets', False): datasets_to_add = make_datasets(tile, output_filename, task) datasets_to_archive = xr_apply(tile.sources, _single_dataset, dtype='O') output_datasets = datasets_to_add else: datasets_to_update = xr_apply(tile.sources, _single_dataset, dtype='O') output_datasets = datasets_to_update data = datacube.api.GridWorkflow.load(tile, dask_chunks=dict(time=1)) # TODO: chunk along output NetCDF chunk? data['dataset'] = datasets_to_doc(output_datasets) nco = create_netcdf_storage_unit(output_filename, data.crs, data.coords, data.data_vars, variable_params, global_attributes) for name, variable in data.data_vars.items(): try: with dask.set_options(get=dask.async.get_sync): da.store(variable.data, nco[name], lock=True) except ValueError: nco[name][:] = netcdf_writer.netcdfy_data(variable.values) nco.sync() nco.close() return datasets_to_add, datasets_to_update, datasets_to_archive
def write_dataset_to_netcdf(access_unit, global_attributes, variable_params, filename): if filename.exists(): raise RuntimeError('Storage Unit already exists: %s' % filename) try: filename.parent.mkdir(parents=True) except OSError: pass # _LOG.info("Writing storage unit: %s", filename) nco = netcdf_writer.create_netcdf(str(filename)) for name, coord in access_unit.coords.items(): netcdf_writer.create_coordinate(nco, name, coord.values, coord.units) netcdf_writer.create_grid_mapping_variable(nco, access_unit.crs) for name, variable in access_unit.data_vars.items(): # Create variable var_params = variable_params.get(name, {}) data_var = netcdf_writer.create_variable( nco, name, Variable(variable.dtype, getattr(variable, 'nodata', None), variable.dims, getattr(variable, 'units', '1')), **var_params) # Write data data_var[:] = netcdf_writer.netcdfy_data(variable.values) # TODO: 'flags_definition', 'spectral_definition'? for key, value in variable_params.get(name, {}).get('attrs', {}).items(): setattr(data_var, key, value) # write global atrributes for key, value in global_attributes.items(): setattr(nco, key, value) nco.close()
def write_dataset_to_netcdf(dataset, filename, global_attributes=None, variable_params=None, netcdfparams=None): """ Write a Data Cube style xarray Dataset to a NetCDF file Requires a spatial Dataset, with attached coordinates and global crs attribute. :param `xarray.Dataset` dataset: :param filename: Output filename :param global_attributes: Global file attributes. dict of attr_name: attr_value :param variable_params: dict of variable_name: {param_name: param_value, [...]} Allows setting storage and compression options per variable. See the `netCDF4.Dataset.createVariable` for available parameters. :param netcdfparams: Optional params affecting netCDF file creation """ global_attributes = global_attributes or {} variable_params = variable_params or {} filename = Path(filename) if not dataset.data_vars.keys(): raise DatacubeException('Cannot save empty dataset to disk.') if not hasattr(dataset, 'crs'): raise DatacubeException( 'Dataset does not contain CRS, cannot write to NetCDF file.') nco = create_netcdf_storage_unit(filename, dataset.crs, dataset.coords, dataset.data_vars, variable_params, global_attributes, netcdfparams) for name, variable in dataset.data_vars.items(): nco[name][:] = netcdf_writer.netcdfy_data(variable.values) nco.close()
def compute_and_write(self): """ Computes the wofs confidence and filtered summary bands and write to the corresponding NetCDF file. The file template and location etc are read from the configs. """ geo_box = self.grid_spec.tile_geobox(self.tile_index) # Compute metadata env = self.cfg.get_env_of_product('wofs_filtered_summary') with Datacube(app='wofs-confidence', env=env) as dc: product = dc.index.products.get_by_name('wofs_filtered_summary') extent = self.grid_spec.tile_geobox(self.tile_index).extent center_time = datetime.now() uri = self.get_filtered_uri() dts = make_dataset(product=product, sources=self.factor_sources, extent=extent, center_time=center_time, uri=uri) metadata = yaml.dump(dts.metadata_doc, Dumper=SafeDumper, encoding='utf-8') # Compute dataset coords coords = dict() coords['time'] = Coordinate( netcdf_writer.netcdfy_coord( np.array([datetime_to_seconds_since_1970(center_time)])), ['seconds since 1970-01-01 00:00:00']) for dim in geo_box.dimensions: coords[dim] = Coordinate( netcdf_writer.netcdfy_coord(geo_box.coordinates[dim].values), geo_box.coordinates[dim].units) # Compute dataset variables spatial_var = Variable(dtype=np.dtype(DEFAULT_TYPE), nodata=DEFAULT_FLOAT_NODATA, dims=('time', ) + geo_box.dimensions, units=('seconds since 1970-01-01 00:00:00', ) + geo_box.crs.units) band1 = self.cfg.cfg['wofs_filtered_summary']['confidence'] band2 = self.cfg.cfg['wofs_filtered_summary']['confidence_filtered'] vars = {band1: spatial_var, band2: spatial_var} vars_params = {band1: {}, band2: {}} global_atts = self.cfg.cfg['global_attributes'] # Get crs string crs = self.cfg.cfg['storage']['crs'] if self.cfg.cfg['storage'].get( 'crs') else DEFAULT_CRS # Create a dataset container filename = self.get_filtered_uri() logger.info('creating', file=filename.name) netcdf_unit = create_netcdf_storage_unit(filename=filename, crs=CRS(crs), coordinates=coords, variables=vars, global_attributes=global_atts, variable_params=vars_params) # Confidence layer: Fill variable data and set attributes confidence = self.compute_confidence() netcdf_unit[band1][:] = netcdf_writer.netcdfy_data(confidence) netcdf_unit[band1].units = '1' netcdf_unit[band1].valid_range = [0, 1.0] netcdf_unit[band1].coverage_content_type = 'modelResult' netcdf_unit[band1].long_name = \ 'Wofs Confidence Layer predicted by {}'.format(self.confidence_model.factors.__str__()) # Confidence filtered wofs-stats frequency layer: Fill variable data and set attributes confidence_filtered = self.compute_confidence_filtered() netcdf_unit[band2][:] = netcdf_writer.netcdfy_data(confidence_filtered) netcdf_unit[band2].units = '1' netcdf_unit[band2].valid_range = [0, 1.0] netcdf_unit[band2].coverage_content_type = 'modelResult' netcdf_unit[ band2].long_name = 'WOfS-Stats frequency confidence filtered layer' # Metadata dataset_data = DataArray(data=[metadata], dims=('time', )) netcdf_writer.create_variable(netcdf_unit, 'dataset', dataset_data, zlib=True) netcdf_unit['dataset'][:] = netcdf_writer.netcdfy_data( dataset_data.values) netcdf_unit.close() logger.info('completed', file=filename.name)
def main(argv=None): if argv is None: argv = sys.argv print(sys.argv) # If no user arguments provided if len(argv) < 2: str_usage = "You must specify a polygon ID" print(str_usage) sys.exit() # Set ITEM polygon for analysis polygon_id = int(argv[1]) # polygon_id = 33 # Import configuration details from NIDEM_configuration.ini config = configparser.ConfigParser() config.read('NIDEM_configuration.ini') # Set paths to ITEM relative, confidence and offset products item_offset_path = config['ITEM inputs']['item_offset_path'] item_relative_path = config['ITEM inputs']['item_relative_path'] item_conf_path = config['ITEM inputs']['item_conf_path'] item_polygon_path = config['ITEM inputs']['item_polygon_path'] # Set paths to elevation, bathymetry and shapefile datasets used to create NIDEM mask srtm30_raster = config['Masking inputs']['srtm30_raster'] ausbath09_raster = config['Masking inputs']['ausbath09_raster'] gbr30_raster = config['Masking inputs']['gbr30_raster'] nthaus30_raster = config['Masking inputs']['nthaus30_raster'] # Print run details print('Processing polygon {} from {}'.format(polygon_id, item_offset_path)) ################################## # Import and prepare ITEM raster # ################################## # Contours generated by `skimage.measure.find_contours` stop before the edge of nodata pixels. To prevent gaps # from occurring between adjacent NIDEM tiles, the following steps 'fill' pixels directly on the boundary of # two NIDEM tiles with the value of the nearest pixel with data. # Import raster item_filename = glob.glob('{}/ITEM_REL_{}_*.tif'.format(item_relative_path, polygon_id))[0] item_ds = gdal.Open(item_filename) item_array = item_ds.GetRasterBand(1).ReadAsArray() # Get coord string of polygon from ITEM array name to use for output names coord_str = item_filename[-17:-4] # Extract shape, projection info and geotransform data yrows, xcols = item_array.shape prj = item_ds.GetProjection() geotrans = item_ds.GetGeoTransform() upleft_x, x_size, x_rotation, upleft_y, y_rotation, y_size = geotrans bottomright_x = upleft_x + (x_size * xcols) bottomright_y = upleft_y + (y_size * yrows) # Identify valid intertidal area by selecting pixels between the lowest and highest ITEM intervals. This is # subsequently used to restrict the extent of interpolated elevation data to match the input ITEM polygons. valid_intertidal_extent = np.where((item_array > 0) & (item_array < 9), 1, 0) # Convert datatype to float to allow assigning nodata -6666 values to NaN item_array = item_array.astype('float32') item_array[item_array == -6666] = np.nan # First, identify areas to be filled by dilating non-NaN pixels by two pixels (i.e. ensuring vertical, horizontal # and diagonally adjacent pixels are filled): dilated_mask = nd.morphology.binary_dilation(~np.isnan(item_array), iterations=2) # For every pixel, identify the indices of the nearest pixel with data (i.e. data pixels will return their own # indices; nodata pixels will return the indices of the nearest data pixel). This output can be used to index # back into the original array, returning a new array where data pixels remain the same, but every nodata pixel # is filled with the value of the nearest data pixel: nearest_inds = nd.distance_transform_edt(input=np.isnan(item_array), return_distances=False, return_indices=True) item_array = item_array[tuple(nearest_inds)] # As we only want to fill pixels on the boundary of NIDEM tiles, set pixels outside the dilated area back to NaN: item_array[~dilated_mask] = np.nan ########################################## # Median and SD tide height per interval # ########################################## # Each ITEM v2.0 tidal interval boundary was produced from a composite of multiple Landsat images that cover a # range of tidal heights. To obtain an elevation relative to modelled mean sea level for each interval boundary, # we import a precomputed file containing the median tidal height for all Landsat images that were used to # generate the interval (Sagar et al. 2017, https://doi.org/10.1016/j.rse.2017.04.009). # Import ITEM offset values for each ITEM tidal interval, dividing by 1000 to give metre units item_offsets = np.loadtxt('{}/elevation.txt'.format(item_offset_path), delimiter=',', dtype='str') item_offsets = {int(key): [float(val) / 1000.0 for val in value.split(' ')] for (key, value) in item_offsets} contour_offsets = item_offsets[polygon_id] # The range of tide heights used to compute the above median tide height can vary significantly between tidal # modelling polygons. To quantify this range, we take the standard deviation of tide heights for all Landsat # images used to produce each ITEM interval. This represents a measure of the 'uncertainty' (not to be confused # with accuracy) of NIDEM elevations in m units for each contour. These values are subsequently interpolated to # return an estimate of uncertainty for each individual pixel in the NIDEM datasets: larger values indicate the # ITEM interval was produced from a composite of images with a larger range of tide heights. # Compute uncertainties for each interval, and create a lookup dict to link uncertainties to each NIDEM contour uncertainty_array = interval_uncertainty(polygon_id=polygon_id, item_polygon_path=item_polygon_path) uncertainty_dict = dict(zip(contour_offsets, uncertainty_array)) #################### # Extract contours # #################### # Here, we use `skimage.measure.find_contours` to extract contours along the boundary of each ITEM tidal interval # (e.g. 0.5 is the boundary between ITEM interval 0 and interval 1; 5.5 is the boundary between interval 5 and # interval 6). This function outputs a dictionary with ITEM interval boundaries as keys and lists of xy point # arrays as values. Contours are also exported as a shapefile with elevation and uncertainty attributes in metres. contour_dict = contour_extract(z_values=np.arange(0.5, 9.5, 1.0), ds_array=item_array, ds_crs='EPSG:3577', ds_affine=geotrans, output_shp=f'output_data/shapefile/nidem_contours/' f'NIDEM_contours_{polygon_id}_{coord_str}.shp', attribute_data={'elev_m': contour_offsets, 'uncert_m': uncertainty_array}, attribute_dtypes={'elev_m': 'float:9.2', 'uncert_m': 'float:9.2'}) ####################################################################### # Interpolate contours using TIN/Delaunay triangulation interpolation # ####################################################################### # Here we assign each previously generated contour with its modelled height relative to MSL, producing a set of # tidally tagged xyz points that can be used to interpolate elevations across the intertidal zone. We use the # linear interpolation method from `scipy.interpolate.griddata`, which computes a TIN/Delaunay triangulation of # the input data using Qhull before performing linear barycentric interpolation on each triangle. # If contours include valid data, proceed with interpolation try: # Combine all individual contours for each contour height, and insert a height above MSL column into array elev_contours = [np.insert(np.concatenate(v), 2, contour_offsets[i], axis=1) for i, v in enumerate(contour_dict.values())] # Combine all contour heights into a single array, and then extract xy points and z-values all_contours = np.concatenate(elev_contours) points_xy = all_contours[:, [1, 0]] values_elev = all_contours[:, 2] # Create a matching list of uncertainty values for each xy point values_uncert = np.array([np.round(uncertainty_dict[i], 2) for i in values_elev]) # Calculate bounds of ITEM layer to create interpolation grid (from-to-by values in metre units) grid_y, grid_x = np.mgrid[upleft_y:bottomright_y:1j * yrows, upleft_x:bottomright_x:1j * xcols] # Interpolate between points onto grid. This uses the 'linear' method from # scipy.interpolate.griddata, which computes a TIN/Delaunay triangulation of the input # data with Qhull and performs linear barycentric interpolation on each triangle print('Interpolating data for polygon {}'.format(polygon_id)) interp_elev_array = scipy.interpolate.griddata(points_xy, values_elev, (grid_y, grid_x), method='linear') interp_uncert_array = scipy.interpolate.griddata(points_xy, values_uncert, (grid_y, grid_x), method='linear') except ValueError: # If contours contain no valid data, create empty arrays interp_elev_array = np.full((yrows, xcols), -9999) interp_uncert_array = np.full((yrows, xcols), -9999) ######################################################### # Create ITEM confidence and elevation/bathymetry masks # ######################################################### # The following code applies a range of masks to remove pixels where elevation values are likely to be invalid: # # 1. Non-coastal terrestrial pixels with elevations greater than 25 m above MSL. This mask is computed using # SRTM-derived 1 Second Digital Elevation Model data (http://pid.geoscience.gov.au/dataset/ga/69769). # 2. Sub-tidal pixels with bathymetry values deeper than -25 m below MSL. This mask is computed by identifying # any pixels that are < -25 m in all of the national Australian Bathymetry and Topography Grid # (http://pid.geoscience.gov.au/dataset/ga/67703), gbr30 High-resolution depth model for the Great # Barrier Reef (http://pid.geoscience.gov.au/dataset/ga/115066) and nthaus30 High-resolution depth model # for Northern Australia (http://pid.geoscience.gov.au/dataset/ga/121620). # 3. Pixels with high ITEM confidence NDWI standard deviation (i.e. areas where inundation patterns are not driven # by tidal influences). This mask is computed using ITEM v2.0 confidence layer data from DEA. # Import ITEM confidence NDWI standard deviation array for polygon conf_filename = glob.glob('{}/ITEM_STD_{}_*.tif'.format(item_conf_path, polygon_id))[0] conf_ds = gdal.Open(conf_filename) # Reproject SRTM-derived 1 Second DEM to cell size and projection of NIDEM srtm30_reproj = reproject_to_template(input_raster=srtm30_raster, template_raster=item_filename, output_raster='scratch/temp.tif', nodata_val=-9999) # Reproject Australian Bathymetry and Topography Grid to cell size and projection of NIDEM ausbath09_reproj = reproject_to_template(input_raster=ausbath09_raster, template_raster=item_filename, output_raster='scratch/temp.tif', nodata_val=-9999) # Reproject gbr30 bathymetry to cell size and projection of NIDEM gbr30_reproj = reproject_to_template(input_raster=gbr30_raster, template_raster=item_filename, output_raster='scratch/temp.tif', nodata_val=-9999) # Reproject nthaus30 bathymetry to cell size and projection of NIDEM nthaus30_reproj = reproject_to_template(input_raster=nthaus30_raster, template_raster=item_filename, output_raster='scratch/temp.tif', nodata_val=-9999) # Convert raster datasets to arrays conf_array = conf_ds.GetRasterBand(1).ReadAsArray() srtm30_array = srtm30_reproj.GetRasterBand(1).ReadAsArray() ausbath09_array = ausbath09_reproj.GetRasterBand(1).ReadAsArray() gbr30_array = gbr30_reproj.GetRasterBand(1).ReadAsArray() nthaus30_array = nthaus30_reproj.GetRasterBand(1).ReadAsArray() # Convert arrays to boolean masks: # For elevation: any elevations > 25 m in SRTM 30m DEM # For bathymetry: any depths < -25 m in GBR30 AND nthaus30 AND Ausbath09 bathymetry # For ITEM confidence: any cells with NDWI STD > 0.25 elev_mask = srtm30_array > 25 bathy_mask = (ausbath09_array < -25) & (gbr30_array < -25) & (nthaus30_array < -25) conf_mask = conf_array > 0.25 # Create a combined mask with -9999 nodata in unmasked areas and where: # 1 = elevation mask # 2 = bathymetry mask # 3 = ITEM confidence mask nidem_mask = np.full(item_array.shape, -9999) nidem_mask[elev_mask] = 1 nidem_mask[bathy_mask] = 2 nidem_mask[conf_mask] = 3 ################################ # Export output NIDEM geoTIFFs # ################################ # Because the lowest and highest ITEM intervals (0 and 9) cannot be correctly interpolated as they have no lower # or upper bounds, the NIDEM layers are constrained to valid intertidal terrain (ITEM intervals 1-8). nidem_uncertainty = np.where(valid_intertidal_extent, interp_uncert_array, -9999).astype(np.float32) nidem_unfiltered = np.where(valid_intertidal_extent, interp_elev_array, -9999).astype(np.float32) # NIDEM is exported as two DEMs: an unfiltered layer, and a layer that is filtered to remove terrestrial (> 25 m) # and sub-tidal terrain (< -25 m) and pixels with high ITEM confidence NDWI standard deviation. Here we mask # the unfiltered layer by NIDEM mask to produce a filtered NIDEM layer: nidem_filtered = np.where(nidem_mask > 0, -9999, nidem_unfiltered).astype(np.float32) # Export filtered NIDEM as a GeoTIFF print(f'Exporting filtered NIDEM for polygon {polygon_id}') array_to_geotiff(fname=f'output_data/geotiff/nidem/NIDEM_{polygon_id}_{coord_str}.tif', data=nidem_filtered, geo_transform=geotrans, projection=prj, nodata_val=-9999) # Export unfiltered NIDEM as a GeoTIFF print(f'Exporting unfiltered NIDEM for polygon {polygon_id}') array_to_geotiff(fname=f'output_data/geotiff/nidem_unfiltered/NIDEM_unfiltered_{polygon_id}_{coord_str}.tif', data=nidem_unfiltered, geo_transform=geotrans, projection=prj, nodata_val=-9999) # Export NIDEM uncertainty layer as a GeoTIFF print(f'Exporting NIDEM uncertainty for polygon {polygon_id}') array_to_geotiff(fname=f'output_data/geotiff/nidem_uncertainty/NIDEM_uncertainty_{polygon_id}_{coord_str}.tif', data=nidem_uncertainty, geo_transform=geotrans, projection=prj, nodata_val=-9999) # Export NIDEM mask as a GeoTIFF print(f'Exporting NIDEM mask for polygon {polygon_id}') array_to_geotiff(fname=f'output_data/geotiff/nidem_mask/NIDEM_mask_{polygon_id}_{coord_str}.tif', data=nidem_mask.astype(int), geo_transform=geotrans, projection=prj, dtype=gdal.GDT_Int16, nodata_val=-9999) ###################### # Export NetCDF data # ###################### # If netcdf file already exists, delete it filename_netcdf = f'output_data/netcdf/NIDEM_{polygon_id}_{coord_str}.nc' if os.path.exists(filename_netcdf): os.remove(filename_netcdf) # Compute coords x_coords = netcdf_writer.netcdfy_coord(np.linspace(upleft_x + 12.5, bottomright_x - 12.5, num=xcols)) y_coords = netcdf_writer.netcdfy_coord(np.linspace(upleft_y - 12.5, bottomright_y + 12.5, num=yrows)) # Define output compression parameters comp_params = dict(zlib=True, complevel=9, shuffle=True, fletcher32=True) # Create new dataset output_netcdf = create_netcdf_storage_unit(filename=filename_netcdf, crs=CRS('EPSG:3577'), coordinates={'x': Coordinate(x_coords, 'metres'), 'y': Coordinate(y_coords, 'metres')}, variables={'nidem': Variable(dtype=np.dtype('float32'), nodata=-9999, dims=('y', 'x'), units='metres'), 'nidem_unfiltered': Variable(dtype=np.dtype('float32'), nodata=-9999, dims=('y', 'x'), units='metres'), 'nidem_uncertainty': Variable(dtype=np.dtype('float32'), nodata=-9999, dims=('y', 'x'), units='metres'), 'nidem_mask': Variable(dtype=np.dtype('int16'), nodata=-9999, dims=('y', 'x'), units='1')}, variable_params={'nidem': comp_params, 'nidem_unfiltered': comp_params, 'nidem_uncertainty': comp_params, 'nidem_mask': comp_params}) # dem: assign data and set variable attributes output_netcdf['nidem'][:] = netcdf_writer.netcdfy_data(nidem_filtered) output_netcdf['nidem'].valid_range = [-25.0, 25.0] output_netcdf['nidem'].standard_name = 'height_above_mean_sea_level' output_netcdf['nidem'].coverage_content_type = 'modelResult' output_netcdf['nidem'].long_name = 'National Intertidal Digital Elevation Model (NIDEM): elevation data in metre ' \ 'units relative to mean sea level for each pixel of intertidal terrain across ' \ 'the Australian coastline. Cleaned by masking out non-intertidal pixels' \ 'and pixels where tidal processes poorly explain patterns of inundation.' # dem_unfiltered: assign data and set variable attributes output_netcdf['nidem_unfiltered'][:] = netcdf_writer.netcdfy_data(nidem_unfiltered) output_netcdf['nidem_unfiltered'].standard_name = 'height_above_mean_sea_level' output_netcdf['nidem_unfiltered'].coverage_content_type = 'modelResult' output_netcdf['nidem_unfiltered'].long_name = 'NIDEM unfiltered: uncleaned elevation data in metre units ' \ 'relative to mean sea level for each pixel of intertidal terrain ' \ 'across the Australian coastline. Compared to the default NIDEM ' \ 'product, these layers have not been filtered to remove noise, ' \ 'artifacts or invalid elevation values.' # uncertainty: assign data and set variable attributes output_netcdf['nidem_uncertainty'][:] = netcdf_writer.netcdfy_data(nidem_uncertainty) output_netcdf['nidem_uncertainty'].standard_name = 'height_above_mean_sea_level' output_netcdf['nidem_uncertainty'].coverage_content_type = 'modelResult' output_netcdf['nidem_uncertainty'].long_name = 'NIDEM uncertainty: provides a measure of the uncertainty (not ' \ 'accuracy) of NIDEM elevations in metre units for each pixel. ' \ 'Represents the standard deviation of tide heights of all Landsat ' \ 'observations used to produce each ITEM 2.0 ten percent tidal ' \ 'interval.' # mask: assign data and set variable attributes output_netcdf['nidem_mask'][:] = netcdf_writer.netcdfy_data(nidem_mask) output_netcdf['nidem_mask'].valid_range = [1, 3] output_netcdf['nidem_mask'].coverage_content_type = 'qualityInformation' output_netcdf['nidem_mask'].long_name = 'NIDEM mask: flags non-intertidal terrestrial pixels with elevations ' \ 'greater than 25 m (value = 1), sub-tidal pixels with depths greater ' \ 'than -25 m (value = 2), and pixels where tidal processes poorly ' \ 'explain patterns of inundation (value = 3).' # Add global attributes output_netcdf.title = 'National Intertidal Digital Elevation Model 25m 1.0.0' output_netcdf.institution = 'Commonwealth of Australia (Geoscience Australia)' output_netcdf.product_version = '1.0.0' output_netcdf.license = 'CC BY Attribution 4.0 International License' output_netcdf.time_coverage_start = '1986-01-01' output_netcdf.time_coverage_end = '2016-12-31' output_netcdf.cdm_data_type = 'Grid' output_netcdf.contact = '*****@*****.**' output_netcdf.publisher_email = '*****@*****.**' output_netcdf.source = 'ITEM v2.0' output_netcdf.keywords = 'Tidal, Topography, Landsat, Elevation, Intertidal, MSL, ITEM, NIDEM, DEM, Coastal' output_netcdf.summary = "The National Intertidal Digital Elevation Model (NIDEM) product is a continental-scale " \ "dataset providing continuous elevation data for Australia's exposed intertidal zone. " \ "NIDEM provides the first three-dimensional representation of Australia's intertidal " \ "zone (excluding off-shore Territories and intertidal mangroves) at 25 m spatial " \ "resolution, addressing a key gap between the availability of sub-tidal bathymetry and " \ "terrestrial elevation data. NIDEM was generated by combining global tidal modelling " \ "with a 30-year time series archive of spatially and spectrally calibrated Landsat " \ "satellite data managed within the Digital Earth Australia (DEA) platform. NIDEM " \ "complements existing intertidal extent products, and provides data to support a new " \ "suite of use cases that require a more detailed understanding of the three-dimensional " \ "topography of the intertidal zone, such as hydrodynamic modelling, coastal risk " \ "management and ecological habitat mapping." # Close dataset output_netcdf.close()
def write_data(self, prod_name, measurement_name, tile_index, values): self._output_file_handles[prod_name][measurement_name][ (0, ) + tile_index[1:]] = netcdf_writer.netcdfy_data(values) self._output_file_handles[prod_name].sync() _LOG.debug("Updated %s %s", measurement_name, tile_index[1:])