def write_data(self, prod_name, measurement_name, tile_index, values):
self._output_file_handles[prod_name][measurement_name][
(0, ) + tile_index[1:]] = netcdf_writer.netcdfy_data(values)
self._output_file_handles[prod_name].sync()
_LOG.debug(
"Updated %s %s", measurement_name,
"({})".format(", ".join(prettier_slice(x)
for x in tile_index[1:])))
def write_data_variables(data_vars, nco):
for name, variable in data_vars.items():
try:
with dask.set_options(get=dask.async.get_sync):
da.store(variable.data, nco[name], lock=True)
except ValueError:
nco[name][:] = netcdf_writer.netcdfy_data(variable.values)
nco.sync()
def make_fake_netcdf_dataset(nc_name, yaml_doc):
from datacube.model import Variable
from datacube.storage.netcdf_writer import create_variable, netcdfy_data
from netCDF4 import Dataset
import numpy as np
content = yaml_doc.read_text()
npdata = np.array([content], dtype=bytes)
with Dataset(nc_name, 'w') as nco:
var = Variable(npdata.dtype, None, ('time', ), None)
nco.createDimension('time', size=1)
create_variable(nco, 'dataset', var)
nco['dataset'][:] = netcdfy_data(npdata)
def do_stack_task(config, task):
global_attributes = config['global_attributes']
global_attributes['history'] = get_history_attribute(config, task)
variable_params = config['variable_params']
output_filename = Path(task['output_filename'])
tile = task['tile']
data = datacube.api.GridWorkflow.load(
tile, dask_chunks=config['storage']['chunking'])
unwrapped_datasets = xr_apply(tile.sources,
_unwrap_dataset_list,
dtype='O')
data['dataset'] = datasets_to_doc(unwrapped_datasets)
nco = create_netcdf_storage_unit(output_filename, data.crs, data.coords,
data.data_vars, variable_params,
global_attributes)
for name, variable in data.data_vars.items():
try:
with dask.set_options(get=dask. async .get_sync):
da.store(variable.data, nco[name], lock=True)
except ValueError:
nco[name][:] = netcdf_writer.netcdfy_data(variable.values)
nco.sync()
nco.close()
def update_dataset_location(labels, dataset):
new_dataset = copy.copy(dataset)
new_dataset.local_uri = output_filename.absolute().as_uri()
return [dataset]
updated_datasets = xr_apply(unwrapped_datasets,
update_dataset_location,
dtype='O')
new_tile = datacube.api.Tile(sources=updated_datasets, geobox=tile.geobox)
new_data = datacube.api.GridWorkflow.load(
new_tile, dask_chunks=config['storage']['chunking'])
if not data.identical(new_data):
_LOG.error("Mismatch found for %s, not indexing", output_filename)
raise ValueError("Mismatch found for %s, not indexing" %
output_filename)
return unwrapped_datasets, output_filename.absolute().as_uri()
def test_create_string_variable(tmpnetcdf_filename):
nco = create_netcdf(tmpnetcdf_filename)
coord = create_coordinate(nco, 'greg', numpy.array([1.0, 3.0, 9.0]), 'cubic gregs')
dtype = numpy.dtype('S100')
data = numpy.array(["test-str1", "test-str2", "test-str3"], dtype=dtype)
var = create_variable(nco, 'str_var', Variable(dtype, None, ('greg',), None))
var[:] = netcdfy_data(data)
nco.close()
with netCDF4.Dataset(tmpnetcdf_filename) as nco:
assert 'str_var' in nco.variables
assert netCDF4.chartostring(nco['str_var'][0]) == data[0]
def _create_storage_unit(self, stat: OutputProduct, output_filename: Path):
all_measurement_defns = list(stat.product.measurements.values())
datasets = self._find_source_datasets(stat,
uri=output_filename.as_uri())
variable_params = self._create_netcdf_var_params(stat)
nco = self._nco_from_sources(datasets, self._geobox,
all_measurement_defns, variable_params,
output_filename)
netcdf_writer.create_variable(nco, 'dataset', datasets, zlib=True)
nco['dataset'][:] = netcdf_writer.netcdfy_data(datasets.values)
return nco
def write_dataset_to_netcdf(access_unit,
global_attributes,
variable_params,
filename,
netcdfparams=None):
nco = create_netcdf_storage_unit(filename, access_unit.crs,
access_unit.coords, access_unit.data_vars,
variable_params, global_attributes,
netcdfparams)
for name, variable in access_unit.data_vars.items():
nco[name][:] = netcdf_writer.netcdfy_data(variable.values)
nco.close()
def test_create_string_variable(tmpnetcdf_filename):
nco = create_netcdf(tmpnetcdf_filename)
coord = create_coordinate(nco, 'greg', numpy.array([1.0, 3.0, 9.0]),
'cubic gregs')
dtype = numpy.dtype('S100')
data = numpy.array(["test-str1", "test-str2", "test-str3"], dtype=dtype)
var = create_variable(nco, 'str_var',
Variable(dtype, None, ('greg', ), None))
var[:] = netcdfy_data(data)
nco.close()
with netCDF4.Dataset(tmpnetcdf_filename) as nco:
assert 'str_var' in nco.variables
assert netCDF4.chartostring(nco['str_var'][0]) == data[0]
def saveNC(output,filename, history):
nco=netcdf_writer.create_netcdf(filename)
nco.history = (history.decode('utf-8').encode('ascii','replace'))
coords=output.coords
cnames=()
for x in coords:
netcdf_writer.create_coordinate(nco, x, coords[x].values, coords[x].units)
cnames=cnames+(x,)
netcdf_writer.create_grid_mapping_variable(nco, output.crs)
for band in output.data_vars:
output.data_vars[band].values[np.isnan(output.data_vars[band].values)]=nodata
var= netcdf_writer.create_variable(nco, band, Variable(output.data_vars[band].dtype, nodata, cnames, None) ,set_crs=True)
var[:] = netcdf_writer.netcdfy_data(output.data_vars[band].values)
nco.close()
def test_create_string_variable(tmpdir, s1, s2, s3):
tmpnetcdf_filename = get_tmpnetcdf_filename(tmpdir)
str_var = 'str_var'
nco = create_netcdf(tmpnetcdf_filename)
coord = create_coordinate(nco, 'greg', numpy.array([1.0, 3.0, 9.0]), 'cubic gregs')
dtype = numpy.dtype('S100')
data = numpy.array([s1, s2, s3], dtype=dtype)
var = create_variable(nco, str_var, Variable(dtype, None, ('greg',), None))
var[:] = netcdfy_data(data)
nco.close()
with netCDF4.Dataset(tmpnetcdf_filename) as nco:
assert str_var in nco.variables
for returned, expected in zip(read_strings_from_netcdf(tmpnetcdf_filename, variable=str_var), (s1, s2, s3)):
assert returned == expected
def do_stack_task(task):
datasets_to_add = None
datasets_to_update = None
datasets_to_archive = None
global_attributes = task['global_attributes']
variable_params = task['variable_params']
output_filename = Path(task['output_filename'])
tile = task['tile']
if task.get('make_new_datasets', False):
datasets_to_add = make_datasets(tile, output_filename, task)
datasets_to_archive = xr_apply(tile.sources, _single_dataset, dtype='O')
output_datasets = datasets_to_add
else:
datasets_to_update = xr_apply(tile.sources, _single_dataset, dtype='O')
output_datasets = datasets_to_update
data = datacube.api.GridWorkflow.load(tile, dask_chunks=dict(time=1)) # TODO: chunk along output NetCDF chunk?
data['dataset'] = datasets_to_doc(output_datasets)
nco = create_netcdf_storage_unit(output_filename,
data.crs,
data.coords,
data.data_vars,
variable_params,
global_attributes)
for name, variable in data.data_vars.items():
try:
with dask.set_options(get=dask.async.get_sync):
da.store(variable.data, nco[name], lock=True)
except ValueError:
nco[name][:] = netcdf_writer.netcdfy_data(variable.values)
nco.sync()
nco.close()
return datasets_to_add, datasets_to_update, datasets_to_archive
def write_dataset_to_netcdf(access_unit, global_attributes, variable_params,
filename):
if filename.exists():
raise RuntimeError('Storage Unit already exists: %s' % filename)
try:
filename.parent.mkdir(parents=True)
except OSError:
pass
# _LOG.info("Writing storage unit: %s", filename)
nco = netcdf_writer.create_netcdf(str(filename))
for name, coord in access_unit.coords.items():
netcdf_writer.create_coordinate(nco, name, coord.values, coord.units)
netcdf_writer.create_grid_mapping_variable(nco, access_unit.crs)
for name, variable in access_unit.data_vars.items():
# Create variable
var_params = variable_params.get(name, {})
data_var = netcdf_writer.create_variable(
nco, name,
Variable(variable.dtype, getattr(variable, 'nodata',
None), variable.dims,
getattr(variable, 'units', '1')), **var_params)
# Write data
data_var[:] = netcdf_writer.netcdfy_data(variable.values)
# TODO: 'flags_definition', 'spectral_definition'?
for key, value in variable_params.get(name, {}).get('attrs',
{}).items():
setattr(data_var, key, value)
# write global atrributes
for key, value in global_attributes.items():
setattr(nco, key, value)
nco.close()
def write_dataset_to_netcdf(dataset,
filename,
global_attributes=None,
variable_params=None,
netcdfparams=None):
"""
Write a Data Cube style xarray Dataset to a NetCDF file
Requires a spatial Dataset, with attached coordinates and global crs attribute.
:param `xarray.Dataset` dataset:
:param filename: Output filename
:param global_attributes: Global file attributes. dict of attr_name: attr_value
:param variable_params: dict of variable_name: {param_name: param_value, [...]}
Allows setting storage and compression options per variable.
See the `netCDF4.Dataset.createVariable` for available
parameters.
:param netcdfparams: Optional params affecting netCDF file creation
"""
global_attributes = global_attributes or {}
variable_params = variable_params or {}
filename = Path(filename)
if not dataset.data_vars.keys():
raise DatacubeException('Cannot save empty dataset to disk.')
if not hasattr(dataset, 'crs'):
raise DatacubeException(
'Dataset does not contain CRS, cannot write to NetCDF file.')
nco = create_netcdf_storage_unit(filename, dataset.crs, dataset.coords,
dataset.data_vars, variable_params,
global_attributes, netcdfparams)
for name, variable in dataset.data_vars.items():
nco[name][:] = netcdf_writer.netcdfy_data(variable.values)
nco.close()
def compute_and_write(self):
"""
Computes the wofs confidence and filtered summary bands and write to the
corresponding NetCDF file. The file template and location etc are read from the configs.
"""
geo_box = self.grid_spec.tile_geobox(self.tile_index)
# Compute metadata
env = self.cfg.get_env_of_product('wofs_filtered_summary')
with Datacube(app='wofs-confidence', env=env) as dc:
product = dc.index.products.get_by_name('wofs_filtered_summary')
extent = self.grid_spec.tile_geobox(self.tile_index).extent
center_time = datetime.now()
uri = self.get_filtered_uri()
dts = make_dataset(product=product,
sources=self.factor_sources,
extent=extent,
center_time=center_time,
uri=uri)
metadata = yaml.dump(dts.metadata_doc,
Dumper=SafeDumper,
encoding='utf-8')
# Compute dataset coords
coords = dict()
coords['time'] = Coordinate(
netcdf_writer.netcdfy_coord(
np.array([datetime_to_seconds_since_1970(center_time)])),
['seconds since 1970-01-01 00:00:00'])
for dim in geo_box.dimensions:
coords[dim] = Coordinate(
netcdf_writer.netcdfy_coord(geo_box.coordinates[dim].values),
geo_box.coordinates[dim].units)
# Compute dataset variables
spatial_var = Variable(dtype=np.dtype(DEFAULT_TYPE),
nodata=DEFAULT_FLOAT_NODATA,
dims=('time', ) + geo_box.dimensions,
units=('seconds since 1970-01-01 00:00:00', ) +
geo_box.crs.units)
band1 = self.cfg.cfg['wofs_filtered_summary']['confidence']
band2 = self.cfg.cfg['wofs_filtered_summary']['confidence_filtered']
vars = {band1: spatial_var, band2: spatial_var}
vars_params = {band1: {}, band2: {}}
global_atts = self.cfg.cfg['global_attributes']
# Get crs string
crs = self.cfg.cfg['storage']['crs'] if self.cfg.cfg['storage'].get(
'crs') else DEFAULT_CRS
# Create a dataset container
filename = self.get_filtered_uri()
logger.info('creating', file=filename.name)
netcdf_unit = create_netcdf_storage_unit(filename=filename,
crs=CRS(crs),
coordinates=coords,
variables=vars,
global_attributes=global_atts,
variable_params=vars_params)
# Confidence layer: Fill variable data and set attributes
confidence = self.compute_confidence()
netcdf_unit[band1][:] = netcdf_writer.netcdfy_data(confidence)
netcdf_unit[band1].units = '1'
netcdf_unit[band1].valid_range = [0, 1.0]
netcdf_unit[band1].coverage_content_type = 'modelResult'
netcdf_unit[band1].long_name = \
'Wofs Confidence Layer predicted by {}'.format(self.confidence_model.factors.__str__())
# Confidence filtered wofs-stats frequency layer: Fill variable data and set attributes
confidence_filtered = self.compute_confidence_filtered()
netcdf_unit[band2][:] = netcdf_writer.netcdfy_data(confidence_filtered)
netcdf_unit[band2].units = '1'
netcdf_unit[band2].valid_range = [0, 1.0]
netcdf_unit[band2].coverage_content_type = 'modelResult'
netcdf_unit[
band2].long_name = 'WOfS-Stats frequency confidence filtered layer'
# Metadata
dataset_data = DataArray(data=[metadata], dims=('time', ))
netcdf_writer.create_variable(netcdf_unit,
'dataset',
dataset_data,
zlib=True)
netcdf_unit['dataset'][:] = netcdf_writer.netcdfy_data(
dataset_data.values)
netcdf_unit.close()
logger.info('completed', file=filename.name)
def main(argv=None):
if argv is None:
argv = sys.argv
print(sys.argv)
# If no user arguments provided
if len(argv) < 2:
str_usage = "You must specify a polygon ID"
print(str_usage)
sys.exit()
# Set ITEM polygon for analysis
polygon_id = int(argv[1]) # polygon_id = 33
# Import configuration details from NIDEM_configuration.ini
config = configparser.ConfigParser()
config.read('NIDEM_configuration.ini')
# Set paths to ITEM relative, confidence and offset products
item_offset_path = config['ITEM inputs']['item_offset_path']
item_relative_path = config['ITEM inputs']['item_relative_path']
item_conf_path = config['ITEM inputs']['item_conf_path']
item_polygon_path = config['ITEM inputs']['item_polygon_path']
# Set paths to elevation, bathymetry and shapefile datasets used to create NIDEM mask
srtm30_raster = config['Masking inputs']['srtm30_raster']
ausbath09_raster = config['Masking inputs']['ausbath09_raster']
gbr30_raster = config['Masking inputs']['gbr30_raster']
nthaus30_raster = config['Masking inputs']['nthaus30_raster']
# Print run details
print('Processing polygon {} from {}'.format(polygon_id, item_offset_path))
##################################
# Import and prepare ITEM raster #
##################################
# Contours generated by `skimage.measure.find_contours` stop before the edge of nodata pixels. To prevent gaps
# from occurring between adjacent NIDEM tiles, the following steps 'fill' pixels directly on the boundary of
# two NIDEM tiles with the value of the nearest pixel with data.
# Import raster
item_filename = glob.glob('{}/ITEM_REL_{}_*.tif'.format(item_relative_path, polygon_id))[0]
item_ds = gdal.Open(item_filename)
item_array = item_ds.GetRasterBand(1).ReadAsArray()
# Get coord string of polygon from ITEM array name to use for output names
coord_str = item_filename[-17:-4]
# Extract shape, projection info and geotransform data
yrows, xcols = item_array.shape
prj = item_ds.GetProjection()
geotrans = item_ds.GetGeoTransform()
upleft_x, x_size, x_rotation, upleft_y, y_rotation, y_size = geotrans
bottomright_x = upleft_x + (x_size * xcols)
bottomright_y = upleft_y + (y_size * yrows)
# Identify valid intertidal area by selecting pixels between the lowest and highest ITEM intervals. This is
# subsequently used to restrict the extent of interpolated elevation data to match the input ITEM polygons.
valid_intertidal_extent = np.where((item_array > 0) & (item_array < 9), 1, 0)
# Convert datatype to float to allow assigning nodata -6666 values to NaN
item_array = item_array.astype('float32')
item_array[item_array == -6666] = np.nan
# First, identify areas to be filled by dilating non-NaN pixels by two pixels (i.e. ensuring vertical, horizontal
# and diagonally adjacent pixels are filled):
dilated_mask = nd.morphology.binary_dilation(~np.isnan(item_array), iterations=2)
# For every pixel, identify the indices of the nearest pixel with data (i.e. data pixels will return their own
# indices; nodata pixels will return the indices of the nearest data pixel). This output can be used to index
# back into the original array, returning a new array where data pixels remain the same, but every nodata pixel
# is filled with the value of the nearest data pixel:
nearest_inds = nd.distance_transform_edt(input=np.isnan(item_array), return_distances=False, return_indices=True)
item_array = item_array[tuple(nearest_inds)]
# As we only want to fill pixels on the boundary of NIDEM tiles, set pixels outside the dilated area back to NaN:
item_array[~dilated_mask] = np.nan
##########################################
# Median and SD tide height per interval #
##########################################
# Each ITEM v2.0 tidal interval boundary was produced from a composite of multiple Landsat images that cover a
# range of tidal heights. To obtain an elevation relative to modelled mean sea level for each interval boundary,
# we import a precomputed file containing the median tidal height for all Landsat images that were used to
# generate the interval (Sagar et al. 2017, https://doi.org/10.1016/j.rse.2017.04.009).
# Import ITEM offset values for each ITEM tidal interval, dividing by 1000 to give metre units
item_offsets = np.loadtxt('{}/elevation.txt'.format(item_offset_path), delimiter=',', dtype='str')
item_offsets = {int(key): [float(val) / 1000.0 for val in value.split(' ')] for (key, value) in item_offsets}
contour_offsets = item_offsets[polygon_id]
# The range of tide heights used to compute the above median tide height can vary significantly between tidal
# modelling polygons. To quantify this range, we take the standard deviation of tide heights for all Landsat
# images used to produce each ITEM interval. This represents a measure of the 'uncertainty' (not to be confused
# with accuracy) of NIDEM elevations in m units for each contour. These values are subsequently interpolated to
# return an estimate of uncertainty for each individual pixel in the NIDEM datasets: larger values indicate the
# ITEM interval was produced from a composite of images with a larger range of tide heights.
# Compute uncertainties for each interval, and create a lookup dict to link uncertainties to each NIDEM contour
uncertainty_array = interval_uncertainty(polygon_id=polygon_id, item_polygon_path=item_polygon_path)
uncertainty_dict = dict(zip(contour_offsets, uncertainty_array))
####################
# Extract contours #
####################
# Here, we use `skimage.measure.find_contours` to extract contours along the boundary of each ITEM tidal interval
# (e.g. 0.5 is the boundary between ITEM interval 0 and interval 1; 5.5 is the boundary between interval 5 and
# interval 6). This function outputs a dictionary with ITEM interval boundaries as keys and lists of xy point
# arrays as values. Contours are also exported as a shapefile with elevation and uncertainty attributes in metres.
contour_dict = contour_extract(z_values=np.arange(0.5, 9.5, 1.0),
ds_array=item_array,
ds_crs='EPSG:3577',
ds_affine=geotrans,
output_shp=f'output_data/shapefile/nidem_contours/'
f'NIDEM_contours_{polygon_id}_{coord_str}.shp',
attribute_data={'elev_m': contour_offsets, 'uncert_m': uncertainty_array},
attribute_dtypes={'elev_m': 'float:9.2', 'uncert_m': 'float:9.2'})
#######################################################################
# Interpolate contours using TIN/Delaunay triangulation interpolation #
#######################################################################
# Here we assign each previously generated contour with its modelled height relative to MSL, producing a set of
# tidally tagged xyz points that can be used to interpolate elevations across the intertidal zone. We use the
# linear interpolation method from `scipy.interpolate.griddata`, which computes a TIN/Delaunay triangulation of
# the input data using Qhull before performing linear barycentric interpolation on each triangle.
# If contours include valid data, proceed with interpolation
try:
# Combine all individual contours for each contour height, and insert a height above MSL column into array
elev_contours = [np.insert(np.concatenate(v), 2, contour_offsets[i], axis=1) for i, v in
enumerate(contour_dict.values())]
# Combine all contour heights into a single array, and then extract xy points and z-values
all_contours = np.concatenate(elev_contours)
points_xy = all_contours[:, [1, 0]]
values_elev = all_contours[:, 2]
# Create a matching list of uncertainty values for each xy point
values_uncert = np.array([np.round(uncertainty_dict[i], 2) for i in values_elev])
# Calculate bounds of ITEM layer to create interpolation grid (from-to-by values in metre units)
grid_y, grid_x = np.mgrid[upleft_y:bottomright_y:1j * yrows, upleft_x:bottomright_x:1j * xcols]
# Interpolate between points onto grid. This uses the 'linear' method from
# scipy.interpolate.griddata, which computes a TIN/Delaunay triangulation of the input
# data with Qhull and performs linear barycentric interpolation on each triangle
print('Interpolating data for polygon {}'.format(polygon_id))
interp_elev_array = scipy.interpolate.griddata(points_xy, values_elev, (grid_y, grid_x), method='linear')
interp_uncert_array = scipy.interpolate.griddata(points_xy, values_uncert, (grid_y, grid_x), method='linear')
except ValueError:
# If contours contain no valid data, create empty arrays
interp_elev_array = np.full((yrows, xcols), -9999)
interp_uncert_array = np.full((yrows, xcols), -9999)
#########################################################
# Create ITEM confidence and elevation/bathymetry masks #
#########################################################
# The following code applies a range of masks to remove pixels where elevation values are likely to be invalid:
#
# 1. Non-coastal terrestrial pixels with elevations greater than 25 m above MSL. This mask is computed using
# SRTM-derived 1 Second Digital Elevation Model data (http://pid.geoscience.gov.au/dataset/ga/69769).
# 2. Sub-tidal pixels with bathymetry values deeper than -25 m below MSL. This mask is computed by identifying
# any pixels that are < -25 m in all of the national Australian Bathymetry and Topography Grid
# (http://pid.geoscience.gov.au/dataset/ga/67703), gbr30 High-resolution depth model for the Great
# Barrier Reef (http://pid.geoscience.gov.au/dataset/ga/115066) and nthaus30 High-resolution depth model
# for Northern Australia (http://pid.geoscience.gov.au/dataset/ga/121620).
# 3. Pixels with high ITEM confidence NDWI standard deviation (i.e. areas where inundation patterns are not driven
# by tidal influences). This mask is computed using ITEM v2.0 confidence layer data from DEA.
# Import ITEM confidence NDWI standard deviation array for polygon
conf_filename = glob.glob('{}/ITEM_STD_{}_*.tif'.format(item_conf_path, polygon_id))[0]
conf_ds = gdal.Open(conf_filename)
# Reproject SRTM-derived 1 Second DEM to cell size and projection of NIDEM
srtm30_reproj = reproject_to_template(input_raster=srtm30_raster,
template_raster=item_filename,
output_raster='scratch/temp.tif',
nodata_val=-9999)
# Reproject Australian Bathymetry and Topography Grid to cell size and projection of NIDEM
ausbath09_reproj = reproject_to_template(input_raster=ausbath09_raster,
template_raster=item_filename,
output_raster='scratch/temp.tif',
nodata_val=-9999)
# Reproject gbr30 bathymetry to cell size and projection of NIDEM
gbr30_reproj = reproject_to_template(input_raster=gbr30_raster,
template_raster=item_filename,
output_raster='scratch/temp.tif',
nodata_val=-9999)
# Reproject nthaus30 bathymetry to cell size and projection of NIDEM
nthaus30_reproj = reproject_to_template(input_raster=nthaus30_raster,
template_raster=item_filename,
output_raster='scratch/temp.tif',
nodata_val=-9999)
# Convert raster datasets to arrays
conf_array = conf_ds.GetRasterBand(1).ReadAsArray()
srtm30_array = srtm30_reproj.GetRasterBand(1).ReadAsArray()
ausbath09_array = ausbath09_reproj.GetRasterBand(1).ReadAsArray()
gbr30_array = gbr30_reproj.GetRasterBand(1).ReadAsArray()
nthaus30_array = nthaus30_reproj.GetRasterBand(1).ReadAsArray()
# Convert arrays to boolean masks:
# For elevation: any elevations > 25 m in SRTM 30m DEM
# For bathymetry: any depths < -25 m in GBR30 AND nthaus30 AND Ausbath09 bathymetry
# For ITEM confidence: any cells with NDWI STD > 0.25
elev_mask = srtm30_array > 25
bathy_mask = (ausbath09_array < -25) & (gbr30_array < -25) & (nthaus30_array < -25)
conf_mask = conf_array > 0.25
# Create a combined mask with -9999 nodata in unmasked areas and where:
# 1 = elevation mask
# 2 = bathymetry mask
# 3 = ITEM confidence mask
nidem_mask = np.full(item_array.shape, -9999)
nidem_mask[elev_mask] = 1
nidem_mask[bathy_mask] = 2
nidem_mask[conf_mask] = 3
################################
# Export output NIDEM geoTIFFs #
################################
# Because the lowest and highest ITEM intervals (0 and 9) cannot be correctly interpolated as they have no lower
# or upper bounds, the NIDEM layers are constrained to valid intertidal terrain (ITEM intervals 1-8).
nidem_uncertainty = np.where(valid_intertidal_extent, interp_uncert_array, -9999).astype(np.float32)
nidem_unfiltered = np.where(valid_intertidal_extent, interp_elev_array, -9999).astype(np.float32)
# NIDEM is exported as two DEMs: an unfiltered layer, and a layer that is filtered to remove terrestrial (> 25 m)
# and sub-tidal terrain (< -25 m) and pixels with high ITEM confidence NDWI standard deviation. Here we mask
# the unfiltered layer by NIDEM mask to produce a filtered NIDEM layer:
nidem_filtered = np.where(nidem_mask > 0, -9999, nidem_unfiltered).astype(np.float32)
# Export filtered NIDEM as a GeoTIFF
print(f'Exporting filtered NIDEM for polygon {polygon_id}')
array_to_geotiff(fname=f'output_data/geotiff/nidem/NIDEM_{polygon_id}_{coord_str}.tif',
data=nidem_filtered,
geo_transform=geotrans,
projection=prj,
nodata_val=-9999)
# Export unfiltered NIDEM as a GeoTIFF
print(f'Exporting unfiltered NIDEM for polygon {polygon_id}')
array_to_geotiff(fname=f'output_data/geotiff/nidem_unfiltered/NIDEM_unfiltered_{polygon_id}_{coord_str}.tif',
data=nidem_unfiltered,
geo_transform=geotrans,
projection=prj,
nodata_val=-9999)
# Export NIDEM uncertainty layer as a GeoTIFF
print(f'Exporting NIDEM uncertainty for polygon {polygon_id}')
array_to_geotiff(fname=f'output_data/geotiff/nidem_uncertainty/NIDEM_uncertainty_{polygon_id}_{coord_str}.tif',
data=nidem_uncertainty,
geo_transform=geotrans,
projection=prj,
nodata_val=-9999)
# Export NIDEM mask as a GeoTIFF
print(f'Exporting NIDEM mask for polygon {polygon_id}')
array_to_geotiff(fname=f'output_data/geotiff/nidem_mask/NIDEM_mask_{polygon_id}_{coord_str}.tif',
data=nidem_mask.astype(int),
geo_transform=geotrans,
projection=prj,
dtype=gdal.GDT_Int16,
nodata_val=-9999)
######################
# Export NetCDF data #
######################
# If netcdf file already exists, delete it
filename_netcdf = f'output_data/netcdf/NIDEM_{polygon_id}_{coord_str}.nc'
if os.path.exists(filename_netcdf):
os.remove(filename_netcdf)
# Compute coords
x_coords = netcdf_writer.netcdfy_coord(np.linspace(upleft_x + 12.5, bottomright_x - 12.5, num=xcols))
y_coords = netcdf_writer.netcdfy_coord(np.linspace(upleft_y - 12.5, bottomright_y + 12.5, num=yrows))
# Define output compression parameters
comp_params = dict(zlib=True, complevel=9, shuffle=True, fletcher32=True)
# Create new dataset
output_netcdf = create_netcdf_storage_unit(filename=filename_netcdf,
crs=CRS('EPSG:3577'),
coordinates={'x': Coordinate(x_coords, 'metres'),
'y': Coordinate(y_coords, 'metres')},
variables={'nidem': Variable(dtype=np.dtype('float32'),
nodata=-9999,
dims=('y', 'x'),
units='metres'),
'nidem_unfiltered': Variable(dtype=np.dtype('float32'),
nodata=-9999,
dims=('y', 'x'),
units='metres'),
'nidem_uncertainty': Variable(dtype=np.dtype('float32'),
nodata=-9999,
dims=('y', 'x'),
units='metres'),
'nidem_mask': Variable(dtype=np.dtype('int16'),
nodata=-9999,
dims=('y', 'x'),
units='1')},
variable_params={'nidem': comp_params,
'nidem_unfiltered': comp_params,
'nidem_uncertainty': comp_params,
'nidem_mask': comp_params})
# dem: assign data and set variable attributes
output_netcdf['nidem'][:] = netcdf_writer.netcdfy_data(nidem_filtered)
output_netcdf['nidem'].valid_range = [-25.0, 25.0]
output_netcdf['nidem'].standard_name = 'height_above_mean_sea_level'
output_netcdf['nidem'].coverage_content_type = 'modelResult'
output_netcdf['nidem'].long_name = 'National Intertidal Digital Elevation Model (NIDEM): elevation data in metre ' \
'units relative to mean sea level for each pixel of intertidal terrain across ' \
'the Australian coastline. Cleaned by masking out non-intertidal pixels' \
'and pixels where tidal processes poorly explain patterns of inundation.'
# dem_unfiltered: assign data and set variable attributes
output_netcdf['nidem_unfiltered'][:] = netcdf_writer.netcdfy_data(nidem_unfiltered)
output_netcdf['nidem_unfiltered'].standard_name = 'height_above_mean_sea_level'
output_netcdf['nidem_unfiltered'].coverage_content_type = 'modelResult'
output_netcdf['nidem_unfiltered'].long_name = 'NIDEM unfiltered: uncleaned elevation data in metre units ' \
'relative to mean sea level for each pixel of intertidal terrain ' \
'across the Australian coastline. Compared to the default NIDEM ' \
'product, these layers have not been filtered to remove noise, ' \
'artifacts or invalid elevation values.'
# uncertainty: assign data and set variable attributes
output_netcdf['nidem_uncertainty'][:] = netcdf_writer.netcdfy_data(nidem_uncertainty)
output_netcdf['nidem_uncertainty'].standard_name = 'height_above_mean_sea_level'
output_netcdf['nidem_uncertainty'].coverage_content_type = 'modelResult'
output_netcdf['nidem_uncertainty'].long_name = 'NIDEM uncertainty: provides a measure of the uncertainty (not ' \
'accuracy) of NIDEM elevations in metre units for each pixel. ' \
'Represents the standard deviation of tide heights of all Landsat ' \
'observations used to produce each ITEM 2.0 ten percent tidal ' \
'interval.'
# mask: assign data and set variable attributes
output_netcdf['nidem_mask'][:] = netcdf_writer.netcdfy_data(nidem_mask)
output_netcdf['nidem_mask'].valid_range = [1, 3]
output_netcdf['nidem_mask'].coverage_content_type = 'qualityInformation'
output_netcdf['nidem_mask'].long_name = 'NIDEM mask: flags non-intertidal terrestrial pixels with elevations ' \
'greater than 25 m (value = 1), sub-tidal pixels with depths greater ' \
'than -25 m (value = 2), and pixels where tidal processes poorly ' \
'explain patterns of inundation (value = 3).'
# Add global attributes
output_netcdf.title = 'National Intertidal Digital Elevation Model 25m 1.0.0'
output_netcdf.institution = 'Commonwealth of Australia (Geoscience Australia)'
output_netcdf.product_version = '1.0.0'
output_netcdf.license = 'CC BY Attribution 4.0 International License'
output_netcdf.time_coverage_start = '1986-01-01'
output_netcdf.time_coverage_end = '2016-12-31'
output_netcdf.cdm_data_type = 'Grid'
output_netcdf.contact = '*****@*****.**'
output_netcdf.publisher_email = '*****@*****.**'
output_netcdf.source = 'ITEM v2.0'
output_netcdf.keywords = 'Tidal, Topography, Landsat, Elevation, Intertidal, MSL, ITEM, NIDEM, DEM, Coastal'
output_netcdf.summary = "The National Intertidal Digital Elevation Model (NIDEM) product is a continental-scale " \
"dataset providing continuous elevation data for Australia's exposed intertidal zone. " \
"NIDEM provides the first three-dimensional representation of Australia's intertidal " \
"zone (excluding off-shore Territories and intertidal mangroves) at 25 m spatial " \
"resolution, addressing a key gap between the availability of sub-tidal bathymetry and " \
"terrestrial elevation data. NIDEM was generated by combining global tidal modelling " \
"with a 30-year time series archive of spatially and spectrally calibrated Landsat " \
"satellite data managed within the Digital Earth Australia (DEA) platform. NIDEM " \
"complements existing intertidal extent products, and provides data to support a new " \
"suite of use cases that require a more detailed understanding of the three-dimensional " \
"topography of the intertidal zone, such as hydrodynamic modelling, coastal risk " \
"management and ecological habitat mapping."
# Close dataset
output_netcdf.close()
def write_data(self, prod_name, measurement_name, tile_index, values):
self._output_file_handles[prod_name][measurement_name][
(0, ) + tile_index[1:]] = netcdf_writer.netcdfy_data(values)
self._output_file_handles[prod_name].sync()
_LOG.debug("Updated %s %s", measurement_name, tile_index[1:])
