def make_storage_unit(su, is_diskless=False, include_lineage=False):
"""convert search result into StorageUnit object
:param su: database index storage unit
:param is_diskless: Use a cached object for the source of data, rather than the file
:param include_lineage: Include an 'extra_metadata' variable containing detailed lineage information.
Note: This can cause the query to be slow for large datasets, as it is not lazy-loaded.
"""
crs = {
dim: su.descriptor['coordinates'][dim].get('units', None)
for dim in su.storage_type.dimensions
}
for dim in crs.keys():
if dim in su.storage_type.spatial_dimensions:
crs[dim] = su.storage_type.crs
coordinates = su.coordinates
variables = {
varname: Variable(dtype=numpy.dtype(attributes['dtype']),
nodata=attributes.get('nodata', None),
dimensions=su.storage_type.dimensions,
units=attributes.get('units', None))
for varname, attributes in su.storage_type.measurements.items()
}
if 'extra_metadata' not in variables.keys() and include_lineage:
variables['extra_metadata'] = Variable(numpy.dtype('S30000'), None,
('time', ), None)
attributes = {
'storage_type': su.storage_type,
'dataset_ids': su.dataset_ids
}
if is_diskless:
return make_in_memory_storage_unit(su,
coordinates=coordinates,
variables=variables,
attributes=attributes,
crs=crs)
if su.storage_type.driver == 'NetCDF CF':
return NetCDF4StorageUnit(su.local_path,
coordinates=coordinates,
variables=variables,
attributes=attributes,
crs=crs)
if su.storage_type.driver == 'GeoTiff':
result = GeoTifStorageUnit(su.local_path,
coordinates=coordinates,
variables=variables,
attributes=attributes)
time = datetime.datetime.strptime(su.descriptor['extents']['time_min'],
'%Y-%m-%dT%H:%M:%S.%f')
return StorageUnitDimensionProxy(result, time_coordinate_value(time))
raise RuntimeError('unsupported storage unit access driver %s' %
su.storage_type.driver)
def test_chunksizes(tmpnetcdf_filename):
nco = create_netcdf(tmpnetcdf_filename)
coord1 = create_coordinate(nco, 'greg', numpy.array([1.0, 2.0, 3.0]), 'cubic gregs')
coord2 = create_coordinate(nco, 'bleh', numpy.array([1.0, 2.0, 3.0, 4.0, 5.0]), 'metric blehs')
no_chunks = create_variable(nco, 'no_chunks', Variable(numpy.dtype('int16'), None, ('greg', 'bleh'), None))
min_max_chunks = create_variable(nco, 'min_max_chunks', Variable(numpy.dtype('int16'), None,
('greg', 'bleh'), None), chunksizes=[2, 50])
nco.close()
with netCDF4.Dataset(tmpnetcdf_filename) as nco:
assert nco['no_chunks'].chunking() == 'contiguous'
assert nco['min_max_chunks'].chunking() == [2, 5]
def make_sample_netcdf(tmpdir):
"""Make a test Geospatial NetCDF file, 4000x4000 int16 random data, in a variable named `sample`.
Return the GDAL access string."""
sample_nc = str(tmpdir.mkdir('netcdfs').join('sample.nc'))
geobox = GeoBox(4000,
4000,
affine=Affine(25.0, 0.0, 1200000, 0.0, -25.0, -4200000),
crs=CRS('EPSG:3577'))
sample_data = np.random.randint(10000, size=(4000, 4000), dtype=np.int16)
variables = {
'sample':
Variable(sample_data.dtype,
nodata=-999,
dims=geobox.dimensions,
units=1)
}
nco = create_netcdf_storage_unit(sample_nc,
geobox.crs,
geobox.coordinates,
variables=variables,
variable_params={})
nco['sample'][:] = sample_data
nco.close()
return "NetCDF:%s:sample" % sample_nc, geobox, sample_data
def from_file(cls, file_path):
coordinates = {}
variables = {}
grid_mappings = {}
standard_names = {}
with _GLOBAL_LOCK, contextlib.closing(_open_dataset(file_path)) as ncds:
attributes = {k: getattr(ncds, k) for k in ncds.ncattrs()}
for name, var in ncds.variables.items():
dims = var.dimensions
units = getattr(var, 'units', None)
if hasattr(var, 'grid_mapping_name') and hasattr(var, 'spatial_ref'):
grid_mappings[getattr(var, 'grid_mapping_name', None)] = getattr(var, 'spatial_ref', None)
elif len(dims) == 1 and name == dims[0]:
coordinates[name] = Coordinate(dtype=numpy.dtype(var.dtype),
begin=var[0].item(), end=var[var.size-1].item(),
length=var.shape[0], units=units)
standard_name = getattr(var, 'standard_name', None)
if standard_name:
standard_names[standard_name] = name
else:
dims, dtype = _get_dims_and_dtype(var)
ndv = (getattr(var, '_FillValue', None) or
getattr(var, 'missing_value', None) or
getattr(var, 'fill_value', None))
ndv = ndv.item() if ndv else None
variables[name] = Variable(dtype, ndv, dims, units)
crs = _make_crs(grid_mappings, standard_names)
return cls(file_path, variables=variables, coordinates=coordinates, attributes=attributes, crs=crs)
def make_fake_netcdf_dataset(nc_name, yaml_doc):
from datacube.model import Variable
from datacube.storage.netcdf_writer import create_variable, netcdfy_data
from netCDF4 import Dataset
import numpy as np
content = yaml_doc.read_text()
npdata = np.array([content], dtype=bytes)
with Dataset(nc_name, 'w') as nco:
var = Variable(npdata.dtype, None, ('time', ), None)
nco.createDimension('time', size=1)
create_variable(nco, 'dataset', var)
nco['dataset'][:] = netcdfy_data(npdata)
def test_chunksizes(tmpnetcdf_filename):
nco = create_netcdf(tmpnetcdf_filename)
x = numpy.arange(3, dtype='float32')
y = numpy.arange(5, dtype='float32')
coord1 = create_coordinate(nco, 'x', x, 'm')
coord2 = create_coordinate(nco, 'y', y, 'm')
assert coord1 is not None and coord2 is not None
no_chunks = create_variable(
nco, 'no_chunks', Variable(numpy.dtype('int16'), None, ('x', 'y'),
None))
min_max_chunks = create_variable(nco,
'min_max_chunks',
Variable(numpy.dtype('int16'), None,
('x', 'y'), None),
chunksizes=(2, 50))
assert no_chunks is not None
assert min_max_chunks is not None
strings = numpy.array(["AAa", 'bbb', 'CcC'], dtype='S')
strings = xr.DataArray(strings, dims=['x'], coords={'x': x})
create_variable(nco, 'strings_unchunked', strings)
create_variable(nco, 'strings_chunked', strings, chunksizes=(1, ))
nco.close()
with netCDF4.Dataset(tmpnetcdf_filename) as nco:
assert nco['no_chunks'].chunking() == 'contiguous'
assert nco['min_max_chunks'].chunking() == [2, 5]
assert nco['strings_unchunked'].chunking() == 'contiguous'
assert nco['strings_chunked'].chunking() == [1, 3]
def test_create_string_variable(tmpnetcdf_filename):
nco = create_netcdf(tmpnetcdf_filename)
coord = create_coordinate(nco, 'greg', numpy.array([1.0, 3.0, 9.0]),
'cubic gregs')
dtype = numpy.dtype('S100')
data = numpy.array(["test-str1", "test-str2", "test-str3"], dtype=dtype)
var = create_variable(nco, 'str_var',
Variable(dtype, None, ('greg', ), None))
var[:] = netcdfy_data(data)
nco.close()
with netCDF4.Dataset(tmpnetcdf_filename) as nco:
assert 'str_var' in nco.variables
assert netCDF4.chartostring(nco['str_var'][0]) == data[0]
def saveNC(output,filename, history):
nco=netcdf_writer.create_netcdf(filename)
nco.history = (history.decode('utf-8').encode('ascii','replace'))
coords=output.coords
cnames=()
for x in coords:
netcdf_writer.create_coordinate(nco, x, coords[x].values, coords[x].units)
cnames=cnames+(x,)
netcdf_writer.create_grid_mapping_variable(nco, output.crs)
for band in output.data_vars:
output.data_vars[band].values[np.isnan(output.data_vars[band].values)]=nodata
var= netcdf_writer.create_variable(nco, band, Variable(output.data_vars[band].dtype, nodata, cnames, None) ,set_crs=True)
var[:] = netcdf_writer.netcdfy_data(output.data_vars[band].values)
nco.close()
def nco_from_sources(sources, geobox, measurements, variable_params, filename):
coordinates = {
name: Coordinate(coord.values, coord.units)
for name, coord in sources.coords.items()
}
coordinates.update(geobox.coordinates)
variables = {
variable['name']: Variable(dtype=numpy.dtype(variable['dtype']),
nodata=variable['nodata'],
dims=sources.dims + geobox.dimensions,
units=variable['units'])
for variable in measurements
}
return create_netcdf_storage_unit(filename, geobox.crs, coordinates,
variables, variable_params)
def test_create_string_variable(tmpdir, s1, s2, s3):
tmpnetcdf_filename = get_tmpnetcdf_filename(tmpdir)
str_var = 'str_var'
nco = create_netcdf(tmpnetcdf_filename)
coord = create_coordinate(nco, 'greg', numpy.array([1.0, 3.0, 9.0]), 'cubic gregs')
dtype = numpy.dtype('S100')
data = numpy.array([s1, s2, s3], dtype=dtype)
var = create_variable(nco, str_var, Variable(dtype, None, ('greg',), None))
var[:] = netcdfy_data(data)
nco.close()
with netCDF4.Dataset(tmpnetcdf_filename) as nco:
assert str_var in nco.variables
for returned, expected in zip(read_strings_from_netcdf(tmpnetcdf_filename, variable=str_var), (s1, s2, s3)):
assert returned == expected
def write_dataset_to_netcdf(access_unit, global_attributes, variable_params,
filename):
if filename.exists():
raise RuntimeError('Storage Unit already exists: %s' % filename)
try:
filename.parent.mkdir(parents=True)
except OSError:
pass
# _LOG.info("Writing storage unit: %s", filename)
nco = netcdf_writer.create_netcdf(str(filename))
for name, coord in access_unit.coords.items():
netcdf_writer.create_coordinate(nco, name, coord.values, coord.units)
netcdf_writer.create_grid_mapping_variable(nco, access_unit.crs)
for name, variable in access_unit.data_vars.items():
# Create variable
var_params = variable_params.get(name, {})
data_var = netcdf_writer.create_variable(
nco, name,
Variable(variable.dtype, getattr(variable, 'nodata',
None), variable.dims,
getattr(variable, 'units', '1')), **var_params)
# Write data
data_var[:] = netcdf_writer.netcdfy_data(variable.values)
# TODO: 'flags_definition', 'spectral_definition'?
for key, value in variable_params.get(name, {}).get('attrs',
{}).items():
setattr(data_var, key, value)
# write global atrributes
for key, value in global_attributes.items():
setattr(nco, key, value)
nco.close()
def _nco_from_sources(self, sources, geobox, measurements, variable_params,
filename):
coordinates = OrderedDict(
(name, geometry.Coordinate(coord.values, coord.units))
for name, coord in sources.coords.items())
coordinates.update(geobox.coordinates)
variables = OrderedDict(
(variable['name'],
Variable(dtype=numpy.dtype(variable['dtype']),
nodata=variable['nodata'],
dims=sources.dims + geobox.dimensions,
units=variable['units'])) for variable in measurements)
return create_netcdf_storage_unit(
filename,
crs=geobox.crs,
coordinates=coordinates,
variables=variables,
variable_params=variable_params,
global_attributes=self.global_attributes)
def compute_and_write(self):
"""
Computes the wofs confidence and filtered summary bands and write to the
corresponding NetCDF file. The file template and location etc are read from the configs.
"""
geo_box = self.grid_spec.tile_geobox(self.tile_index)
# Compute metadata
env = self.cfg.get_env_of_product('wofs_filtered_summary')
with Datacube(app='wofs-confidence', env=env) as dc:
product = dc.index.products.get_by_name('wofs_filtered_summary')
extent = self.grid_spec.tile_geobox(self.tile_index).extent
center_time = datetime.now()
uri = self.get_filtered_uri()
dts = make_dataset(product=product,
sources=self.factor_sources,
extent=extent,
center_time=center_time,
uri=uri)
metadata = yaml.dump(dts.metadata_doc,
Dumper=SafeDumper,
encoding='utf-8')
# Compute dataset coords
coords = dict()
coords['time'] = Coordinate(
netcdf_writer.netcdfy_coord(
np.array([datetime_to_seconds_since_1970(center_time)])),
['seconds since 1970-01-01 00:00:00'])
for dim in geo_box.dimensions:
coords[dim] = Coordinate(
netcdf_writer.netcdfy_coord(geo_box.coordinates[dim].values),
geo_box.coordinates[dim].units)
# Compute dataset variables
spatial_var = Variable(dtype=np.dtype(DEFAULT_TYPE),
nodata=DEFAULT_FLOAT_NODATA,
dims=('time', ) + geo_box.dimensions,
units=('seconds since 1970-01-01 00:00:00', ) +
geo_box.crs.units)
band1 = self.cfg.cfg['wofs_filtered_summary']['confidence']
band2 = self.cfg.cfg['wofs_filtered_summary']['confidence_filtered']
vars = {band1: spatial_var, band2: spatial_var}
vars_params = {band1: {}, band2: {}}
global_atts = self.cfg.cfg['global_attributes']
# Get crs string
crs = self.cfg.cfg['storage']['crs'] if self.cfg.cfg['storage'].get(
'crs') else DEFAULT_CRS
# Create a dataset container
filename = self.get_filtered_uri()
logger.info('creating', file=filename.name)
netcdf_unit = create_netcdf_storage_unit(filename=filename,
crs=CRS(crs),
coordinates=coords,
variables=vars,
global_attributes=global_atts,
variable_params=vars_params)
# Confidence layer: Fill variable data and set attributes
confidence = self.compute_confidence()
netcdf_unit[band1][:] = netcdf_writer.netcdfy_data(confidence)
netcdf_unit[band1].units = '1'
netcdf_unit[band1].valid_range = [0, 1.0]
netcdf_unit[band1].coverage_content_type = 'modelResult'
netcdf_unit[band1].long_name = \
'Wofs Confidence Layer predicted by {}'.format(self.confidence_model.factors.__str__())
# Confidence filtered wofs-stats frequency layer: Fill variable data and set attributes
confidence_filtered = self.compute_confidence_filtered()
netcdf_unit[band2][:] = netcdf_writer.netcdfy_data(confidence_filtered)
netcdf_unit[band2].units = '1'
netcdf_unit[band2].valid_range = [0, 1.0]
netcdf_unit[band2].coverage_content_type = 'modelResult'
netcdf_unit[
band2].long_name = 'WOfS-Stats frequency confidence filtered layer'
# Metadata
dataset_data = DataArray(data=[metadata], dims=('time', ))
netcdf_writer.create_variable(netcdf_unit,
'dataset',
dataset_data,
zlib=True)
netcdf_unit['dataset'][:] = netcdf_writer.netcdfy_data(
dataset_data.values)
netcdf_unit.close()
logger.info('completed', file=filename.name)
def main(argv=None):
if argv is None:
argv = sys.argv
print(sys.argv)
# If no user arguments provided
if len(argv) < 2:
str_usage = "You must specify a polygon ID"
print(str_usage)
sys.exit()
# Set ITEM polygon for analysis
polygon_id = int(argv[1]) # polygon_id = 33
# Import configuration details from NIDEM_configuration.ini
config = configparser.ConfigParser()
config.read('NIDEM_configuration.ini')
# Set paths to ITEM relative, confidence and offset products
item_offset_path = config['ITEM inputs']['item_offset_path']
item_relative_path = config['ITEM inputs']['item_relative_path']
item_conf_path = config['ITEM inputs']['item_conf_path']
item_polygon_path = config['ITEM inputs']['item_polygon_path']
# Set paths to elevation, bathymetry and shapefile datasets used to create NIDEM mask
srtm30_raster = config['Masking inputs']['srtm30_raster']
ausbath09_raster = config['Masking inputs']['ausbath09_raster']
gbr30_raster = config['Masking inputs']['gbr30_raster']
nthaus30_raster = config['Masking inputs']['nthaus30_raster']
# Print run details
print('Processing polygon {} from {}'.format(polygon_id, item_offset_path))
##################################
# Import and prepare ITEM raster #
##################################
# Contours generated by `skimage.measure.find_contours` stop before the edge of nodata pixels. To prevent gaps
# from occurring between adjacent NIDEM tiles, the following steps 'fill' pixels directly on the boundary of
# two NIDEM tiles with the value of the nearest pixel with data.
# Import raster
item_filename = glob.glob('{}/ITEM_REL_{}_*.tif'.format(item_relative_path, polygon_id))[0]
item_ds = gdal.Open(item_filename)
item_array = item_ds.GetRasterBand(1).ReadAsArray()
# Get coord string of polygon from ITEM array name to use for output names
coord_str = item_filename[-17:-4]
# Extract shape, projection info and geotransform data
yrows, xcols = item_array.shape
prj = item_ds.GetProjection()
geotrans = item_ds.GetGeoTransform()
upleft_x, x_size, x_rotation, upleft_y, y_rotation, y_size = geotrans
bottomright_x = upleft_x + (x_size * xcols)
bottomright_y = upleft_y + (y_size * yrows)
# Identify valid intertidal area by selecting pixels between the lowest and highest ITEM intervals. This is
# subsequently used to restrict the extent of interpolated elevation data to match the input ITEM polygons.
valid_intertidal_extent = np.where((item_array > 0) & (item_array < 9), 1, 0)
# Convert datatype to float to allow assigning nodata -6666 values to NaN
item_array = item_array.astype('float32')
item_array[item_array == -6666] = np.nan
# First, identify areas to be filled by dilating non-NaN pixels by two pixels (i.e. ensuring vertical, horizontal
# and diagonally adjacent pixels are filled):
dilated_mask = nd.morphology.binary_dilation(~np.isnan(item_array), iterations=2)
# For every pixel, identify the indices of the nearest pixel with data (i.e. data pixels will return their own
# indices; nodata pixels will return the indices of the nearest data pixel). This output can be used to index
# back into the original array, returning a new array where data pixels remain the same, but every nodata pixel
# is filled with the value of the nearest data pixel:
nearest_inds = nd.distance_transform_edt(input=np.isnan(item_array), return_distances=False, return_indices=True)
item_array = item_array[tuple(nearest_inds)]
# As we only want to fill pixels on the boundary of NIDEM tiles, set pixels outside the dilated area back to NaN:
item_array[~dilated_mask] = np.nan
##########################################
# Median and SD tide height per interval #
##########################################
# Each ITEM v2.0 tidal interval boundary was produced from a composite of multiple Landsat images that cover a
# range of tidal heights. To obtain an elevation relative to modelled mean sea level for each interval boundary,
# we import a precomputed file containing the median tidal height for all Landsat images that were used to
# generate the interval (Sagar et al. 2017, https://doi.org/10.1016/j.rse.2017.04.009).
# Import ITEM offset values for each ITEM tidal interval, dividing by 1000 to give metre units
item_offsets = np.loadtxt('{}/elevation.txt'.format(item_offset_path), delimiter=',', dtype='str')
item_offsets = {int(key): [float(val) / 1000.0 for val in value.split(' ')] for (key, value) in item_offsets}
contour_offsets = item_offsets[polygon_id]
# The range of tide heights used to compute the above median tide height can vary significantly between tidal
# modelling polygons. To quantify this range, we take the standard deviation of tide heights for all Landsat
# images used to produce each ITEM interval. This represents a measure of the 'uncertainty' (not to be confused
# with accuracy) of NIDEM elevations in m units for each contour. These values are subsequently interpolated to
# return an estimate of uncertainty for each individual pixel in the NIDEM datasets: larger values indicate the
# ITEM interval was produced from a composite of images with a larger range of tide heights.
# Compute uncertainties for each interval, and create a lookup dict to link uncertainties to each NIDEM contour
uncertainty_array = interval_uncertainty(polygon_id=polygon_id, item_polygon_path=item_polygon_path)
uncertainty_dict = dict(zip(contour_offsets, uncertainty_array))
####################
# Extract contours #
####################
# Here, we use `skimage.measure.find_contours` to extract contours along the boundary of each ITEM tidal interval
# (e.g. 0.5 is the boundary between ITEM interval 0 and interval 1; 5.5 is the boundary between interval 5 and
# interval 6). This function outputs a dictionary with ITEM interval boundaries as keys and lists of xy point
# arrays as values. Contours are also exported as a shapefile with elevation and uncertainty attributes in metres.
contour_dict = contour_extract(z_values=np.arange(0.5, 9.5, 1.0),
ds_array=item_array,
ds_crs='EPSG:3577',
ds_affine=geotrans,
output_shp=f'output_data/shapefile/nidem_contours/'
f'NIDEM_contours_{polygon_id}_{coord_str}.shp',
attribute_data={'elev_m': contour_offsets, 'uncert_m': uncertainty_array},
attribute_dtypes={'elev_m': 'float:9.2', 'uncert_m': 'float:9.2'})
#######################################################################
# Interpolate contours using TIN/Delaunay triangulation interpolation #
#######################################################################
# Here we assign each previously generated contour with its modelled height relative to MSL, producing a set of
# tidally tagged xyz points that can be used to interpolate elevations across the intertidal zone. We use the
# linear interpolation method from `scipy.interpolate.griddata`, which computes a TIN/Delaunay triangulation of
# the input data using Qhull before performing linear barycentric interpolation on each triangle.
# If contours include valid data, proceed with interpolation
try:
# Combine all individual contours for each contour height, and insert a height above MSL column into array
elev_contours = [np.insert(np.concatenate(v), 2, contour_offsets[i], axis=1) for i, v in
enumerate(contour_dict.values())]
# Combine all contour heights into a single array, and then extract xy points and z-values
all_contours = np.concatenate(elev_contours)
points_xy = all_contours[:, [1, 0]]
values_elev = all_contours[:, 2]
# Create a matching list of uncertainty values for each xy point
values_uncert = np.array([np.round(uncertainty_dict[i], 2) for i in values_elev])
# Calculate bounds of ITEM layer to create interpolation grid (from-to-by values in metre units)
grid_y, grid_x = np.mgrid[upleft_y:bottomright_y:1j * yrows, upleft_x:bottomright_x:1j * xcols]
# Interpolate between points onto grid. This uses the 'linear' method from
# scipy.interpolate.griddata, which computes a TIN/Delaunay triangulation of the input
# data with Qhull and performs linear barycentric interpolation on each triangle
print('Interpolating data for polygon {}'.format(polygon_id))
interp_elev_array = scipy.interpolate.griddata(points_xy, values_elev, (grid_y, grid_x), method='linear')
interp_uncert_array = scipy.interpolate.griddata(points_xy, values_uncert, (grid_y, grid_x), method='linear')
except ValueError:
# If contours contain no valid data, create empty arrays
interp_elev_array = np.full((yrows, xcols), -9999)
interp_uncert_array = np.full((yrows, xcols), -9999)
#########################################################
# Create ITEM confidence and elevation/bathymetry masks #
#########################################################
# The following code applies a range of masks to remove pixels where elevation values are likely to be invalid:
#
# 1. Non-coastal terrestrial pixels with elevations greater than 25 m above MSL. This mask is computed using
# SRTM-derived 1 Second Digital Elevation Model data (http://pid.geoscience.gov.au/dataset/ga/69769).
# 2. Sub-tidal pixels with bathymetry values deeper than -25 m below MSL. This mask is computed by identifying
# any pixels that are < -25 m in all of the national Australian Bathymetry and Topography Grid
# (http://pid.geoscience.gov.au/dataset/ga/67703), gbr30 High-resolution depth model for the Great
# Barrier Reef (http://pid.geoscience.gov.au/dataset/ga/115066) and nthaus30 High-resolution depth model
# for Northern Australia (http://pid.geoscience.gov.au/dataset/ga/121620).
# 3. Pixels with high ITEM confidence NDWI standard deviation (i.e. areas where inundation patterns are not driven
# by tidal influences). This mask is computed using ITEM v2.0 confidence layer data from DEA.
# Import ITEM confidence NDWI standard deviation array for polygon
conf_filename = glob.glob('{}/ITEM_STD_{}_*.tif'.format(item_conf_path, polygon_id))[0]
conf_ds = gdal.Open(conf_filename)
# Reproject SRTM-derived 1 Second DEM to cell size and projection of NIDEM
srtm30_reproj = reproject_to_template(input_raster=srtm30_raster,
template_raster=item_filename,
output_raster='scratch/temp.tif',
nodata_val=-9999)
# Reproject Australian Bathymetry and Topography Grid to cell size and projection of NIDEM
ausbath09_reproj = reproject_to_template(input_raster=ausbath09_raster,
template_raster=item_filename,
output_raster='scratch/temp.tif',
nodata_val=-9999)
# Reproject gbr30 bathymetry to cell size and projection of NIDEM
gbr30_reproj = reproject_to_template(input_raster=gbr30_raster,
template_raster=item_filename,
output_raster='scratch/temp.tif',
nodata_val=-9999)
# Reproject nthaus30 bathymetry to cell size and projection of NIDEM
nthaus30_reproj = reproject_to_template(input_raster=nthaus30_raster,
template_raster=item_filename,
output_raster='scratch/temp.tif',
nodata_val=-9999)
# Convert raster datasets to arrays
conf_array = conf_ds.GetRasterBand(1).ReadAsArray()
srtm30_array = srtm30_reproj.GetRasterBand(1).ReadAsArray()
ausbath09_array = ausbath09_reproj.GetRasterBand(1).ReadAsArray()
gbr30_array = gbr30_reproj.GetRasterBand(1).ReadAsArray()
nthaus30_array = nthaus30_reproj.GetRasterBand(1).ReadAsArray()
# Convert arrays to boolean masks:
# For elevation: any elevations > 25 m in SRTM 30m DEM
# For bathymetry: any depths < -25 m in GBR30 AND nthaus30 AND Ausbath09 bathymetry
# For ITEM confidence: any cells with NDWI STD > 0.25
elev_mask = srtm30_array > 25
bathy_mask = (ausbath09_array < -25) & (gbr30_array < -25) & (nthaus30_array < -25)
conf_mask = conf_array > 0.25
# Create a combined mask with -9999 nodata in unmasked areas and where:
# 1 = elevation mask
# 2 = bathymetry mask
# 3 = ITEM confidence mask
nidem_mask = np.full(item_array.shape, -9999)
nidem_mask[elev_mask] = 1
nidem_mask[bathy_mask] = 2
nidem_mask[conf_mask] = 3
################################
# Export output NIDEM geoTIFFs #
################################
# Because the lowest and highest ITEM intervals (0 and 9) cannot be correctly interpolated as they have no lower
# or upper bounds, the NIDEM layers are constrained to valid intertidal terrain (ITEM intervals 1-8).
nidem_uncertainty = np.where(valid_intertidal_extent, interp_uncert_array, -9999).astype(np.float32)
nidem_unfiltered = np.where(valid_intertidal_extent, interp_elev_array, -9999).astype(np.float32)
# NIDEM is exported as two DEMs: an unfiltered layer, and a layer that is filtered to remove terrestrial (> 25 m)
# and sub-tidal terrain (< -25 m) and pixels with high ITEM confidence NDWI standard deviation. Here we mask
# the unfiltered layer by NIDEM mask to produce a filtered NIDEM layer:
nidem_filtered = np.where(nidem_mask > 0, -9999, nidem_unfiltered).astype(np.float32)
# Export filtered NIDEM as a GeoTIFF
print(f'Exporting filtered NIDEM for polygon {polygon_id}')
array_to_geotiff(fname=f'output_data/geotiff/nidem/NIDEM_{polygon_id}_{coord_str}.tif',
data=nidem_filtered,
geo_transform=geotrans,
projection=prj,
nodata_val=-9999)
# Export unfiltered NIDEM as a GeoTIFF
print(f'Exporting unfiltered NIDEM for polygon {polygon_id}')
array_to_geotiff(fname=f'output_data/geotiff/nidem_unfiltered/NIDEM_unfiltered_{polygon_id}_{coord_str}.tif',
data=nidem_unfiltered,
geo_transform=geotrans,
projection=prj,
nodata_val=-9999)
# Export NIDEM uncertainty layer as a GeoTIFF
print(f'Exporting NIDEM uncertainty for polygon {polygon_id}')
array_to_geotiff(fname=f'output_data/geotiff/nidem_uncertainty/NIDEM_uncertainty_{polygon_id}_{coord_str}.tif',
data=nidem_uncertainty,
geo_transform=geotrans,
projection=prj,
nodata_val=-9999)
# Export NIDEM mask as a GeoTIFF
print(f'Exporting NIDEM mask for polygon {polygon_id}')
array_to_geotiff(fname=f'output_data/geotiff/nidem_mask/NIDEM_mask_{polygon_id}_{coord_str}.tif',
data=nidem_mask.astype(int),
geo_transform=geotrans,
projection=prj,
dtype=gdal.GDT_Int16,
nodata_val=-9999)
######################
# Export NetCDF data #
######################
# If netcdf file already exists, delete it
filename_netcdf = f'output_data/netcdf/NIDEM_{polygon_id}_{coord_str}.nc'
if os.path.exists(filename_netcdf):
os.remove(filename_netcdf)
# Compute coords
x_coords = netcdf_writer.netcdfy_coord(np.linspace(upleft_x + 12.5, bottomright_x - 12.5, num=xcols))
y_coords = netcdf_writer.netcdfy_coord(np.linspace(upleft_y - 12.5, bottomright_y + 12.5, num=yrows))
# Define output compression parameters
comp_params = dict(zlib=True, complevel=9, shuffle=True, fletcher32=True)
# Create new dataset
output_netcdf = create_netcdf_storage_unit(filename=filename_netcdf,
crs=CRS('EPSG:3577'),
coordinates={'x': Coordinate(x_coords, 'metres'),
'y': Coordinate(y_coords, 'metres')},
variables={'nidem': Variable(dtype=np.dtype('float32'),
nodata=-9999,
dims=('y', 'x'),
units='metres'),
'nidem_unfiltered': Variable(dtype=np.dtype('float32'),
nodata=-9999,
dims=('y', 'x'),
units='metres'),
'nidem_uncertainty': Variable(dtype=np.dtype('float32'),
nodata=-9999,
dims=('y', 'x'),
units='metres'),
'nidem_mask': Variable(dtype=np.dtype('int16'),
nodata=-9999,
dims=('y', 'x'),
units='1')},
variable_params={'nidem': comp_params,
'nidem_unfiltered': comp_params,
'nidem_uncertainty': comp_params,
'nidem_mask': comp_params})
# dem: assign data and set variable attributes
output_netcdf['nidem'][:] = netcdf_writer.netcdfy_data(nidem_filtered)
output_netcdf['nidem'].valid_range = [-25.0, 25.0]
output_netcdf['nidem'].standard_name = 'height_above_mean_sea_level'
output_netcdf['nidem'].coverage_content_type = 'modelResult'
output_netcdf['nidem'].long_name = 'National Intertidal Digital Elevation Model (NIDEM): elevation data in metre ' \
'units relative to mean sea level for each pixel of intertidal terrain across ' \
'the Australian coastline. Cleaned by masking out non-intertidal pixels' \
'and pixels where tidal processes poorly explain patterns of inundation.'
# dem_unfiltered: assign data and set variable attributes
output_netcdf['nidem_unfiltered'][:] = netcdf_writer.netcdfy_data(nidem_unfiltered)
output_netcdf['nidem_unfiltered'].standard_name = 'height_above_mean_sea_level'
output_netcdf['nidem_unfiltered'].coverage_content_type = 'modelResult'
output_netcdf['nidem_unfiltered'].long_name = 'NIDEM unfiltered: uncleaned elevation data in metre units ' \
'relative to mean sea level for each pixel of intertidal terrain ' \
'across the Australian coastline. Compared to the default NIDEM ' \
'product, these layers have not been filtered to remove noise, ' \
'artifacts or invalid elevation values.'
# uncertainty: assign data and set variable attributes
output_netcdf['nidem_uncertainty'][:] = netcdf_writer.netcdfy_data(nidem_uncertainty)
output_netcdf['nidem_uncertainty'].standard_name = 'height_above_mean_sea_level'
output_netcdf['nidem_uncertainty'].coverage_content_type = 'modelResult'
output_netcdf['nidem_uncertainty'].long_name = 'NIDEM uncertainty: provides a measure of the uncertainty (not ' \
'accuracy) of NIDEM elevations in metre units for each pixel. ' \
'Represents the standard deviation of tide heights of all Landsat ' \
'observations used to produce each ITEM 2.0 ten percent tidal ' \
'interval.'
# mask: assign data and set variable attributes
output_netcdf['nidem_mask'][:] = netcdf_writer.netcdfy_data(nidem_mask)
output_netcdf['nidem_mask'].valid_range = [1, 3]
output_netcdf['nidem_mask'].coverage_content_type = 'qualityInformation'
output_netcdf['nidem_mask'].long_name = 'NIDEM mask: flags non-intertidal terrestrial pixels with elevations ' \
'greater than 25 m (value = 1), sub-tidal pixels with depths greater ' \
'than -25 m (value = 2), and pixels where tidal processes poorly ' \
'explain patterns of inundation (value = 3).'
# Add global attributes
output_netcdf.title = 'National Intertidal Digital Elevation Model 25m 1.0.0'
output_netcdf.institution = 'Commonwealth of Australia (Geoscience Australia)'
output_netcdf.product_version = '1.0.0'
output_netcdf.license = 'CC BY Attribution 4.0 International License'
output_netcdf.time_coverage_start = '1986-01-01'
output_netcdf.time_coverage_end = '2016-12-31'
output_netcdf.cdm_data_type = 'Grid'
output_netcdf.contact = '*****@*****.**'
output_netcdf.publisher_email = '*****@*****.**'
output_netcdf.source = 'ITEM v2.0'
output_netcdf.keywords = 'Tidal, Topography, Landsat, Elevation, Intertidal, MSL, ITEM, NIDEM, DEM, Coastal'
output_netcdf.summary = "The National Intertidal Digital Elevation Model (NIDEM) product is a continental-scale " \
"dataset providing continuous elevation data for Australia's exposed intertidal zone. " \
"NIDEM provides the first three-dimensional representation of Australia's intertidal " \
"zone (excluding off-shore Territories and intertidal mangroves) at 25 m spatial " \
"resolution, addressing a key gap between the availability of sub-tidal bathymetry and " \
"terrestrial elevation data. NIDEM was generated by combining global tidal modelling " \
"with a 30-year time series archive of spatially and spectrally calibrated Landsat " \
"satellite data managed within the Digital Earth Australia (DEA) platform. NIDEM " \
"complements existing intertidal extent products, and provides data to support a new " \
"suite of use cases that require a more detailed understanding of the three-dimensional " \
"topography of the intertidal zone, such as hydrodynamic modelling, coastal risk " \
"management and ecological habitat mapping."
# Close dataset
output_netcdf.close()
def band2var(i):
return Variable(dataset.dtypes[i], dataset.nodatavals[i],
('y', 'x'), '1')
def expand_var(var):
return Variable(var.dtype, var.nodata,
self._dimensions + var.dimensions, var.units)
