def get_bandnumber(self, src):
# If `band` property is set to an integer it overrides any other logic
band = self._measurement.get('band')
if band is not None:
if isinstance(band, integer_types):
return band
else:
_LOG.warning('Expected "band" property to be of integer type')
if 'netcdf' not in self._dataset.format.lower():
layer_id = self._measurement.get('layer', 1)
return layer_id if isinstance(layer_id, integer_types) else 1
tag_name = GDAL_NETCDF_DIM + 'time'
if tag_name not in src.tags(
1): # TODO: support time-less datasets properly
return 1
time = self._dataset.center_time
sec_since_1970 = datetime_to_seconds_since_1970(time)
idx = 0
dist = float('+inf')
for i in range(1, src.count + 1):
v = float(src.tags(i)[tag_name])
if abs(sec_since_1970 - v) < dist:
idx = i
dist = abs(sec_since_1970 - v)
return idx
def wheres_my_band(self, src, time):
sec_since_1970 = datetime_to_seconds_since_1970(time)
idx = 0
dist = float('+inf')
for i in range(1, src.count + 1):
v = float(src.tags(i)['NETCDF_DIM_time'])
if abs(sec_since_1970 - v) < dist:
idx = i
return idx
def test_rd_internals_bidx(data_folder):
base = "file://" + str(data_folder) + "/metadata.yml"
bi = mk_band('a',
base,
path="multi_doc.nc",
format=NetCDF,
timestamp=datetime.utcfromtimestamp(1),
layer='a')
assert bi.uri.endswith('multi_doc.nc')
assert datetime_to_seconds_since_1970(bi.center_time) == 1
rio_fname = _rio_uri(bi)
assert rio_fname.startswith('NETCDF:')
with rasterio.open(rio_fname) as src:
# timestamp search
bidx = _rio_band_idx(bi, src)
assert bidx == 2
# extract from .uri
bi.uri = bi.uri + "#part=5"
assert _rio_band_idx(bi, src) == 5
# extract from .band
bi.band = 33
assert _rio_band_idx(bi, src) == 33
bi = mk_band('a', base, path="test.tif", format=GeoTIFF)
with rasterio.open(_rio_uri(bi), 'r') as src:
# should default to 1
assert _rio_band_idx(bi, src) == 1
# layer containing int should become index
bi = mk_band('a', base, path="test.tif", format=GeoTIFF, layer=2)
assert _rio_band_idx(bi, src) == 2
# band is the keyword
bi = mk_band('a', base, path="test.tif", format=GeoTIFF, band=3)
assert _rio_band_idx(bi, src) == 3
# Pretend it's netcdf to test missing time dimension
bi = mk_band('a', base, path="test.tif", format=NetCDF)
assert _rio_band_idx(bi, src) == 1
# TODO: make single time-slice netcdf, for now pretend that this tiff is netcdf
with rasterio.open(str(data_folder) + "/sample_tile_151_-29.tif",
'r') as src:
bi = mk_band('a', base, path='sample_tile_151_-29.tif', format=NetCDF)
assert src.count == 1
assert _rio_band_idx(bi, src) == 1
def get_bandnumber(self, src=None):
# If `band` property is set to an integer it overrides any other logic
band = self._measurement.get('band')
if band is not None:
if isinstance(band, integer_types):
return band
else:
_LOG.warning('Expected "band" property to be of integer type')
if not self._netcdf:
layer_id = self._measurement.get('layer', 1)
return layer_id if isinstance(layer_id, integer_types) else 1
# Netcdf only below
if self._part is not None:
return self._part + 1 # Convert to rasterio 1-based indexing
if src is None:
# File wasnt' open, could be unstacked file in a new format, or
# stacked/unstacked in old. We assume caller knows what to do
# (maybe based on some side-channel information), so just report
# undefined.
return None
if src.count == 1: # Single-slice netcdf file
return 1
_LOG.debug(
"Encountered stacked netcdf file without recorded index\n - %s",
src.name)
# Below is backwards compatibility code
tag_name = GDAL_NETCDF_DIM + 'time'
if tag_name not in src.tags(
1): # TODO: support time-less datasets properly
return 1
time = self._dataset.center_time
sec_since_1970 = datetime_to_seconds_since_1970(time)
idx = 0
dist = float('+inf')
for i in range(1, src.count + 1):
v = float(src.tags(i)[tag_name])
if abs(sec_since_1970 - v) < dist:
idx = i
dist = abs(sec_since_1970 - v)
return idx
def get_bandnumber(self):
time = self._band.center_time
sec_since_1970 = datetime_to_seconds_since_1970(time)
s3_dataset = self._s3_metadata[self._band.name]['s3_dataset']
if s3_dataset.regular_dims[0]: # If time is regular
return int((sec_since_1970 - s3_dataset.regular_index[0]) /
s3_dataset.regular_index[2])
else:
epsilon = DriverUtils.epsilon('time')
for idx, timestamp in enumerate(s3_dataset.irregular_index[0]):
if abs(sec_since_1970 - timestamp / 1000000000.0) < epsilon:
return idx
raise ValueError('Cannot find band number for centre time %s' % time)
def get_bandnumber(self, src):
if 'netcdf' not in self._dataset.format.lower():
layer_id = self._measurement.get('layer', 1)
return layer_id if isinstance(layer_id, integer_types) else 1
tag_name = GDAL_NETCDF_DIM + 'time'
if tag_name not in src.tags(
1): # TODO: support time-less datasets properly
return 1
time = self._dataset.center_time
sec_since_1970 = datetime_to_seconds_since_1970(time)
idx = 0
dist = float('+inf')
for i in range(1, src.count + 1):
v = float(src.tags(i)[tag_name])
if abs(sec_since_1970 - v) < dist:
idx = i
dist = abs(sec_since_1970 - v)
return idx
def _find_netcdf_band_by_time(src: DatasetReader, ts: datetime) -> int:
""" backwards compatibility code.
finds band that is nearest to a given timestamp
"""
time_tag = 'NETCDF_DIM_time'
ts0 = datetime_to_seconds_since_1970(ts)
def get_ts() -> Iterator[Tuple[int, float]]:
for bidx in range(1, src.count+1):
tag_value = src.get_tag_item(time_tag, bidx=bidx)
if tag_value is not None:
yield (bidx, float(tag_value))
all_ts = list((bidx, abs(ts - ts0))
for bidx, ts in get_ts())
if len(all_ts) == 0:
return 1 # TODO: copying previous behaviour, should at least log something
all_ts.sort(key=lambda xx: xx[1])
bidx, _ = all_ts[0]
return bidx
def compute_and_write(self):
"""
Computes the wofs confidence and filtered summary bands and write to the
corresponding NetCDF file. The file template and location etc are read from the configs.
"""
geo_box = self.grid_spec.tile_geobox(self.tile_index)
# Compute metadata
env = self.cfg.get_env_of_product('wofs_filtered_summary')
with Datacube(app='wofs-confidence', env=env) as dc:
product = dc.index.products.get_by_name('wofs_filtered_summary')
extent = self.grid_spec.tile_geobox(self.tile_index).extent
center_time = datetime.now()
uri = self.get_filtered_uri()
dts = make_dataset(product=product,
sources=self.factor_sources,
extent=extent,
center_time=center_time,
uri=uri)
metadata = yaml.dump(dts.metadata_doc,
Dumper=SafeDumper,
encoding='utf-8')
# Compute dataset coords
coords = dict()
coords['time'] = Coordinate(
netcdf_writer.netcdfy_coord(
np.array([datetime_to_seconds_since_1970(center_time)])),
['seconds since 1970-01-01 00:00:00'])
for dim in geo_box.dimensions:
coords[dim] = Coordinate(
netcdf_writer.netcdfy_coord(geo_box.coordinates[dim].values),
geo_box.coordinates[dim].units)
# Compute dataset variables
spatial_var = Variable(dtype=np.dtype(DEFAULT_TYPE),
nodata=DEFAULT_FLOAT_NODATA,
dims=('time', ) + geo_box.dimensions,
units=('seconds since 1970-01-01 00:00:00', ) +
geo_box.crs.units)
band1 = self.cfg.cfg['wofs_filtered_summary']['confidence']
band2 = self.cfg.cfg['wofs_filtered_summary']['confidence_filtered']
vars = {band1: spatial_var, band2: spatial_var}
vars_params = {band1: {}, band2: {}}
global_atts = self.cfg.cfg['global_attributes']
# Get crs string
crs = self.cfg.cfg['storage']['crs'] if self.cfg.cfg['storage'].get(
'crs') else DEFAULT_CRS
# Create a dataset container
filename = self.get_filtered_uri()
logger.info('creating', file=filename.name)
netcdf_unit = create_netcdf_storage_unit(filename=filename,
crs=CRS(crs),
coordinates=coords,
variables=vars,
global_attributes=global_atts,
variable_params=vars_params)
# Confidence layer: Fill variable data and set attributes
confidence = self.compute_confidence()
netcdf_unit[band1][:] = netcdf_writer.netcdfy_data(confidence)
netcdf_unit[band1].units = '1'
netcdf_unit[band1].valid_range = [0, 1.0]
netcdf_unit[band1].coverage_content_type = 'modelResult'
netcdf_unit[band1].long_name = \
'Wofs Confidence Layer predicted by {}'.format(self.confidence_model.factors.__str__())
# Confidence filtered wofs-stats frequency layer: Fill variable data and set attributes
confidence_filtered = self.compute_confidence_filtered()
netcdf_unit[band2][:] = netcdf_writer.netcdfy_data(confidence_filtered)
netcdf_unit[band2].units = '1'
netcdf_unit[band2].valid_range = [0, 1.0]
netcdf_unit[band2].coverage_content_type = 'modelResult'
netcdf_unit[
band2].long_name = 'WOfS-Stats frequency confidence filtered layer'
# Metadata
dataset_data = DataArray(data=[metadata], dims=('time', ))
netcdf_writer.create_variable(netcdf_unit,
'dataset',
dataset_data,
zlib=True)
netcdf_unit['dataset'][:] = netcdf_writer.netcdfy_data(
dataset_data.values)
netcdf_unit.close()
logger.info('completed', file=filename.name)
def datetime_to_timestamp(dt):
if not isinstance(dt, datetime.datetime) and not isinstance(
dt, datetime.date):
dt = to_datetime(dt)
return datetime_to_seconds_since_1970(dt)
def datetime_to_timestamp(dt):
if not isinstance(dt, datetime.datetime) and not isinstance(dt, datetime.date):
dt = to_datetime(dt)
return datetime_to_seconds_since_1970(dt)
def time_coordinate_value(time):
return CoordinateValue(dimension_name='time',
value=datetime_to_seconds_since_1970(time),
dtype=numpy.dtype(numpy.float64),
units='seconds since 1970-01-01 00:00:00')
def time_coordinate_value(time):
return CoordinateValue(dimension_name='time',
value=datetime_to_seconds_since_1970(time),
dtype=numpy.dtype(numpy.float64),
units='seconds since 1970-01-01 00:00:00')