def get_bandnumber(self, src): # If `band` property is set to an integer it overrides any other logic band = self._measurement.get('band') if band is not None: if isinstance(band, integer_types): return band else: _LOG.warning('Expected "band" property to be of integer type') if 'netcdf' not in self._dataset.format.lower(): layer_id = self._measurement.get('layer', 1) return layer_id if isinstance(layer_id, integer_types) else 1 tag_name = GDAL_NETCDF_DIM + 'time' if tag_name not in src.tags( 1): # TODO: support time-less datasets properly return 1 time = self._dataset.center_time sec_since_1970 = datetime_to_seconds_since_1970(time) idx = 0 dist = float('+inf') for i in range(1, src.count + 1): v = float(src.tags(i)[tag_name]) if abs(sec_since_1970 - v) < dist: idx = i dist = abs(sec_since_1970 - v) return idx
def wheres_my_band(self, src, time): sec_since_1970 = datetime_to_seconds_since_1970(time) idx = 0 dist = float('+inf') for i in range(1, src.count + 1): v = float(src.tags(i)['NETCDF_DIM_time']) if abs(sec_since_1970 - v) < dist: idx = i return idx
def test_rd_internals_bidx(data_folder): base = "file://" + str(data_folder) + "/metadata.yml" bi = mk_band('a', base, path="multi_doc.nc", format=NetCDF, timestamp=datetime.utcfromtimestamp(1), layer='a') assert bi.uri.endswith('multi_doc.nc') assert datetime_to_seconds_since_1970(bi.center_time) == 1 rio_fname = _rio_uri(bi) assert rio_fname.startswith('NETCDF:') with rasterio.open(rio_fname) as src: # timestamp search bidx = _rio_band_idx(bi, src) assert bidx == 2 # extract from .uri bi.uri = bi.uri + "#part=5" assert _rio_band_idx(bi, src) == 5 # extract from .band bi.band = 33 assert _rio_band_idx(bi, src) == 33 bi = mk_band('a', base, path="test.tif", format=GeoTIFF) with rasterio.open(_rio_uri(bi), 'r') as src: # should default to 1 assert _rio_band_idx(bi, src) == 1 # layer containing int should become index bi = mk_band('a', base, path="test.tif", format=GeoTIFF, layer=2) assert _rio_band_idx(bi, src) == 2 # band is the keyword bi = mk_band('a', base, path="test.tif", format=GeoTIFF, band=3) assert _rio_band_idx(bi, src) == 3 # Pretend it's netcdf to test missing time dimension bi = mk_band('a', base, path="test.tif", format=NetCDF) assert _rio_band_idx(bi, src) == 1 # TODO: make single time-slice netcdf, for now pretend that this tiff is netcdf with rasterio.open(str(data_folder) + "/sample_tile_151_-29.tif", 'r') as src: bi = mk_band('a', base, path='sample_tile_151_-29.tif', format=NetCDF) assert src.count == 1 assert _rio_band_idx(bi, src) == 1
def get_bandnumber(self, src=None): # If `band` property is set to an integer it overrides any other logic band = self._measurement.get('band') if band is not None: if isinstance(band, integer_types): return band else: _LOG.warning('Expected "band" property to be of integer type') if not self._netcdf: layer_id = self._measurement.get('layer', 1) return layer_id if isinstance(layer_id, integer_types) else 1 # Netcdf only below if self._part is not None: return self._part + 1 # Convert to rasterio 1-based indexing if src is None: # File wasnt' open, could be unstacked file in a new format, or # stacked/unstacked in old. We assume caller knows what to do # (maybe based on some side-channel information), so just report # undefined. return None if src.count == 1: # Single-slice netcdf file return 1 _LOG.debug( "Encountered stacked netcdf file without recorded index\n - %s", src.name) # Below is backwards compatibility code tag_name = GDAL_NETCDF_DIM + 'time' if tag_name not in src.tags( 1): # TODO: support time-less datasets properly return 1 time = self._dataset.center_time sec_since_1970 = datetime_to_seconds_since_1970(time) idx = 0 dist = float('+inf') for i in range(1, src.count + 1): v = float(src.tags(i)[tag_name]) if abs(sec_since_1970 - v) < dist: idx = i dist = abs(sec_since_1970 - v) return idx
def get_bandnumber(self): time = self._band.center_time sec_since_1970 = datetime_to_seconds_since_1970(time) s3_dataset = self._s3_metadata[self._band.name]['s3_dataset'] if s3_dataset.regular_dims[0]: # If time is regular return int((sec_since_1970 - s3_dataset.regular_index[0]) / s3_dataset.regular_index[2]) else: epsilon = DriverUtils.epsilon('time') for idx, timestamp in enumerate(s3_dataset.irregular_index[0]): if abs(sec_since_1970 - timestamp / 1000000000.0) < epsilon: return idx raise ValueError('Cannot find band number for centre time %s' % time)
def get_bandnumber(self, src): if 'netcdf' not in self._dataset.format.lower(): layer_id = self._measurement.get('layer', 1) return layer_id if isinstance(layer_id, integer_types) else 1 tag_name = GDAL_NETCDF_DIM + 'time' if tag_name not in src.tags( 1): # TODO: support time-less datasets properly return 1 time = self._dataset.center_time sec_since_1970 = datetime_to_seconds_since_1970(time) idx = 0 dist = float('+inf') for i in range(1, src.count + 1): v = float(src.tags(i)[tag_name]) if abs(sec_since_1970 - v) < dist: idx = i dist = abs(sec_since_1970 - v) return idx
def _find_netcdf_band_by_time(src: DatasetReader, ts: datetime) -> int: """ backwards compatibility code. finds band that is nearest to a given timestamp """ time_tag = 'NETCDF_DIM_time' ts0 = datetime_to_seconds_since_1970(ts) def get_ts() -> Iterator[Tuple[int, float]]: for bidx in range(1, src.count+1): tag_value = src.get_tag_item(time_tag, bidx=bidx) if tag_value is not None: yield (bidx, float(tag_value)) all_ts = list((bidx, abs(ts - ts0)) for bidx, ts in get_ts()) if len(all_ts) == 0: return 1 # TODO: copying previous behaviour, should at least log something all_ts.sort(key=lambda xx: xx[1]) bidx, _ = all_ts[0] return bidx
def compute_and_write(self): """ Computes the wofs confidence and filtered summary bands and write to the corresponding NetCDF file. The file template and location etc are read from the configs. """ geo_box = self.grid_spec.tile_geobox(self.tile_index) # Compute metadata env = self.cfg.get_env_of_product('wofs_filtered_summary') with Datacube(app='wofs-confidence', env=env) as dc: product = dc.index.products.get_by_name('wofs_filtered_summary') extent = self.grid_spec.tile_geobox(self.tile_index).extent center_time = datetime.now() uri = self.get_filtered_uri() dts = make_dataset(product=product, sources=self.factor_sources, extent=extent, center_time=center_time, uri=uri) metadata = yaml.dump(dts.metadata_doc, Dumper=SafeDumper, encoding='utf-8') # Compute dataset coords coords = dict() coords['time'] = Coordinate( netcdf_writer.netcdfy_coord( np.array([datetime_to_seconds_since_1970(center_time)])), ['seconds since 1970-01-01 00:00:00']) for dim in geo_box.dimensions: coords[dim] = Coordinate( netcdf_writer.netcdfy_coord(geo_box.coordinates[dim].values), geo_box.coordinates[dim].units) # Compute dataset variables spatial_var = Variable(dtype=np.dtype(DEFAULT_TYPE), nodata=DEFAULT_FLOAT_NODATA, dims=('time', ) + geo_box.dimensions, units=('seconds since 1970-01-01 00:00:00', ) + geo_box.crs.units) band1 = self.cfg.cfg['wofs_filtered_summary']['confidence'] band2 = self.cfg.cfg['wofs_filtered_summary']['confidence_filtered'] vars = {band1: spatial_var, band2: spatial_var} vars_params = {band1: {}, band2: {}} global_atts = self.cfg.cfg['global_attributes'] # Get crs string crs = self.cfg.cfg['storage']['crs'] if self.cfg.cfg['storage'].get( 'crs') else DEFAULT_CRS # Create a dataset container filename = self.get_filtered_uri() logger.info('creating', file=filename.name) netcdf_unit = create_netcdf_storage_unit(filename=filename, crs=CRS(crs), coordinates=coords, variables=vars, global_attributes=global_atts, variable_params=vars_params) # Confidence layer: Fill variable data and set attributes confidence = self.compute_confidence() netcdf_unit[band1][:] = netcdf_writer.netcdfy_data(confidence) netcdf_unit[band1].units = '1' netcdf_unit[band1].valid_range = [0, 1.0] netcdf_unit[band1].coverage_content_type = 'modelResult' netcdf_unit[band1].long_name = \ 'Wofs Confidence Layer predicted by {}'.format(self.confidence_model.factors.__str__()) # Confidence filtered wofs-stats frequency layer: Fill variable data and set attributes confidence_filtered = self.compute_confidence_filtered() netcdf_unit[band2][:] = netcdf_writer.netcdfy_data(confidence_filtered) netcdf_unit[band2].units = '1' netcdf_unit[band2].valid_range = [0, 1.0] netcdf_unit[band2].coverage_content_type = 'modelResult' netcdf_unit[ band2].long_name = 'WOfS-Stats frequency confidence filtered layer' # Metadata dataset_data = DataArray(data=[metadata], dims=('time', )) netcdf_writer.create_variable(netcdf_unit, 'dataset', dataset_data, zlib=True) netcdf_unit['dataset'][:] = netcdf_writer.netcdfy_data( dataset_data.values) netcdf_unit.close() logger.info('completed', file=filename.name)
def datetime_to_timestamp(dt): if not isinstance(dt, datetime.datetime) and not isinstance( dt, datetime.date): dt = to_datetime(dt) return datetime_to_seconds_since_1970(dt)
def datetime_to_timestamp(dt): if not isinstance(dt, datetime.datetime) and not isinstance(dt, datetime.date): dt = to_datetime(dt) return datetime_to_seconds_since_1970(dt)
def time_coordinate_value(time): return CoordinateValue(dimension_name='time', value=datetime_to_seconds_since_1970(time), dtype=numpy.dtype(numpy.float64), units='seconds since 1970-01-01 00:00:00')