def read_data(self, tile_specifications, file_path, output_tile):
with Dataset(file_path) as ds:
for section_spec, dimtoslice in tile_specifications:
tile = nexusproto.TimeSeriesTile()
instance_dimension = next(
iter([dim for dim in ds[self.variable_to_read].dimensions if dim != self.time]))
tile.latitude.CopyFrom(
to_shaped_array(numpy.ma.filled(ds[self.latitude][dimtoslice[instance_dimension]], numpy.NaN)))
tile.longitude.CopyFrom(
to_shaped_array(numpy.ma.filled(ds[self.longitude][dimtoslice[instance_dimension]], numpy.NaN)))
# Before we read the data we need to make sure the dimensions are in the proper order so we don't
# have any indexing issues
ordered_slices = get_ordered_slices(ds, self.variable_to_read, dimtoslice)
# Read data using the ordered slices, replacing masked values with NaN
data_array = numpy.ma.filled(ds[self.variable_to_read][tuple(ordered_slices.values())], numpy.NaN)
tile.variable_data.CopyFrom(to_shaped_array(data_array))
if self.metadata is not None:
tile.meta_data.add().CopyFrom(
to_metadata(self.metadata, ds[self.metadata][tuple(ordered_slices.values())]))
tile.time.CopyFrom(
to_shaped_array(numpy.ma.filled(ds[self.time][dimtoslice[self.time]], numpy.NaN)))
output_tile.tile.time_series_tile.CopyFrom(tile)
yield output_tile
def test_extract_timestamp_timeseries_exception(self):
test_file = path.join(path.dirname(__file__), 'datafiles',
'not_empty_modis.nc')
input_tile = nexusproto.NexusTile()
tile_summary = nexusproto.TileSummary()
tile_summary.granule = "file:%s" % test_file
tile_summary.section_spec = "time:0:1,lat:0:10,lon:0:10"
input_tile.summary.CopyFrom(tile_summary)
input_tile.tile.time_series_tile.CopyFrom(nexusproto.TimeSeriesTile())
with self.assertRaises(BadTimestampExtractionException):
list(self.module.process_nexus_tile(
input_tile))[0].tile.time_series_tile.time
def _generate_tile(self, ds: xr.Dataset,
dimensions_to_slices: Dict[str, slice], input_tile):
new_tile = nexusproto.TimeSeriesTile()
lat_subset = ds[self.latitude][type(self)._slices_for_variable(
ds[self.latitude], dimensions_to_slices)]
lon_subset = ds[self.longitude][type(self)._slices_for_variable(
ds[self.longitude], dimensions_to_slices)]
lat_subset = np.ma.filled(lat_subset, np.NaN)
lon_subset = np.ma.filled(lon_subset, np.NaN)
data_subset = ds[self.variable][type(self)._slices_for_variable(
ds[self.variable], dimensions_to_slices)]
data_subset = np.ma.filled(data_subset, np.NaN)
if self.depth:
depth_dim, depth_slice = list(
type(self)._slices_for_variable(
ds[self.depth], dimensions_to_slices).items())[0]
depth_slice_len = depth_slice.stop - depth_slice.start
if depth_slice_len > 1:
raise RuntimeError(
"Depth slices must have length 1, but '{dim}' has length {dim_len}."
.format(dim=depth_dim, dim_len=depth_slice_len))
new_tile.depth = ds[self.depth][depth_slice].item()
time_subset = ds[self.time][type(self)._slices_for_variable(
ds[self.time], dimensions_to_slices)]
time_subset = np.ma.filled(
type(self)._convert_to_timestamp(time_subset), np.NaN)
new_tile.latitude.CopyFrom(to_shaped_array(lat_subset))
new_tile.longitude.CopyFrom(to_shaped_array(lon_subset))
new_tile.variable_data.CopyFrom(to_shaped_array(data_subset))
new_tile.time.CopyFrom(to_shaped_array(time_subset))
input_tile.tile.time_series_tile.CopyFrom(new_tile)
return input_tile