def read_data(self, tile_specifications, file_path, output_tile):
with xr.decode_cf(xr.open_dataset(file_path, decode_cf=False),
decode_times=False) as ds:
for section_spec, dimtoslice in tile_specifications:
tile = nexusproto.SwathTile()
# Time Lat Long Data and metadata should all be indexed by the same dimensions, order the incoming spec once using the data variable
ordered_slices = get_ordered_slices(ds, self.variable_to_read,
dimtoslice)
tile.latitude.CopyFrom(
to_shaped_array(
numpy.ma.filled(
ds[self.latitude].data[tuple(
ordered_slices.values())], numpy.NaN)))
tile.longitude.CopyFrom(
to_shaped_array(
numpy.ma.filled(
ds[self.longitude].data[tuple(
ordered_slices.values())], numpy.NaN)))
timetile = ds[self.time][tuple([
ordered_slices[time_dim] for time_dim in ds[self.time].dims
])].astype('float64', casting='same_kind', copy=False)
timeunits = ds[self.time].attrs['units']
try:
start_of_day_date = datetime.datetime.strptime(
ds.attrs[self.start_of_day], self.start_of_day_pattern)
except Exception:
start_of_day_date = None
for index in numpy.ndindex(timetile.shape):
timetile[index] = to_seconds_from_epoch(
timetile[index].item(),
timeunits=timeunits,
start_day=start_of_day_date,
timeoffset=self.time_offset)
tile.time.CopyFrom(to_shaped_array(timetile))
# Read the data converting masked values to NaN
data_array = numpy.ma.filled(
ds[self.variable_to_read].data[tuple(
ordered_slices.values())], numpy.NaN)
tile.variable_data.CopyFrom(to_shaped_array(data_array))
if self.metadata is not None:
tile.meta_data.add().CopyFrom(
to_metadata(
self.metadata, ds[self.metadata].data[tuple(
ordered_slices.values())]))
output_tile.tile.swath_tile.CopyFrom(tile)
yield output_tile
def test_extract_timestamp_swath_exception(self):
test_file = path.join(path.dirname(__file__), 'datafiles',
'not_empty_modis.nc')
input_tile = nexusproto.NexusTile()
tile_summary = nexusproto.TileSummary()
tile_summary.granule = "file:%s" % test_file
tile_summary.section_spec = "time:0:1,lat:0:10,lon:0:10"
input_tile.summary.CopyFrom(tile_summary)
input_tile.tile.swath_tile.CopyFrom(nexusproto.SwathTile())
with self.assertRaises(BadTimestampExtractionException):
list(self.module.process_nexus_tile(
input_tile))[0].tile.swath_tile.time
def _generate_tile(self, ds: xr.Dataset,
dimensions_to_slices: Dict[str, slice], input_tile):
new_tile = nexusproto.SwathTile()
lat_subset = ds[self.latitude][type(self)._slices_for_variable(
ds[self.latitude], dimensions_to_slices)]
lon_subset = ds[self.longitude][type(self)._slices_for_variable(
ds[self.longitude], dimensions_to_slices)]
lat_subset = np.ma.filled(lat_subset, np.NaN)
lon_subset = np.ma.filled(lon_subset, np.NaN)
time_subset = ds[self.time][type(self)._slices_for_variable(
ds[self.time], dimensions_to_slices)]
time_subset = np.ma.filled(
type(self)._convert_to_timestamp(time_subset), np.NaN)
data_subset = ds[self.variable][type(self)._slices_for_variable(
ds[self.variable], dimensions_to_slices)]
data_subset = np.ma.filled(data_subset, np.NaN)
if self.depth:
depth_dim, depth_slice = list(
type(self)._slices_for_variable(
ds[self.depth], dimensions_to_slices).items())[0]
depth_slice_len = depth_slice.stop - depth_slice.start
if depth_slice_len > 1:
raise RuntimeError(
"Depth slices must have length 1, but '{dim}' has length {dim_len}."
.format(dim=depth_dim, dim_len=depth_slice_len))
new_tile.depth = ds[self.depth][depth_slice].item()
new_tile.latitude.CopyFrom(to_shaped_array(lat_subset))
new_tile.longitude.CopyFrom(to_shaped_array(lon_subset))
new_tile.variable_data.CopyFrom(to_shaped_array(data_subset))
new_tile.time.CopyFrom(to_shaped_array(time_subset))
input_tile.tile.swath_tile.CopyFrom(new_tile)
return input_tile