def read_data(self, tile_specifications, file_path, output_tile):
with Dataset(file_path) as ds:
for section_spec, dimtoslice in tile_specifications:
tile = nexusproto.TimeSeriesTile()
instance_dimension = next(
iter([dim for dim in ds[self.variable_to_read].dimensions if dim != self.time]))
tile.latitude.CopyFrom(
to_shaped_array(numpy.ma.filled(ds[self.latitude][dimtoslice[instance_dimension]], numpy.NaN)))
tile.longitude.CopyFrom(
to_shaped_array(numpy.ma.filled(ds[self.longitude][dimtoslice[instance_dimension]], numpy.NaN)))
# Before we read the data we need to make sure the dimensions are in the proper order so we don't
# have any indexing issues
ordered_slices = get_ordered_slices(ds, self.variable_to_read, dimtoslice)
# Read data using the ordered slices, replacing masked values with NaN
data_array = numpy.ma.filled(ds[self.variable_to_read][tuple(ordered_slices.values())], numpy.NaN)
tile.variable_data.CopyFrom(to_shaped_array(data_array))
if self.metadata is not None:
tile.meta_data.add().CopyFrom(
to_metadata(self.metadata, ds[self.metadata][tuple(ordered_slices.values())]))
tile.time.CopyFrom(
to_shaped_array(numpy.ma.filled(ds[self.time][dimtoslice[self.time]], numpy.NaN)))
output_tile.tile.time_series_tile.CopyFrom(tile)
yield output_tile
def read_grid_data(self, section_spec_dataset):
tile_specifications, file_path = parse_input(section_spec_dataset)
# Time is optional for Grid data
try:
time = environ['TIME']
except KeyError:
time = None
with Dataset(file_path) as ds:
for section_spec, dimtoslice in tile_specifications:
tile = nexusproto.GridTile()
tile.latitude.CopyFrom(
to_shaped_array(
numpy.ma.filled(ds[latitude][dimtoslice[latitude]],
numpy.NaN)))
tile.longitude.CopyFrom(
to_shaped_array(
numpy.ma.filled(ds[longitude][dimtoslice[longitude]],
numpy.NaN)))
# Before we read the data we need to make sure the dimensions are in the proper order so we don't have any
# indexing issues
ordered_slices = get_ordered_slices(ds, variable_to_read,
dimtoslice)
# Read data using the ordered slices, replacing masked values with NaN
data_array = numpy.ma.filled(
ds[variable_to_read][tuple(ordered_slices.itervalues())],
numpy.NaN)
tile.variable_data.CopyFrom(to_shaped_array(data_array))
if metadata is not None:
tile.meta_data.add().CopyFrom(
to_metadata(
metadata,
ds[metadata][tuple(ordered_slices.itervalues())]))
if time is not None:
timevar = ds[time]
# Note assumption is that index of time is start value in dimtoslice
tile.time = to_seconds_from_epoch(
timevar[dimtoslice[time].start],
timeunits=timevar.getncattr('units'),
timeoffset=time_offset)
nexus_tile = new_nexus_tile(file_path, section_spec)
nexus_tile.tile.grid_tile.CopyFrom(tile)
yield nexus_tile.SerializeToString()
# If temp dir is defined, delete the temporary file
if temp_dir is not None:
remove(file_path)
def read_data(self, tile_specifications, file_path, output_tile):
with Dataset(file_path) as ds:
for section_spec, dimtoslice in tile_specifications:
tile = nexusproto.SwathTile()
# Time Lat Long Data and metadata should all be indexed by the same dimensions, order the incoming spec once using the data variable
ordered_slices = get_ordered_slices(ds, self.variable_to_read,
dimtoslice)
tile.latitude.CopyFrom(
to_shaped_array(
numpy.ma.filled(
ds[self.latitude][tuple(ordered_slices.values())],
numpy.NaN)))
tile.longitude.CopyFrom(
to_shaped_array(
numpy.ma.filled(
ds[self.longitude][tuple(ordered_slices.values())],
numpy.NaN)))
timetile = ds[self.time][tuple([
ordered_slices[time_dim]
for time_dim in ds[self.time].dimensions
])].astype('float64', casting='same_kind', copy=False)
timeunits = ds[self.time].getncattr('units')
try:
start_of_day_date = datetime.datetime.strptime(
ds.getncattr(self.start_of_day),
self.start_of_day_pattern)
except Exception:
start_of_day_date = None
for index in numpy.ndindex(timetile.shape):
timetile[index] = to_seconds_from_epoch(
timetile[index].item(),
timeunits=timeunits,
start_day=start_of_day_date,
timeoffset=self.time_offset)
tile.time.CopyFrom(to_shaped_array(timetile))
# Read the data converting masked values to NaN
data_array = numpy.ma.filled(
ds[self.variable_to_read][tuple(ordered_slices.values())],
numpy.NaN)
tile.variable_data.CopyFrom(to_shaped_array(data_array))
if self.metadata is not None:
tile.meta_data.add().CopyFrom(
to_metadata(
self.metadata,
ds[self.metadata][tuple(ordered_slices.values())]))
output_tile.tile.swath_tile.CopyFrom(tile)
yield output_tile
def read_data(self, tile_specifications, file_path, output_tile):
# Time is optional for Grid data
time = self.environ['TIME']
with Dataset(file_path) as ds:
for section_spec, dimtoslice in tile_specifications:
tile = nexusproto.GridTile()
tile.latitude.CopyFrom(
to_shaped_array(
numpy.ma.filled(
ds[self.latitude][dimtoslice[self.latitude]],
numpy.NaN)))
tile.longitude.CopyFrom(
to_shaped_array(
numpy.ma.filled(
ds[self.longitude][dimtoslice[self.longitude]],
numpy.NaN)))
# Before we read the data we need to make sure the dimensions are in the proper order so we don't have any
# indexing issues
ordered_slices = get_ordered_slices(ds, self.variable_to_read,
dimtoslice)
# Read data using the ordered slices, replacing masked values with NaN
data_array = numpy.ma.filled(
ds[self.variable_to_read][tuple(ordered_slices.values())],
numpy.NaN)
tile.variable_data.CopyFrom(to_shaped_array(data_array))
if self.metadata is not None:
tile.meta_data.add().CopyFrom(
to_metadata(
self.metadata,
ds[self.metadata][tuple(ordered_slices.values())]))
if time is not None:
timevar = ds[time]
# Note assumption is that index of time is start value in dimtoslice
tile.time = to_seconds_from_epoch(
timevar[dimtoslice[time].start],
timeunits=timevar.getncattr('units'),
timeoffset=self.time_offset)
output_tile.tile.grid_tile.CopyFrom(tile)
yield output_tile
def read_swath_data(self, section_spec_dataset):
tile_specifications, file_path = parse_input(section_spec_dataset)
# Time is required for swath data
time = environ['TIME']
with Dataset(file_path) as ds:
for section_spec, dimtoslice in tile_specifications:
tile = nexusproto.SwathTile()
# Time Lat Long Data and metadata should all be indexed by the same dimensions, order the incoming spec once using the data variable
ordered_slices = get_ordered_slices(ds, variable_to_read,
dimtoslice)
tile.latitude.CopyFrom(
to_shaped_array(ds[latitude][tuple(
ordered_slices.itervalues())]))
tile.longitude.CopyFrom(
to_shaped_array(ds[longitude][tuple(
ordered_slices.itervalues())]))
timeunits = ds[time].getncattr('units')
timetile = ds[time][tuple(ordered_slices.itervalues())]
for index in numpy.ndindex(timetile.shape):
timetile[index] = to_seconds_from_epoch(
timetile[index].item(), timeunits)
tile.time.CopyFrom(to_shaped_array(timetile))
# Read the data converting masked values to NaN
data_array = numpy.ma.filled(
ds[variable_to_read][tuple(ordered_slices.itervalues())],
numpy.NaN)
tile.variable_data.CopyFrom(to_shaped_array(data_array))
if metadata is not None:
tile.meta_data.add().CopyFrom(
to_metadata(
metadata,
ds[metadata][tuple(ordered_slices.itervalues())]))
nexus_tile = new_nexus_tile(file_path, section_spec)
nexus_tile.tile.swath_tile.CopyFrom(tile)
yield nexus_tile.SerializeToString()
# If temp dir is defined, delete the temporary file
if temp_dir is not None:
remove(file_path)