def process(self, tile, *args, **kwargs):
"""
This method will reverse the ordering of latitude values in a tile if necessary to ensure that the latitude values are ascending.
:param self:
:param tile: The nexus_tile
:return: Tile data with altered latitude values
"""
the_tile_type = tile.tile.WhichOneof("tile_type")
the_tile_data = getattr(tile.tile, the_tile_type)
latitudes = from_shaped_array(the_tile_data.latitude)
data = from_shaped_array(the_tile_data.variable_data)
# Only reverse latitude ordering if current ordering is descending.
if len(latitudes) > 1:
delta = latitudes[1] - latitudes[0]
if delta < 0:
latitudes = np.flip(latitudes)
data = np.flip(data, axis=0)
the_tile_data.latitude.CopyFrom(to_shaped_array(latitudes))
the_tile_data.variable_data.CopyFrom(to_shaped_array(data))
return tile
def read_data(self, tile_specifications, file_path, output_tile):
with Dataset(file_path) as ds:
for section_spec, dimtoslice in tile_specifications:
tile = nexusproto.TimeSeriesTile()
instance_dimension = next(
iter([dim for dim in ds[self.variable_to_read].dimensions if dim != self.time]))
tile.latitude.CopyFrom(
to_shaped_array(numpy.ma.filled(ds[self.latitude][dimtoslice[instance_dimension]], numpy.NaN)))
tile.longitude.CopyFrom(
to_shaped_array(numpy.ma.filled(ds[self.longitude][dimtoslice[instance_dimension]], numpy.NaN)))
# Before we read the data we need to make sure the dimensions are in the proper order so we don't
# have any indexing issues
ordered_slices = get_ordered_slices(ds, self.variable_to_read, dimtoslice)
# Read data using the ordered slices, replacing masked values with NaN
data_array = numpy.ma.filled(ds[self.variable_to_read][tuple(ordered_slices.values())], numpy.NaN)
tile.variable_data.CopyFrom(to_shaped_array(data_array))
if self.metadata is not None:
tile.meta_data.add().CopyFrom(
to_metadata(self.metadata, ds[self.metadata][tuple(ordered_slices.values())]))
tile.time.CopyFrom(
to_shaped_array(numpy.ma.filled(ds[self.time][dimtoslice[self.time]], numpy.NaN)))
output_tile.tile.time_series_tile.CopyFrom(tile)
yield output_tile
def read_grid_data(self, section_spec_dataset):
tile_specifications, file_path = parse_input(section_spec_dataset)
# Time is optional for Grid data
try:
time = environ['TIME']
except KeyError:
time = None
with Dataset(file_path) as ds:
for section_spec, dimtoslice in tile_specifications:
tile = nexusproto.GridTile()
tile.latitude.CopyFrom(
to_shaped_array(
numpy.ma.filled(ds[latitude][dimtoslice[latitude]],
numpy.NaN)))
tile.longitude.CopyFrom(
to_shaped_array(
numpy.ma.filled(ds[longitude][dimtoslice[longitude]],
numpy.NaN)))
# Before we read the data we need to make sure the dimensions are in the proper order so we don't have any
# indexing issues
ordered_slices = get_ordered_slices(ds, variable_to_read,
dimtoslice)
# Read data using the ordered slices, replacing masked values with NaN
data_array = numpy.ma.filled(
ds[variable_to_read][tuple(ordered_slices.itervalues())],
numpy.NaN)
tile.variable_data.CopyFrom(to_shaped_array(data_array))
if metadata is not None:
tile.meta_data.add().CopyFrom(
to_metadata(
metadata,
ds[metadata][tuple(ordered_slices.itervalues())]))
if time is not None:
timevar = ds[time]
# Note assumption is that index of time is start value in dimtoslice
tile.time = to_seconds_from_epoch(
timevar[dimtoslice[time].start],
timeunits=timevar.getncattr('units'),
timeoffset=time_offset)
nexus_tile = new_nexus_tile(file_path, section_spec)
nexus_tile.tile.grid_tile.CopyFrom(tile)
yield nexus_tile.SerializeToString()
# If temp dir is defined, delete the temporary file
if temp_dir is not None:
remove(file_path)
def process_nexus_tile(self, nexus_tile):
the_tile_type = nexus_tile.tile.WhichOneof("tile_type")
the_tile_data = getattr(nexus_tile.tile, the_tile_type)
wind_speed = from_shaped_array(the_tile_data.variable_data)
wind_dir = from_shaped_array(
next(meta for meta in the_tile_data.meta_data
if meta.name == 'wind_dir').meta_data)
assert wind_speed.shape == wind_dir.shape
wind_u_component = numpy.ma.empty(wind_speed.shape, dtype=float)
wind_v_component = numpy.ma.empty(wind_speed.shape, dtype=float)
wind_speed_iter = numpy.nditer(wind_speed, flags=['multi_index'])
while not wind_speed_iter.finished:
speed = wind_speed_iter[0]
current_index = wind_speed_iter.multi_index
direction = wind_dir[current_index]
# Convert degrees to radians
direction = radians(direction)
# Calculate component values
wind_u_component[current_index] = calculate_u_component_value(
direction, speed)
wind_v_component[current_index] = calculate_v_component_value(
direction, speed)
wind_speed_iter.iternext()
# Stick the original data into the meta data
wind_speed_meta = the_tile_data.meta_data.add()
wind_speed_meta.name = 'wind_speed'
wind_speed_meta.meta_data.CopyFrom(to_shaped_array(wind_speed))
# The u_or_v variable specifies which component variable is the 'data variable' for this tile
# Replace data with the appropriate component value and put the other component in metadata
if self.u_or_v == U_OR_V_ENUM.U:
the_tile_data.variable_data.CopyFrom(
to_shaped_array(wind_u_component))
wind_component_meta = the_tile_data.meta_data.add()
wind_component_meta.name = 'wind_v'
wind_component_meta.meta_data.CopyFrom(
to_shaped_array(wind_v_component))
elif self.u_or_v == U_OR_V_ENUM.V:
the_tile_data.variable_data.CopyFrom(
to_shaped_array(wind_v_component))
wind_component_meta = the_tile_data.meta_data.add()
wind_component_meta.name = 'wind_u'
wind_component_meta.meta_data.CopyFrom(
to_shaped_array(wind_u_component))
yield nexus_tile
def read_data(self, tile_specifications, file_path, output_tile):
with Dataset(file_path) as ds:
for section_spec, dimtoslice in tile_specifications:
tile = nexusproto.SwathTile()
# Time Lat Long Data and metadata should all be indexed by the same dimensions, order the incoming spec once using the data variable
ordered_slices = get_ordered_slices(ds, self.variable_to_read,
dimtoslice)
tile.latitude.CopyFrom(
to_shaped_array(
numpy.ma.filled(
ds[self.latitude][tuple(ordered_slices.values())],
numpy.NaN)))
tile.longitude.CopyFrom(
to_shaped_array(
numpy.ma.filled(
ds[self.longitude][tuple(ordered_slices.values())],
numpy.NaN)))
timetile = ds[self.time][tuple([
ordered_slices[time_dim]
for time_dim in ds[self.time].dimensions
])].astype('float64', casting='same_kind', copy=False)
timeunits = ds[self.time].getncattr('units')
try:
start_of_day_date = datetime.datetime.strptime(
ds.getncattr(self.start_of_day),
self.start_of_day_pattern)
except Exception:
start_of_day_date = None
for index in numpy.ndindex(timetile.shape):
timetile[index] = to_seconds_from_epoch(
timetile[index].item(),
timeunits=timeunits,
start_day=start_of_day_date,
timeoffset=self.time_offset)
tile.time.CopyFrom(to_shaped_array(timetile))
# Read the data converting masked values to NaN
data_array = numpy.ma.filled(
ds[self.variable_to_read][tuple(ordered_slices.values())],
numpy.NaN)
tile.variable_data.CopyFrom(to_shaped_array(data_array))
if self.metadata is not None:
tile.meta_data.add().CopyFrom(
to_metadata(
self.metadata,
ds[self.metadata][tuple(ordered_slices.values())]))
output_tile.tile.swath_tile.CopyFrom(tile)
yield output_tile
def transform(self, tile_data):
nexus_tile = nexusproto.NexusTile.FromString(tile_data)
the_tile_type = nexus_tile.tile.WhichOneof("tile_type")
the_tile_data = getattr(nexus_tile.tile, the_tile_type)
# Either wind_u or wind_v are in meta. Whichever is not in meta is in variable_data
try:
wind_v = next(meta for meta in the_tile_data.meta_data
if meta.name == wind_v_var_name).meta_data
wind_u = the_tile_data.variable_data
except StopIteration:
try:
wind_u = next(meta for meta in the_tile_data.meta_data
if meta.name == wind_u_var_name).meta_data
wind_v = the_tile_data.variable_data
except StopIteration:
if hasattr(nexus_tile, "summary"):
raise RuntimeError(
"Neither wind_u nor wind_v were found in the meta data for granule %s slice %s."
" Cannot compute wind speed or direction." %
(getattr(nexus_tile.summary, "granule", "unknown"),
getattr(nexus_tile.summary, "section_spec", "unknown")))
else:
raise RuntimeError(
"Neither wind_u nor wind_v were found in the meta data. Cannot compute wind speed or direction."
)
wind_u = from_shaped_array(wind_u)
wind_v = from_shaped_array(wind_v)
assert wind_u.shape == wind_v.shape
# Do calculation
wind_speed_data, wind_dir_data = calculate_speed_direction(wind_u, wind_v)
# Add wind_speed to meta data
wind_speed_meta = the_tile_data.meta_data.add()
wind_speed_meta.name = 'wind_speed'
wind_speed_meta.meta_data.CopyFrom(to_shaped_array(wind_speed_data))
# Add wind_dir to meta data
wind_dir_meta = the_tile_data.meta_data.add()
wind_dir_meta.name = 'wind_dir'
wind_dir_meta.meta_data.CopyFrom(to_shaped_array(wind_dir_data))
yield nexus_tile.SerializeToString()
def read_data(self, tile_specifications, file_path, output_tile):
# Time is optional for Grid data
time = self.environ['TIME']
with Dataset(file_path) as ds:
for section_spec, dimtoslice in tile_specifications:
tile = nexusproto.GridTile()
tile.latitude.CopyFrom(
to_shaped_array(
numpy.ma.filled(
ds[self.latitude][dimtoslice[self.latitude]],
numpy.NaN)))
tile.longitude.CopyFrom(
to_shaped_array(
numpy.ma.filled(
ds[self.longitude][dimtoslice[self.longitude]],
numpy.NaN)))
# Before we read the data we need to make sure the dimensions are in the proper order so we don't have any
# indexing issues
ordered_slices = get_ordered_slices(ds, self.variable_to_read,
dimtoslice)
# Read data using the ordered slices, replacing masked values with NaN
data_array = numpy.ma.filled(
ds[self.variable_to_read][tuple(ordered_slices.values())],
numpy.NaN)
tile.variable_data.CopyFrom(to_shaped_array(data_array))
if self.metadata is not None:
tile.meta_data.add().CopyFrom(
to_metadata(
self.metadata,
ds[self.metadata][tuple(ordered_slices.values())]))
if time is not None:
timevar = ds[time]
# Note assumption is that index of time is start value in dimtoslice
tile.time = to_seconds_from_epoch(
timevar[dimtoslice[time].start],
timeunits=timevar.getncattr('units'),
timeoffset=self.time_offset)
output_tile.tile.grid_tile.CopyFrom(tile)
yield output_tile
def read_swath_data(self, section_spec_dataset):
tile_specifications, file_path = parse_input(section_spec_dataset)
# Time is required for swath data
time = environ['TIME']
with Dataset(file_path) as ds:
for section_spec, dimtoslice in tile_specifications:
tile = nexusproto.SwathTile()
# Time Lat Long Data and metadata should all be indexed by the same dimensions, order the incoming spec once using the data variable
ordered_slices = get_ordered_slices(ds, variable_to_read,
dimtoslice)
tile.latitude.CopyFrom(
to_shaped_array(ds[latitude][tuple(
ordered_slices.itervalues())]))
tile.longitude.CopyFrom(
to_shaped_array(ds[longitude][tuple(
ordered_slices.itervalues())]))
timeunits = ds[time].getncattr('units')
timetile = ds[time][tuple(ordered_slices.itervalues())]
for index in numpy.ndindex(timetile.shape):
timetile[index] = to_seconds_from_epoch(
timetile[index].item(), timeunits)
tile.time.CopyFrom(to_shaped_array(timetile))
# Read the data converting masked values to NaN
data_array = numpy.ma.filled(
ds[variable_to_read][tuple(ordered_slices.itervalues())],
numpy.NaN)
tile.variable_data.CopyFrom(to_shaped_array(data_array))
if metadata is not None:
tile.meta_data.add().CopyFrom(
to_metadata(
metadata,
ds[metadata][tuple(ordered_slices.itervalues())]))
nexus_tile = new_nexus_tile(file_path, section_spec)
nexus_tile.tile.swath_tile.CopyFrom(tile)
yield nexus_tile.SerializeToString()
# If temp dir is defined, delete the temporary file
if temp_dir is not None:
remove(file_path)
def process_nexus_tile(self, nexus_tile):
the_tile_type = nexus_tile.tile.WhichOneof("tile_type")
the_tile_data = getattr(nexus_tile.tile, the_tile_type)
var_data = from_shaped_array(the_tile_data.variable_data) - 273.15
the_tile_data.variable_data.CopyFrom(to_shaped_array(var_data))
yield nexus_tile
def _generate_tile(self, ds: xr.Dataset,
dimensions_to_slices: Dict[str, slice], input_tile):
new_tile = nexusproto.GridTile()
lat_subset = ds[self.latitude][type(self)._slices_for_variable(
ds[self.latitude], dimensions_to_slices)]
lon_subset = ds[self.longitude][type(self)._slices_for_variable(
ds[self.longitude], dimensions_to_slices)]
lat_subset = np.ma.filled(np.squeeze(lat_subset), np.NaN)
lon_subset = np.ma.filled(np.squeeze(lon_subset), np.NaN)
data_subset = ds[self.variable][type(self)._slices_for_variable(
ds[self.variable], dimensions_to_slices)]
data_subset = np.ma.filled(np.squeeze(data_subset), np.NaN)
if self.depth:
depth_dim, depth_slice = list(
type(self)._slices_for_variable(
ds[self.depth], dimensions_to_slices).items())[0]
depth_slice_len = depth_slice.stop - depth_slice.start
if depth_slice_len > 1:
raise RuntimeError(
"Depth slices must have length 1, but '{dim}' has length {dim_len}."
.format(dim=depth_dim, dim_len=depth_slice_len))
new_tile.depth = ds[self.depth][depth_slice].item()
if self.time:
time_slice = dimensions_to_slices[self.time]
time_slice_len = time_slice.stop - time_slice.start
if time_slice_len > 1:
raise RuntimeError(
"Time slices must have length 1, but '{dim}' has length {dim_len}."
.format(dim=self.time, dim_len=time_slice_len))
new_tile.time = int(ds[self.time][time_slice.start].item() / 1e9)
new_tile.latitude.CopyFrom(to_shaped_array(lat_subset))
new_tile.longitude.CopyFrom(to_shaped_array(lon_subset))
new_tile.variable_data.CopyFrom(to_shaped_array(data_subset))
input_tile.tile.grid_tile.CopyFrom(new_tile)
return input_tile
def transform(self, tile_data):
nexus_tile = nexusproto.NexusTile.FromString(tile_data)
the_tile_type = nexus_tile.tile.WhichOneof("tile_type")
the_tile_data = getattr(nexus_tile.tile, the_tile_type)
longitudes = from_shaped_array(the_tile_data.longitude) - 180
the_tile_data.longitude.CopyFrom(to_shaped_array(longitudes))
yield nexus_tile.SerializeToString()
def transform(self, tile_data):
nexus_tile = nexusproto.NexusTile.FromString(tile_data)
the_tile_type = nexus_tile.tile.WhichOneof("tile_type")
the_tile_data = getattr(nexus_tile.tile, the_tile_type)
var_data = from_shaped_array(the_tile_data.variable_data) - 273.15
the_tile_data.variable_data.CopyFrom(to_shaped_array(var_data))
yield nexus_tile.SerializeToString()
def process(self, tile, *args, **kwargs):
"""
This method will transform longitude values in degrees_east from 0 TO 360 to -180 to 180
:param self:
:param nexus_tile: The nexus_tile
:return: Tile data with altered longitude values
"""
the_tile_type = tile.tile.WhichOneof("tile_type")
the_tile_data = getattr(tile.tile, the_tile_type)
longitudes = from_shaped_array(the_tile_data.longitude)
# Only subtract 360 if the longitude is greater than 180
longitudes[longitudes > 180] -= 360
the_tile_data.longitude.CopyFrom(to_shaped_array(longitudes))
return tile
def process_nexus_tile(self, nexus_tile):
axis = [
x.split(':')[0] for x in nexus_tile.summary.section_spec.split(',')
].index(self.dimension)
the_tile_type = nexus_tile.tile.WhichOneof("tile_type")
the_tile_data = getattr(nexus_tile.tile, the_tile_type)
var_data = from_shaped_array(the_tile_data.variable_data)
if numpy.size(var_data, axis) == 1:
the_tile_data.variable_data.CopyFrom(
to_shaped_array(numpy.squeeze(var_data, axis=axis)))
else:
raise RuntimeError(
"Cannot delete axis for dimension %s because length is not 1."
% self.dimension)
yield nexus_tile
def transform(self, tile_data):
"""
This method will transform longitude values in degrees_east from 0 TO 360 to -180 to 180
:param self:
:param tile_data: The tile data
:return: Tile data with altered longitude values
"""
nexus_tile = nexusproto.NexusTile.FromString(tile_data)
the_tile_type = nexus_tile.tile.WhichOneof("tile_type")
the_tile_data = getattr(nexus_tile.tile, the_tile_type)
longitudes = from_shaped_array(the_tile_data.longitude)
# Only subtract 360 if the longitude is greater than 180
longitudes[longitudes > 180] -= 360
the_tile_data.longitude.CopyFrom(to_shaped_array(longitudes))
yield nexus_tile.SerializeToString()
