def process(self, tile, *args, **kwargs): """ This method will reverse the ordering of latitude values in a tile if necessary to ensure that the latitude values are ascending. :param self: :param tile: The nexus_tile :return: Tile data with altered latitude values """ the_tile_type = tile.tile.WhichOneof("tile_type") the_tile_data = getattr(tile.tile, the_tile_type) latitudes = from_shaped_array(the_tile_data.latitude) data = from_shaped_array(the_tile_data.variable_data) # Only reverse latitude ordering if current ordering is descending. if len(latitudes) > 1: delta = latitudes[1] - latitudes[0] if delta < 0: latitudes = np.flip(latitudes) data = np.flip(data, axis=0) the_tile_data.latitude.CopyFrom(to_shaped_array(latitudes)) the_tile_data.variable_data.CopyFrom(to_shaped_array(data)) return tile
def read_data(self, tile_specifications, file_path, output_tile): with Dataset(file_path) as ds: for section_spec, dimtoslice in tile_specifications: tile = nexusproto.TimeSeriesTile() instance_dimension = next( iter([dim for dim in ds[self.variable_to_read].dimensions if dim != self.time])) tile.latitude.CopyFrom( to_shaped_array(numpy.ma.filled(ds[self.latitude][dimtoslice[instance_dimension]], numpy.NaN))) tile.longitude.CopyFrom( to_shaped_array(numpy.ma.filled(ds[self.longitude][dimtoslice[instance_dimension]], numpy.NaN))) # Before we read the data we need to make sure the dimensions are in the proper order so we don't # have any indexing issues ordered_slices = get_ordered_slices(ds, self.variable_to_read, dimtoslice) # Read data using the ordered slices, replacing masked values with NaN data_array = numpy.ma.filled(ds[self.variable_to_read][tuple(ordered_slices.values())], numpy.NaN) tile.variable_data.CopyFrom(to_shaped_array(data_array)) if self.metadata is not None: tile.meta_data.add().CopyFrom( to_metadata(self.metadata, ds[self.metadata][tuple(ordered_slices.values())])) tile.time.CopyFrom( to_shaped_array(numpy.ma.filled(ds[self.time][dimtoslice[self.time]], numpy.NaN))) output_tile.tile.time_series_tile.CopyFrom(tile) yield output_tile
def read_grid_data(self, section_spec_dataset): tile_specifications, file_path = parse_input(section_spec_dataset) # Time is optional for Grid data try: time = environ['TIME'] except KeyError: time = None with Dataset(file_path) as ds: for section_spec, dimtoslice in tile_specifications: tile = nexusproto.GridTile() tile.latitude.CopyFrom( to_shaped_array( numpy.ma.filled(ds[latitude][dimtoslice[latitude]], numpy.NaN))) tile.longitude.CopyFrom( to_shaped_array( numpy.ma.filled(ds[longitude][dimtoslice[longitude]], numpy.NaN))) # Before we read the data we need to make sure the dimensions are in the proper order so we don't have any # indexing issues ordered_slices = get_ordered_slices(ds, variable_to_read, dimtoslice) # Read data using the ordered slices, replacing masked values with NaN data_array = numpy.ma.filled( ds[variable_to_read][tuple(ordered_slices.itervalues())], numpy.NaN) tile.variable_data.CopyFrom(to_shaped_array(data_array)) if metadata is not None: tile.meta_data.add().CopyFrom( to_metadata( metadata, ds[metadata][tuple(ordered_slices.itervalues())])) if time is not None: timevar = ds[time] # Note assumption is that index of time is start value in dimtoslice tile.time = to_seconds_from_epoch( timevar[dimtoslice[time].start], timeunits=timevar.getncattr('units'), timeoffset=time_offset) nexus_tile = new_nexus_tile(file_path, section_spec) nexus_tile.tile.grid_tile.CopyFrom(tile) yield nexus_tile.SerializeToString() # If temp dir is defined, delete the temporary file if temp_dir is not None: remove(file_path)
def process_nexus_tile(self, nexus_tile): the_tile_type = nexus_tile.tile.WhichOneof("tile_type") the_tile_data = getattr(nexus_tile.tile, the_tile_type) wind_speed = from_shaped_array(the_tile_data.variable_data) wind_dir = from_shaped_array( next(meta for meta in the_tile_data.meta_data if meta.name == 'wind_dir').meta_data) assert wind_speed.shape == wind_dir.shape wind_u_component = numpy.ma.empty(wind_speed.shape, dtype=float) wind_v_component = numpy.ma.empty(wind_speed.shape, dtype=float) wind_speed_iter = numpy.nditer(wind_speed, flags=['multi_index']) while not wind_speed_iter.finished: speed = wind_speed_iter[0] current_index = wind_speed_iter.multi_index direction = wind_dir[current_index] # Convert degrees to radians direction = radians(direction) # Calculate component values wind_u_component[current_index] = calculate_u_component_value( direction, speed) wind_v_component[current_index] = calculate_v_component_value( direction, speed) wind_speed_iter.iternext() # Stick the original data into the meta data wind_speed_meta = the_tile_data.meta_data.add() wind_speed_meta.name = 'wind_speed' wind_speed_meta.meta_data.CopyFrom(to_shaped_array(wind_speed)) # The u_or_v variable specifies which component variable is the 'data variable' for this tile # Replace data with the appropriate component value and put the other component in metadata if self.u_or_v == U_OR_V_ENUM.U: the_tile_data.variable_data.CopyFrom( to_shaped_array(wind_u_component)) wind_component_meta = the_tile_data.meta_data.add() wind_component_meta.name = 'wind_v' wind_component_meta.meta_data.CopyFrom( to_shaped_array(wind_v_component)) elif self.u_or_v == U_OR_V_ENUM.V: the_tile_data.variable_data.CopyFrom( to_shaped_array(wind_v_component)) wind_component_meta = the_tile_data.meta_data.add() wind_component_meta.name = 'wind_u' wind_component_meta.meta_data.CopyFrom( to_shaped_array(wind_u_component)) yield nexus_tile
def read_data(self, tile_specifications, file_path, output_tile): with Dataset(file_path) as ds: for section_spec, dimtoslice in tile_specifications: tile = nexusproto.SwathTile() # Time Lat Long Data and metadata should all be indexed by the same dimensions, order the incoming spec once using the data variable ordered_slices = get_ordered_slices(ds, self.variable_to_read, dimtoslice) tile.latitude.CopyFrom( to_shaped_array( numpy.ma.filled( ds[self.latitude][tuple(ordered_slices.values())], numpy.NaN))) tile.longitude.CopyFrom( to_shaped_array( numpy.ma.filled( ds[self.longitude][tuple(ordered_slices.values())], numpy.NaN))) timetile = ds[self.time][tuple([ ordered_slices[time_dim] for time_dim in ds[self.time].dimensions ])].astype('float64', casting='same_kind', copy=False) timeunits = ds[self.time].getncattr('units') try: start_of_day_date = datetime.datetime.strptime( ds.getncattr(self.start_of_day), self.start_of_day_pattern) except Exception: start_of_day_date = None for index in numpy.ndindex(timetile.shape): timetile[index] = to_seconds_from_epoch( timetile[index].item(), timeunits=timeunits, start_day=start_of_day_date, timeoffset=self.time_offset) tile.time.CopyFrom(to_shaped_array(timetile)) # Read the data converting masked values to NaN data_array = numpy.ma.filled( ds[self.variable_to_read][tuple(ordered_slices.values())], numpy.NaN) tile.variable_data.CopyFrom(to_shaped_array(data_array)) if self.metadata is not None: tile.meta_data.add().CopyFrom( to_metadata( self.metadata, ds[self.metadata][tuple(ordered_slices.values())])) output_tile.tile.swath_tile.CopyFrom(tile) yield output_tile
def transform(self, tile_data): nexus_tile = nexusproto.NexusTile.FromString(tile_data) the_tile_type = nexus_tile.tile.WhichOneof("tile_type") the_tile_data = getattr(nexus_tile.tile, the_tile_type) # Either wind_u or wind_v are in meta. Whichever is not in meta is in variable_data try: wind_v = next(meta for meta in the_tile_data.meta_data if meta.name == wind_v_var_name).meta_data wind_u = the_tile_data.variable_data except StopIteration: try: wind_u = next(meta for meta in the_tile_data.meta_data if meta.name == wind_u_var_name).meta_data wind_v = the_tile_data.variable_data except StopIteration: if hasattr(nexus_tile, "summary"): raise RuntimeError( "Neither wind_u nor wind_v were found in the meta data for granule %s slice %s." " Cannot compute wind speed or direction." % (getattr(nexus_tile.summary, "granule", "unknown"), getattr(nexus_tile.summary, "section_spec", "unknown"))) else: raise RuntimeError( "Neither wind_u nor wind_v were found in the meta data. Cannot compute wind speed or direction." ) wind_u = from_shaped_array(wind_u) wind_v = from_shaped_array(wind_v) assert wind_u.shape == wind_v.shape # Do calculation wind_speed_data, wind_dir_data = calculate_speed_direction(wind_u, wind_v) # Add wind_speed to meta data wind_speed_meta = the_tile_data.meta_data.add() wind_speed_meta.name = 'wind_speed' wind_speed_meta.meta_data.CopyFrom(to_shaped_array(wind_speed_data)) # Add wind_dir to meta data wind_dir_meta = the_tile_data.meta_data.add() wind_dir_meta.name = 'wind_dir' wind_dir_meta.meta_data.CopyFrom(to_shaped_array(wind_dir_data)) yield nexus_tile.SerializeToString()
def read_data(self, tile_specifications, file_path, output_tile): # Time is optional for Grid data time = self.environ['TIME'] with Dataset(file_path) as ds: for section_spec, dimtoslice in tile_specifications: tile = nexusproto.GridTile() tile.latitude.CopyFrom( to_shaped_array( numpy.ma.filled( ds[self.latitude][dimtoslice[self.latitude]], numpy.NaN))) tile.longitude.CopyFrom( to_shaped_array( numpy.ma.filled( ds[self.longitude][dimtoslice[self.longitude]], numpy.NaN))) # Before we read the data we need to make sure the dimensions are in the proper order so we don't have any # indexing issues ordered_slices = get_ordered_slices(ds, self.variable_to_read, dimtoslice) # Read data using the ordered slices, replacing masked values with NaN data_array = numpy.ma.filled( ds[self.variable_to_read][tuple(ordered_slices.values())], numpy.NaN) tile.variable_data.CopyFrom(to_shaped_array(data_array)) if self.metadata is not None: tile.meta_data.add().CopyFrom( to_metadata( self.metadata, ds[self.metadata][tuple(ordered_slices.values())])) if time is not None: timevar = ds[time] # Note assumption is that index of time is start value in dimtoslice tile.time = to_seconds_from_epoch( timevar[dimtoslice[time].start], timeunits=timevar.getncattr('units'), timeoffset=self.time_offset) output_tile.tile.grid_tile.CopyFrom(tile) yield output_tile
def read_swath_data(self, section_spec_dataset): tile_specifications, file_path = parse_input(section_spec_dataset) # Time is required for swath data time = environ['TIME'] with Dataset(file_path) as ds: for section_spec, dimtoslice in tile_specifications: tile = nexusproto.SwathTile() # Time Lat Long Data and metadata should all be indexed by the same dimensions, order the incoming spec once using the data variable ordered_slices = get_ordered_slices(ds, variable_to_read, dimtoslice) tile.latitude.CopyFrom( to_shaped_array(ds[latitude][tuple( ordered_slices.itervalues())])) tile.longitude.CopyFrom( to_shaped_array(ds[longitude][tuple( ordered_slices.itervalues())])) timeunits = ds[time].getncattr('units') timetile = ds[time][tuple(ordered_slices.itervalues())] for index in numpy.ndindex(timetile.shape): timetile[index] = to_seconds_from_epoch( timetile[index].item(), timeunits) tile.time.CopyFrom(to_shaped_array(timetile)) # Read the data converting masked values to NaN data_array = numpy.ma.filled( ds[variable_to_read][tuple(ordered_slices.itervalues())], numpy.NaN) tile.variable_data.CopyFrom(to_shaped_array(data_array)) if metadata is not None: tile.meta_data.add().CopyFrom( to_metadata( metadata, ds[metadata][tuple(ordered_slices.itervalues())])) nexus_tile = new_nexus_tile(file_path, section_spec) nexus_tile.tile.swath_tile.CopyFrom(tile) yield nexus_tile.SerializeToString() # If temp dir is defined, delete the temporary file if temp_dir is not None: remove(file_path)
def process_nexus_tile(self, nexus_tile): the_tile_type = nexus_tile.tile.WhichOneof("tile_type") the_tile_data = getattr(nexus_tile.tile, the_tile_type) var_data = from_shaped_array(the_tile_data.variable_data) - 273.15 the_tile_data.variable_data.CopyFrom(to_shaped_array(var_data)) yield nexus_tile
def _generate_tile(self, ds: xr.Dataset, dimensions_to_slices: Dict[str, slice], input_tile): new_tile = nexusproto.GridTile() lat_subset = ds[self.latitude][type(self)._slices_for_variable( ds[self.latitude], dimensions_to_slices)] lon_subset = ds[self.longitude][type(self)._slices_for_variable( ds[self.longitude], dimensions_to_slices)] lat_subset = np.ma.filled(np.squeeze(lat_subset), np.NaN) lon_subset = np.ma.filled(np.squeeze(lon_subset), np.NaN) data_subset = ds[self.variable][type(self)._slices_for_variable( ds[self.variable], dimensions_to_slices)] data_subset = np.ma.filled(np.squeeze(data_subset), np.NaN) if self.depth: depth_dim, depth_slice = list( type(self)._slices_for_variable( ds[self.depth], dimensions_to_slices).items())[0] depth_slice_len = depth_slice.stop - depth_slice.start if depth_slice_len > 1: raise RuntimeError( "Depth slices must have length 1, but '{dim}' has length {dim_len}." .format(dim=depth_dim, dim_len=depth_slice_len)) new_tile.depth = ds[self.depth][depth_slice].item() if self.time: time_slice = dimensions_to_slices[self.time] time_slice_len = time_slice.stop - time_slice.start if time_slice_len > 1: raise RuntimeError( "Time slices must have length 1, but '{dim}' has length {dim_len}." .format(dim=self.time, dim_len=time_slice_len)) new_tile.time = int(ds[self.time][time_slice.start].item() / 1e9) new_tile.latitude.CopyFrom(to_shaped_array(lat_subset)) new_tile.longitude.CopyFrom(to_shaped_array(lon_subset)) new_tile.variable_data.CopyFrom(to_shaped_array(data_subset)) input_tile.tile.grid_tile.CopyFrom(new_tile) return input_tile
def transform(self, tile_data): nexus_tile = nexusproto.NexusTile.FromString(tile_data) the_tile_type = nexus_tile.tile.WhichOneof("tile_type") the_tile_data = getattr(nexus_tile.tile, the_tile_type) longitudes = from_shaped_array(the_tile_data.longitude) - 180 the_tile_data.longitude.CopyFrom(to_shaped_array(longitudes)) yield nexus_tile.SerializeToString()
def transform(self, tile_data): nexus_tile = nexusproto.NexusTile.FromString(tile_data) the_tile_type = nexus_tile.tile.WhichOneof("tile_type") the_tile_data = getattr(nexus_tile.tile, the_tile_type) var_data = from_shaped_array(the_tile_data.variable_data) - 273.15 the_tile_data.variable_data.CopyFrom(to_shaped_array(var_data)) yield nexus_tile.SerializeToString()
def process(self, tile, *args, **kwargs): """ This method will transform longitude values in degrees_east from 0 TO 360 to -180 to 180 :param self: :param nexus_tile: The nexus_tile :return: Tile data with altered longitude values """ the_tile_type = tile.tile.WhichOneof("tile_type") the_tile_data = getattr(tile.tile, the_tile_type) longitudes = from_shaped_array(the_tile_data.longitude) # Only subtract 360 if the longitude is greater than 180 longitudes[longitudes > 180] -= 360 the_tile_data.longitude.CopyFrom(to_shaped_array(longitudes)) return tile
def process_nexus_tile(self, nexus_tile): axis = [ x.split(':')[0] for x in nexus_tile.summary.section_spec.split(',') ].index(self.dimension) the_tile_type = nexus_tile.tile.WhichOneof("tile_type") the_tile_data = getattr(nexus_tile.tile, the_tile_type) var_data = from_shaped_array(the_tile_data.variable_data) if numpy.size(var_data, axis) == 1: the_tile_data.variable_data.CopyFrom( to_shaped_array(numpy.squeeze(var_data, axis=axis))) else: raise RuntimeError( "Cannot delete axis for dimension %s because length is not 1." % self.dimension) yield nexus_tile
def transform(self, tile_data): """ This method will transform longitude values in degrees_east from 0 TO 360 to -180 to 180 :param self: :param tile_data: The tile data :return: Tile data with altered longitude values """ nexus_tile = nexusproto.NexusTile.FromString(tile_data) the_tile_type = nexus_tile.tile.WhichOneof("tile_type") the_tile_data = getattr(nexus_tile.tile, the_tile_type) longitudes = from_shaped_array(the_tile_data.longitude) # Only subtract 360 if the longitude is greater than 180 longitudes[longitudes > 180] -= 360 the_tile_data.longitude.CopyFrom(to_shaped_array(longitudes)) yield nexus_tile.SerializeToString()