def test_datashader_image_nan(): '''Test the datashader image stretch function works correctly if the inputs 2D arraysi containing NaNs in both coordinates and values. Tests the modification made in commit 05962ec''' x = numpy.array( [[numpy.nan, numpy.nan, numpy.nan, numpy.nan, numpy.nan, numpy.nan], [numpy.nan, numpy.nan, numpy.nan, numpy.nan, numpy.nan, numpy.nan], [numpy.nan, numpy.nan, numpy.nan, numpy.nan, numpy.nan, numpy.nan], [-32.12598, -32.07721, -32.02848, numpy.nan, numpy.nan, numpy.nan], [-32.15292, -32.104084, -32.055298, numpy.nan, numpy.nan, numpy.nan], [-32.17996, -32.13106, -32.08221, numpy.nan, numpy.nan, numpy.nan]]) y = numpy.array( [[numpy.nan, numpy.nan, numpy.nan, numpy.nan, numpy.nan, numpy.nan], [numpy.nan, numpy.nan, numpy.nan, numpy.nan, numpy.nan, numpy.nan], [numpy.nan, numpy.nan, numpy.nan, numpy.nan, numpy.nan, numpy.nan], [-40.60027, -40.596905, -40.593544, numpy.nan, numpy.nan, numpy.nan], [-40.643642, -40.640266, -40.636898, numpy.nan, numpy.nan, numpy.nan], [-40.68706, -40.683674, -40.680298, numpy.nan, numpy.nan, numpy.nan]]) z = numpy.array( [[numpy.nan, numpy.nan, numpy.nan, numpy.nan, numpy.nan, numpy.nan], [numpy.nan, numpy.nan, numpy.nan, numpy.nan, numpy.nan, numpy.nan], [276.23, 273.04, 270.75, numpy.nan, numpy.nan, numpy.nan], [277.12, 273.55, 270.82, numpy.nan, numpy.nan, numpy.nan], [numpy.nan, 273.24, 270.16998, numpy.nan, numpy.nan, numpy.nan], [numpy.nan, numpy.nan, numpy.nan, numpy.nan, numpy.nan, numpy.nan]]) result = geo.stretch_image(x, y, z) #this will fail on a ValueError if the NaNs above are handled improperly serialize_json(result) #Should be returning NumPy masked arrays assert numpy.ma.is_masked(result['image'][0])
def load_image_pts(path, variable, pts_3d, pts_4d): key = (path, variable, pts_hash(pts_3d), pts_hash(pts_4d)) if key in IMAGES: return IMAGES[key] else: try: lons, lats, values, units = _load_netcdf4(path, variable, pts_3d, pts_4d) except: lons, lats, values, units = _load_cube(path, variable, pts_3d, pts_4d) # Units if variable in ["precipitation_flux", "stratiform_rainfall_rate"]: if units == "mm h-1": values = values else: values = convert_units(values, units, "kg m-2 hour-1") elif units == "K": values = convert_units(values, "K", "Celsius") # Coarsify images threshold = 200 * 200 # Chosen since TMA WRF is 199 x 199 if values.size > threshold: fraction = 0.25 else: fraction = 1. lons, lats, values = coarsify(lons, lats, values, fraction) image = geo.stretch_image(lons, lats, values) IMAGES[key] = image return image
def load_image(self, path, itime): lons = self.longitudes lats = self.latitudes with netCDF4.Dataset(path) as dataset: values = dataset.variables["data"][itime] fraction = 0.25 lons, lats, values = coarsify(lons, lats, values, fraction) return geo.stretch_image(lons, lats, values)
def image(self, state): '''gets actual data. X and Y passed to :meth:`geo.stretch_image` must be 1D arrays. NWCSAF data are not on a regular grid so must be regridded. `values` passed to :meth:`geo.stretch_image` must be a NumPy Masked Array. :param state: Bokeh State object of info from UI :returns: Output data from :meth:`geo.stretch_image`''' data = empty_image() for nc in self.locator._sets: if str( datetime.datetime.strptime( nc.nominal_product_time.replace('Z', 'UTC'), '%Y-%m-%dT%H:%M:%S%Z') ) == state.valid_time and self.locator.varlist[ state.variable] in nc.variables: #regrid to regular grid x = nc['lon'][:].flatten() # lat & lon both 2D arrays y = nc['lat'][:].flatten() # z = nc[self.locator.varlist[state.variable]][:].flatten() #define grid xi, yi = np.meshgrid( np.linspace(x.min(), x.max(), nc.dimensions['nx'].size), np.linspace(y.min(), y.max(), nc.dimensions['ny'].size), ) zi = griddata(np.array([x, y]).transpose(), z, (xi, yi), method='linear', fill_value=np.nan) zi = np.ma.masked_invalid(zi, copy=False) zi = np.ma.masked_outside( zi, nc[self.locator.varlist[state.variable]].valid_range[0], nc[self.locator.varlist[state.variable]].valid_range[1], copy=False) data = geo.stretch_image(xi[0, :], yi[:, 0], zi) #data = geo.stretch_image(x[0,:], y[:,0], nc[state.variable][:]) data.update( coordinates(state.valid_time, state.initial_time, state.pressures, state.pressure)) data.update({ 'name': [str(nc[self.locator.varlist[state.variable]].long_name)], }) if 'units' in nc[self.locator.varlist[ state.variable]].ncattrs(): data.update({ 'units': [str(nc[self.locator.varlist[state.variable]].units)] }) return data
def load_image(self, path, itime): lons = self.longitudes lats = self.latitudes with xarray.open_dataset(path, engine=ENGINE) as nc: values = nc["data"][itime].values # Use datashader to coarsify images from 4.4km to 8.8km grid scale = 2 return geo.stretch_image(lons, lats, values, plot_width=int(values.shape[1] / scale), plot_height=int(values.shape[0] / scale))
def image(self, state): cube = self._cubes[state.variable] valid_datetime = _to_datetime(state.valid_time) cube = self.extract_cube(cube, valid_datetime) if cube is None: data = empty_image() else: data = geo.stretch_image( cube.coord('longitude').points, cube.coord('latitude').points, cube.data) data.update( coordinates(state.valid_time, state.initial_time, state.pressures, state.pressure)) data.update({'name': [self._label], 'units': [str(cube.units)]}) return data
def _image(self, long_name, initial_time, valid_time, pressures, pressure): data = empty_image() paths = self.locator.glob() long_name_to_variable = self.locator.long_name_to_variable(paths) frequency = dt.timedelta(minutes=15) # TODO: Support arbitrary frequencies for path in self.locator.find_paths(paths, valid_time, frequency): with xarray.open_dataset(path) as nc: if long_name not in long_name_to_variable: continue x = np.ma.masked_invalid(nc['lon'])[:] y = np.ma.masked_invalid(nc['lat'])[:] var = nc[long_name_to_variable[long_name]] z = np.ma.masked_invalid(var)[:] data = geo.stretch_image(x, y, z) data.update(coordinates(valid_time, initial_time, pressures, pressure)) data['name'] = [str(var.long_name)] if 'units' in var.attrs: data['units'] = [str(var.units)] return data
def load_image_pts(path, variable, pts_3d, pts_4d): key = (path, variable, pts_hash(pts_3d), pts_hash(pts_4d)) if key in IMAGES: return IMAGES[key] else: try: lons, lats, values, units = _load_netcdf4(path, variable, pts_3d, pts_4d) except: lons, lats, values, units = _load_cube(path, variable, pts_3d, pts_4d) # Units if variable in ["precipitation_flux", "stratiform_rainfall_rate"]: if units == "mm h-1": values = values else: values = convert_units(values, units, "kg m-2 hour-1") elif units == "K": values = convert_units(values, "K", "Celsius") # Coarsify images threshold = 200 * 200 # Chosen since TMA WRF is 199 x 199 if values.size > threshold: fraction = 0.25 else: fraction = 1. lons, lats, values = coarsify(lons, lats, values, fraction) # Roll input data into [-180, 180] range if np.any(lons > 180.0): shift_by = np.sum(lons > 180.0) lons[lons > 180.0] -= 360. lons = np.roll(lons, shift_by) values = np.roll(values, shift_by, axis=1) image = geo.stretch_image(lons, lats, values) print(image) IMAGES[key] = image return image
def image(self, state): cube = self._cubes[state.variable] valid_datetime = _to_datetime(state.valid_time) cube = cube.extract(iris.Constraint(time=valid_datetime)) if cube is None: data = empty_image() else: data = geo.stretch_image( cube.coord("longitude").points, cube.coord("latitude").points, cube.data, ) data.update( coordinates( state.valid_time, state.initial_time, state.pressures, state.pressure, )) data.update({"name": [self._label], "units": [str(cube.units)]}) return data
def load_image(cls, path, variable, pts): """Load bokeh image glyph data from file using slices""" try: lons, lats, values, units = cls._load_xarray(path, variable, pts) except: lons, lats, values, units = cls._load_cube(path, variable, pts) # Units if variable in ["precipitation_flux", "stratiform_rainfall_rate"]: if units == "mm h-1": values = values else: values = forest.util.convert_units(values, units, "kg m-2 hour-1") units = "kg m-2 hour-1" elif units == "K": values = forest.util.convert_units(values, "K", "Celsius") units = "C" # Coarsify images threshold = 200 * 200 # Chosen since TMA WRF is 199 x 199 if values.size > threshold: fraction = 0.25 else: fraction = 1. lons, lats, values = forest.util.coarsify( lons, lats, values, fraction) # Roll input data into [-180, 180] range if np.any(lons > 180.0): shift_by = np.sum(lons > 180.0) lons[lons > 180.0] -= 360. lons = np.roll(lons, shift_by) values = np.roll(values, shift_by, axis=1) data = geo.stretch_image(lons, lats, values) data["units"] = [units] return data
def _get_bokeh_image( cube, experiment_id, variable_id, institution_id, initial_time, member_id, selected_time, pressure, ): """ A helper function to do the creation of the image dict required by bokeh. This includes downloading the actual data required for the current view, so this function is cached to reduce remote queries. """ def time_comp(select_time, time_cell): # data_time = util.to_datetime(time_cell.point) if abs((select_time - data_time).days) < 2: return True return False def lat_filter(lat): """ Due to the way the current projection of gridded data works, the poles are not well handled, resulting in NaNs if we use the full range of latitudes. The current hack is to chop off latitude greater than 85 degrees north and south. Given the importance of data at the poles in climate change research, we will need to fix this in future. """ return -85.0 < lat < 85.0 def pressure_select(select_pressure, data_pressure): return abs(select_pressure - data_pressure.point) < 1.0 if cube is None or initial_time is None: data = gridded_forecast.empty_image() else: constraint_dict = { 'time': functools.partial(time_comp, selected_time), 'latitude': lat_filter, } coord_names = [c1.name() for c1 in cube.coords()] if 'air_pressure' in coord_names: constraint_dict['air_pressure'] = functools.partial( pressure_select, pressure, ) cube_cropped = cube.extract(iris.Constraint(**constraint_dict)) lat_pts = cube_cropped.coord('latitude').points long_pts = cube_cropped.coord('longitude').points - 180.0 cube_data_cropped = cube_cropped.data cube_width = int(cube_data_cropped.shape[1] / 2) cube_data_cropped = numpy.concatenate([ cube_data_cropped[:, cube_width:], cube_data_cropped[:, :cube_width] ], axis=1) data = geo.stretch_image(long_pts, lat_pts, cube_data_cropped) data['image'] = [ numpy.ma.masked_array(data['image'][0], mask=numpy.isnan(data['image'][0])) ] return data
def load_image(self, imageData): return geo.stretch_image(imageData["longitude"], imageData["latitude"], imageData["data"])