def __init__(self, lons=None, lats=None, nprocs=1):
if type(lons) != type(lats):
raise TypeError('lons and lats must be of same type')
elif lons is not None:
if lons.shape != lats.shape:
raise ValueError('lons and lats must have same shape')
self.nprocs = nprocs
# check the latitutes
if lats is not None and ((lats.min() < -90. or lats.max() > +90.)):
# throw exception
raise ValueError(
'Some latitudes are outside the [-90.;+90] validity range')
else:
self.lats = lats
# check the longitudes
if lons is not None and ((lons.min() < -180. or lons.max() >= +180.)):
# issue warning
warnings.warn('All geometry objects expect longitudes in the [-180:+180[ range. ' +
'We will now automatically wrap your longitudes into [-180:+180[, and continue. ' +
'To avoid this warning next time, use routine utils.wrap_longitudes().')
# wrap longitudes to [-180;+180[
self.lons = utils.wrap_longitudes(lons)
else:
self.lons = lons
self.ndim = None
self.cartesian_coords = None
def open_dataset(fname, dtype='f4', res='C384', tile=1):
"""Reads the binary data for FV3-CHEM input generated by prep_chem_sources.
Parameters
----------
fname : type
Description of parameter `fname`.
**kwargs :
This is the kwargs for the fv3grid. Users can define the res='C384' and
the tile=1.
valid values for them are:
res = 'C768' 'C384' 'C192' 'C96' 'C48'
tile = 1 2 3 4 5 6
Returns
-------
xaray DataArray
Description of returned object.
"""
w = FortranFile(fname)
a = w.read_reals(dtype=dtype)
r = int(res[1:])
s = a.reshape((r, r), order='F')
if has_fv3grid:
grid = fg.get_fv3_grid(res=res, tile=tile)
# grid = grid.set_coords(['latitude', 'longitude', 'grid_lat', 'grid_lon'])
grid['longitude'] = wrap_longitudes(grid.longitude)
# grid = grid.rename({'grid_lat': 'lat_b', 'grid_lon': 'lon_b'})
name = fname.split('.bin')[0]
grid[name] = (('x', 'y'), s)
return grid[name]
else:
return xr.DataArray(s, dims=('x', 'y'))
def test_wrap_longitudes(self):
# test that we indeed wrap to [-180:+180[
step = 60
lons = np.arange(-360,360+step,step)
self.assertTrue((lons.min() < -180) and (lons.max() >= 180) and (+180 in lons))
wlons = utils.wrap_longitudes(lons)
self.assertFalse((wlons.min() < -180) or (wlons.max() >= 180) or (+180 in wlons))
def __init__(self, lons=None, lats=None, nprocs=1):
if type(lons) != type(lats):
raise TypeError('lons and lats must be of same type')
elif lons is not None:
if lons.shape != lats.shape:
raise ValueError('lons and lats must have same shape')
self.nprocs = nprocs
# check the latitutes
if lats is not None and ((lats.min() < -90. or lats.max() > +90.)):
# throw exception
raise ValueError(
'Some latitudes are outside the [-90.;+90] validity range')
else:
self.lats = lats
# check the longitudes
if lons is not None and ((lons.min() < -180. or lons.max() >= +180.)):
# issue warning
warnings.warn('All geometry objects expect longitudes in the [-180:+180[ range. ' +
'We will now automatically wrap your longitudes into [-180:+180[, and continue. ' +
'To avoid this warning next time, use routine utils.wrap_longitudes().')
# wrap longitudes to [-180;+180[
self.lons = utils.wrap_longitudes(lons)
else:
self.lons = lons
self.ndim = None
self.cartesian_coords = None
def test_wrap_longitudes(self):
# test that we indeed wrap to [-180:+180[
step = 60
lons = np.arange(-360, 360 + step, step)
self.assertTrue((lons.min() < -180) and (lons.max() >= 180)
and (+180 in lons))
wlons = utils.wrap_longitudes(lons)
self.assertFalse((wlons.min() < -180) or (wlons.max() >= 180)
or (+180 in wlons))
def __init__(self, lons, lats, nprocs=1):
if lons.shape != lats.shape:
raise ValueError('lon and lat arrays must have same shape')
elif lons.ndim > 2:
raise ValueError('Only 1 and 2 dimensional swaths are allowed')
if lons.shape == lats.shape and lons.dtype == lats.dtype:
self.shape = lons.shape
self.size = lons.size
self.ndim = lons.ndim
self.dtype = lons.dtype
else:
raise ValueError(('%s must be created with either '
'lon/lats of the same shape with same dtype') %
self.__class__.__name__)
if type(lons) != type(lats):
raise TypeError('lons and lats must be of same type')
elif lons is not None:
if lons.shape != lats.shape:
raise ValueError('lons and lats must have same shape')
self.nprocs = nprocs
# check the latitutes
if lats is not None and ((lats.min() < -90. or lats.max() > +90.)):
# throw exception
raise ValueError(
'Some latitudes are outside the [-90.;+90] validity range')
else:
self.lats = lats
# check the longitudes
if lons is not None and ((lons.min() < -180. or lons.max() >= +180.)):
# issue warning
warnings.warn(
'All geometry objects expect longitudes in the [-180:+180[ range. '
+
'We will now automatically wrap your longitudes into [-180:+180[, and continue. '
+
'To avoid this warning next time, use routine utils.wrap_longitudes().'
)
# wrap longitudes to [-180;+180[
self.lons = utils.wrap_longitudes(lons)
else:
self.lons = lons
self.cartesian_coords = None
def open_dataset(fname, dtype="f4", res="C384", tile=1):
"""Reads the binary data for FV3-CHEM input generated by prep_chem_sources.
Parameters
----------
fname : type
Description of parameter `fname`.
**kwargs :
This is the kwargs for the fv3grid. Users can define the res='C384' and
the tile=1.
valid values for them are:
res = 'C768' 'C384' 'C192' 'C96' 'C48'
tile = 1 2 3 4 5 6
Returns
-------
xaray DataArray
Description of returned object.
"""
from pyresample.utils import wrap_longitudes
from scipy.io import FortranFile
w = FortranFile(fname)
a = w.read_reals(dtype=dtype)
r = int(res[1:])
s = a.reshape((r, r), order="F")
if has_fv3grid:
grid = fg.get_fv3_grid(res=res, tile=tile)
# grid = grid.set_coords(['latitude', 'longitude', 'grid_lat', 'grid_lon'])
grid["longitude"] = wrap_longitudes(grid.longitude)
# grid = grid.rename({'grid_lat': 'lat_b', 'grid_lon': 'lon_b'})
name = fname.split(".bin")[0]
grid[name] = (("x", "y"), s)
return grid[name]
else:
print("Please install the fv3grid from https://github.com/bbakernoaa/fv3grid")
print("to gain the full capability of this dataset")
return xr.DataArray(s, dims=("x", "y"))
def open_mfdataset(
fname, earth_radius=6370000, convert_to_ppb=True, var_list=["O3", "PM25"], **kwargs
):
"""Method to open multiple (or single) CESM netcdf files.
This method extends the xarray.open_mfdataset functionality
It is the main method called by the driver. Other functions defined
in this file are internally called by open_mfdataset and are preceeded
by an underscore (e.g. _get_latlon).
Parameters
----------
fname : string or list
fname is the path to the file or files. It will accept wildcards in
strings as well.
earth_radius : float
The earth radius used for map projections
convert_to_ppb : boolean
If true the units of the gas species will be converted to ppbv
and units of aerosols to ug m^-3
var_list : string or list
List of variables to load from the CESM file. Default is to load ozone (O3) and PM2.5 (PM25).
Returns
-------
xarray.DataSet
"""
from pyresample.utils import wrap_longitudes
# check that the files are netcdf format
names, netcdf = _ensure_mfdataset_filenames(fname)
# open the dataset using xarray
try:
if netcdf:
dset_load = xr.open_mfdataset(fname, combine="nested", concat_dim="time", **kwargs)
else:
raise ValueError
except ValueError:
print(
"""File format not recognized. Note that files should be in netcdf
format. Do not mix and match file types."""
)
#############################
# Process the loaded data
# extract variables of choice
dset = dset_load.get(var_list)
# rename altitude dimension to z for monet use
# also rename lon to x and lat to y
dset = dset.rename_dims({"lon": "x", "lat": "y", "lev": "z"})
# convert to -180 to 180 longitude
lon = wrap_longitudes(dset["lon"])
lat = dset["lat"]
lons, lats = meshgrid(lon, lat)
dset.coords["longitude"] = (("y", "x"), lons)
dset.coords["latitude"] = (("y", "x"), lats)
# Set longitude and latitude to be only coordinates
dset = dset.reset_coords()
dset = dset.set_coords(["latitude", "longitude"])
# re-order so surface is associated with the first vertical index
dset = dset.sortby("z", ascending=False)
#############################
# convert units
if convert_to_ppb:
for i in dset.variables:
if "units" in dset[i].attrs:
# convert all gas species from mol/mol to ppbv
if "mol/mol" in dset[i].attrs["units"]:
dset[i] *= 1e09
dset[i].attrs["units"] = "ppbv"
# convert "kg/m3 to \mu g/m3 "
elif "kg/m3" in dset[i].attrs["units"]:
dset[i] *= 1e09
dset[i].attrs["units"] = r"$\mu g m^{-3}$"
return dset
def bathymetry(target_lat, target_lon, blur=None):
CACHE_DIR = current_app.config["CACHE_DIR"]
BATHYMETRY_FILE = current_app.config["BATHYMETRY_FILE"]
fname = str(np.median(target_lat)) + "," + str(np.median(target_lon))
hashed = hashlib.sha1(
"".join(fname + str(target_lat.shape) + str(target_lon.shape)).encode()
).hexdigest()
if _bathymetry_cache.get(hashed) is None:
try:
data = np.load(CACHE_DIR + "/" + hashed + ".npy")
except:
with Dataset(BATHYMETRY_FILE, "r") as ds:
lat = ds.variables["y"]
lon = ds.variables["x"]
z = ds.variables["z"]
def lat_index(v):
return int(round((v - lat[0]) * 60.0))
def lon_index(v):
return int(round((v - lon[0]) * 60.0))
lon_idx_min = target_lon.argmin()
lon_idx_max = target_lon.argmax()
target_lon = wrap_longitudes(target_lon)
minlat = lat_index(np.amin(target_lat))
maxlat = lat_index(np.amax(target_lat))
minlon = lon_index(target_lon.ravel()[lon_idx_min])
maxlon = lon_index(target_lon.ravel()[lon_idx_max])
if minlon > maxlon:
in_lon = np.concatenate((lon[minlon:], lon[0:maxlon]))
in_data = np.concatenate(
(z[minlat:maxlat, minlon:], z[minlat:maxlat, 0:maxlon]), 1
)
else:
in_lon = lon[minlon:maxlon]
in_data = z[minlat:maxlat, minlon:maxlon]
res = in_data.transpose() * -1
lats, lons = np.meshgrid(lat[minlat:maxlat], in_lon)
orig_def = pyresample.geometry.SwathDefinition(lons=lons, lats=lats)
target_def = pyresample.geometry.SwathDefinition(
lons=target_lon.astype(np.float64), lats=target_lat.astype(np.float64)
)
data = pyresample.kd_tree.resample_nearest(
orig_def,
res,
target_def,
radius_of_influence=500000,
fill_value=None,
nprocs=4,
)
def do_save(filename, data):
np.save(filename, data.filled())
if not os.path.isdir(CACHE_DIR):
os.makedirs(CACHE_DIR)
t = threading.Thread(target=do_save, args=(CACHE_DIR + "/" + hashed, data))
t.daemon = True
t.start()
_bathymetry_cache[hashed] = data
else:
data = _bathymetry_cache[hashed]
if blur is not None:
try:
return gaussian_filter(data, sigma=float(blur))
except:
return data
else:
return data
def bathymetry(basemap, target_lat, target_lon, blur=None):
CACHE_DIR = app.config['CACHE_DIR']
BATHYMETRY_FILE = app.config['BATHYMETRY_FILE']
if basemap is None:
fname = str(np.median(target_lat)) + "," + str(np.median(target_lon))
else:
fname = basemap.filename
hashed = hashlib.sha1(fname +
str(target_lat.shape) +
str(target_lon.shape)
).hexdigest()
if _bathymetry_cache.get(hashed) is None:
try:
data = np.load(CACHE_DIR + "/" + hashed + ".npy")
except:
with Dataset(BATHYMETRY_FILE, 'r') as ds:
lat = ds.variables['y']
lon = ds.variables['x']
z = ds.variables['z']
def lat_index(v):
return int(round((v - lat[0]) * 60.0))
def lon_index(v):
return int(round((v - lon[0]) * 60.0))
lon_idx_min = target_lon.argmin()
lon_idx_max = target_lon.argmax()
target_lon = wrap_longitudes(target_lon)
minlat = lat_index(np.amin(target_lat))
maxlat = lat_index(np.amax(target_lat))
minlon = lon_index(target_lon.ravel()[lon_idx_min])
maxlon = lon_index(target_lon.ravel()[lon_idx_max])
if minlon > maxlon:
in_lon = np.concatenate((lon[minlon:], lon[0:maxlon]))
in_data = np.concatenate((z[minlat:maxlat, minlon:],
z[minlat:maxlat, 0:maxlon]), 1)
else:
in_lon = lon[minlon:maxlon]
in_data = z[minlat:maxlat, minlon:maxlon]
res = in_data.transpose() * -1
lats, lons = np.meshgrid(lat[minlat:maxlat], in_lon)
orig_def = pyresample.geometry.SwathDefinition(
lons=lons, lats=lats)
target_def = pyresample.geometry.SwathDefinition(
lons=target_lon.astype(np.float64),
lats=target_lat.astype(np.float64))
data = pyresample.kd_tree.resample_nearest(
orig_def, res,
target_def,
radius_of_influence=500000,
fill_value=None,
nprocs=4)
def do_save(filename, data):
np.save(filename, data.filled())
if not os.path.isdir(CACHE_DIR):
os.makedirs(CACHE_DIR)
t = threading.Thread(
target=do_save, args=(CACHE_DIR + "/" + hashed, data))
t.daemon = True
t.start()
_bathymetry_cache[hashed] = data
else:
data = _bathymetry_cache[hashed]
if blur is not None:
try:
return gaussian_filter(data, sigma=float(blur))
except:
return data
else:
return data
def set_gridlines_info_dict(gridlines_info, area_def):
''' Set the final values for gridlines plotting params, pulling from argument and defaults.
Args:
gridlines_info (dict) : Dictionary of parameters for plotting gridlines.
The following fields are available. If a field is not included in the dictionary,
the field is added to the return dictionary and the default is used.
gridlines_info['grid_lat_spacing'] default auto calculated 5 lat grid lines
gridlines_info['grid_lon_spacing'] default auto calculated 5 lon grid lines
gridlines_info['grid_lat_xoffset'] default None (0.01 * image height)
gridlines_info['grid_lon_xoffset'] default None (0.01 * image width)
gridlines_info['grid_lat_yoffset'] default None (0.01 * image height)
gridlines_info['grid_lon_yoffset'] default None (0.01 * image width)
gridlines_info['grid_lat_fontsize'] default None (plot fontsize)
gridlines_info['grid_lon_fontsize'] default None (plot fontsize)
gridlines_info['left_label'] default True
gridlines_info['right_label'] default False
gridlines_info['top_label'] default True
gridlines_info['bottom_label'] default False
gridlines_info['grid_lat_linewidth'] default 1
gridlines_info['grid_lon_linewidth'] default 1
gridlines_info['grid_lat_color'] default 'black'
gridlines_info['grid_lon_color'] default 'black'
gridlines_info['grid_lat_dashes'] default [4, 2]
gridlines_info['grid_lon_dashes'] default [4, 2]
Returns:
(dict) : gridlines_info dictionary, with fields as specified above.
'''
use_gridlines_info = {}
if gridlines_info is None or 'grid_lat_spacing' not in gridlines_info.keys()\
or 'grid_lon_spacing' not in gridlines_info.keys():
from pyresample import utils
minlat = area_def.area_extent_ll[1]
maxlat = area_def.area_extent_ll[3]
minlon = utils.wrap_longitudes(area_def.area_extent_ll[0])
maxlon = utils.wrap_longitudes(area_def.area_extent_ll[2])
if minlon > maxlon and maxlon < 0:
maxlon = maxlon + 360
lon_extent = maxlon - minlon
lat_extent = maxlat - minlat
if lon_extent > 5:
lon_spacing = int(lon_extent / 5)
elif lon_extent > 2.5:
lon_spacing = 1
else:
lon_spacing = lon_extent / 5.0
if lat_extent > 5:
lat_spacing = int(lat_extent / 5)
elif lat_extent > 2.5:
lat_spacing = 1
else:
lat_spacing = lat_extent / 5.0
LOG.info('lon_extent: %s, lon_spacing: %s', lon_extent, lon_spacing)
use_gridlines_info['grid_lat_spacing'] = lat_spacing
use_gridlines_info['grid_lon_spacing'] = lon_spacing
use_gridlines_info['left_label'] = True
use_gridlines_info['right_label'] = False
use_gridlines_info['top_label'] = True
use_gridlines_info['bottom_label'] = False
use_gridlines_info['grid_lat_linewidth'] = 1
use_gridlines_info['grid_lon_linewidth'] = 1
use_gridlines_info['grid_lat_color'] = 'black'
use_gridlines_info['grid_lon_color'] = 'black'
use_gridlines_info['grid_lat_xoffset'] = None
use_gridlines_info['grid_lon_xoffset'] = None
use_gridlines_info['grid_lat_yoffset'] = None
use_gridlines_info['grid_lon_yoffset'] = None
use_gridlines_info['grid_lat_fontsize'] = None
use_gridlines_info['grid_lon_fontsize'] = None
use_gridlines_info['grid_lat_dashes'] = [4, 2]
use_gridlines_info['grid_lon_dashes'] = [4, 2]
if gridlines_info is not None:
for gkey in gridlines_info.keys():
if gridlines_info[gkey] is not None:
use_gridlines_info[gkey] = gridlines_info[gkey]
return use_gridlines_info
def sector_xarray_spatial(full_xarray,
extent_lonlat,
varnames,
lon_pad=3,
lat_pad=0,
verbose=False):
''' Sector an xarray object spatially. If full_xarray is None, return None.
Parameters:
full_xarray (xarray.Dataset): xarray object to sector spatially
extent_lonlat (list of float): Area to sector: [MINLON, MINLAT, MAXLON, MAXLAT]
varnames (list of str): list of variable names that should be sectored based on 'timestamp'
Returns:
None: if full_xarray is None, return None
time_xarray: return '''
if full_xarray is None:
if verbose:
LOG.info(
'full_xarray is None - not attempting to sector spatially, returning None'
)
return None
sector_xarray = full_xarray.copy()
import numpy
from pyresample import utils
if verbose:
LOG.info('Padding longitudes')
# convert extent longitude to be within 0-360 range
extent_lonlat[0] = utils.wrap_longitudes(extent_lonlat[0]) - lon_pad
extent_lonlat[2] = utils.wrap_longitudes(extent_lonlat[2]) + lon_pad
if extent_lonlat[0] > extent_lonlat[2] and extent_lonlat[2] < 0:
extent_lonlat[2] = extent_lonlat[2] + 360
extent_lonlat[1] = extent_lonlat[1] - lat_pad
extent_lonlat[3] = extent_lonlat[3] + lat_pad
if verbose:
LOG.info('Padding latitudes')
# Make sure we don't extend latitudes beyond -90 / +90
if extent_lonlat[1] < -90.0:
extent_lonlat[1] = -90.0
if extent_lonlat[3] > 90.0:
extent_lonlat[3] = 90.0
if verbose:
LOG.info('Wrapping longitudes')
import xarray
lons = utils.wrap_longitudes(full_xarray['longitude'])
if verbose:
LOG.info('Handling dateline')
if lons.max() > 179.5 and lons.min(
) < -179.5 and extent_lonlat[2] > 0 and extent_lonlat[0] > 0:
lons = xarray.where(full_xarray['longitude'] < 0,
full_xarray['longitude'] + 360,
full_xarray['longitude'])
lats = full_xarray['latitude']
for varname in varnames:
good_speeds = numpy.ma.count(sector_xarray[varname].to_masked_array())
if verbose:
LOG.info('STARTED SPATIAL WITH %s points for %s', good_speeds,
varname)
if verbose:
LOG.info(
'Getting appropriate sector area lon %s to %s lat %s to %s, minlon %s, maxlon %s, minlat %s, maxlat %s, %s points',
extent_lonlat[0], extent_lonlat[2], extent_lonlat[1],
extent_lonlat[3],
lons.min().data,
lons.max().data,
lats.min().data,
lats.max().data, good_speeds)
sector_inds = (lons > extent_lonlat[0])\
& (lons < extent_lonlat[2])\
& (full_xarray['latitude'] > extent_lonlat[1])\
& (full_xarray['latitude'] < extent_lonlat[3])
# from IPython import embed as shell; shell()
covg = False
final_good_points = 0
for varname in varnames:
sector_xarray[varname] = full_xarray[varname].where(sector_inds)
good_speeds = numpy.ma.count(sector_xarray[varname].to_masked_array())
if good_speeds > final_good_points:
final_good_points = good_speeds
if sector_xarray[
'latitude'].size < MIN_POINTS or good_speeds < MIN_POINTS:
if verbose:
LOG.warning(
'INSUFFICIENT SPATIAL DATA between %0.2f and %0.2f lon and %0.2f and %0.2f lat, varname %s, %s points',
extent_lonlat[0], extent_lonlat[2], extent_lonlat[1],
extent_lonlat[3], varname, good_speeds)
else:
if verbose:
LOG.info(
'MATCHED SPATIAL %s points for var %s after location sectoring',
good_speeds, varname)
covg = True
if not covg:
LOG.warning(
'OVERALL INSUFFICIENT SPATIAL DATA between %0.2f and %0.2f lon and %0.2f and %0.2f lat',
extent_lonlat[0], extent_lonlat[2], extent_lonlat[1],
extent_lonlat[3])
return None
LOG.warning(
'OVERALL SUFFICIENT SPATIAL DATA between %0.2f and %0.2f lon and %0.2f and %0.2f lat %s points',
extent_lonlat[0], extent_lonlat[2], extent_lonlat[1], extent_lonlat[3],
final_good_points)
return sector_xarray
def window(self, lat_min, lon_min, lat_max, lon_max):
"""Function to window, ie select a specific region, given the lower left
latitude and longitude and the upper right latitude and longitude
Parameters
----------
lat_min : float
lower left latitude .
lon_min : float
lower left longitude.
lat_max : float
upper right latitude.
lon_max : float
upper right longitude.
Returns
-------
xr.DataSet
returns the windowed object.
"""
try:
from pyresample import utils
from .util.interp_util import nearest_point_swathdefinition as npsd
from .util.interp_util import lonlat_to_swathdefinition as llsd
from numpy import concatenate
has_pyresample = True
except ImportError:
has_pyresample = False
try:
if has_pyresample:
lons, lats = utils.check_and_wrap(self.obj.longitude.values,
self.obj.latitude.values)
swath = llsd(longitude=lons, latitude=lats)
pswath_ll = npsd(longitude=float(lon_min),
latitude=float(lat_min))
pswath_ur = npsd(longitude=float(lon_max),
latitude=float(lat_max))
row, col = utils.generate_nearest_neighbour_linesample_arrays(
swath, pswath_ll, float(1e6))
y_ll, x_ll = row[0][0], col[0][0]
row, col = utils.generate_nearest_neighbour_linesample_arrays(
swath, pswath_ur, float(1e6))
y_ur, x_ur = row[0][0], col[0][0]
if x_ur < x_ll:
x1 = self.obj.x.where(self.obj.x >= x_ll, drop=True).values
x2 = self.obj.x.where(self.obj.x <= x_ur, drop=True).values
xrange = concatenate([x1, x2]).astype(int)
self.obj['longitude'][:] = utils.wrap_longitudes(
self.obj.longitude.values)
# xrange = arange(float(x_ur), float(x_ll), dtype=int)
else:
xrange = slice(x_ll, x_ur)
if y_ur < y_ll:
y1 = self.obj.y.where(self.obj.y >= y_ll, drop=True).values
y2 = self.obj.y.where(self.obj.y <= y_ur, drop=True).values
yrange = concatenate([y1, y2]).astype(int)
else:
yrange = slice(y_ll, y_ur)
return self.obj.sel(x=xrange, y=yrange)
else:
raise ImportError
except ImportError:
print('Window functionality is unavailable without pyresample')