def igridding_simple(grid_info, loncorn, latcorn, vals, errors=None, weights=None, is_flag=False):
"""
Interface method that applies CVM gridding to a simple set of data defined in 2D numpy arrays
:param grid_info: a dictionary that can be given to omi.Grid as omi.Grid(**grid_info)
:param loncorn: the longitude corner coordinates of the data to grid as a 3D numpy array; assumes that the corners
are given along the first dimension, i.e. loncorn[:,i,j] is the slice with the corner coordinates for pixel i, j
:param latcorn: the latitude corner coordinate of the data to grid as a 3D numpy array; same format as loncorn.
:param vals: the values to grid as a 2D numpy array
:return: a 2D numpy array with the values gridded to the lon/lat coordinates defined in grid and a 2D numpy array
with the weighting values
"""
if errors is None:
errors = np.zeros_like(vals)
if weights is None:
weights = np.ones_like(vals)
vals, errors, weights, mask = simple_preprocessing(vals, errors, weights)
grid_in = omi.Grid(**grid_info)
grid = omi.cvm_grid(grid_in, loncorn, latcorn, vals, errors, weights, mask, is_flag=is_flag)
# For now, unlike the normal gridding method, we will retain NaNs in the gridded values and weights
grid_weights = grid.weights
grid.norm()
grid_values = grid.values
return grid_values, grid_weights
def main(start_date, end_date, gridding_method, grid_name, data_path):
# 1. Define a grid
# (a) by giving lower-left and upper-right corner
grid = omi.Grid(
llcrnrlat=19.6, urcrnrlat=25.6,
llcrnrlon=108.8, urcrnrlon=117.6, resolution=0.01
)
# (b) or by reading this data from a JSON file
# (the default file can be found in omi/data/gridds.json)
grid = omi.Grid.by_name(grid_name)
# 2. Define parameter for PSM
# - gamma (smoothing parameter)
# - rho_est (typical maximum value of distribution)
rho_est = 4e16
if gridding_method == 'psm':
# gamma is computed as function of pixel overlap
gamma = omi.compute_smoothing_parameter(1.0, 10.0)
# 3. Define a mapping which maps a key to the path in the
# HDF file. The function
# >>> omi.he5.create_name2dataset(path, list_of_dataset_names, dict)
# can be helpful (see above).
name2datasets = [NAME2DATASET_NO2, NAME2DATASET_PIXEL]
# 4a) data in OMI files can be read by
# >>> data = omi.he5.read_datasets(filename, name2dataset)
# 4b) or by iterating over orbits from start to end date at the following
# location:
# os.path.join(data_path, product, 'level2', year, doy, '*.he5')
#
# (see omi.he5 module for details)
products = ['OMNO2.003', 'OMPIXCOR.003']
for timestamp, orbit, data in omi.he5.iter_orbits(
start_date, end_date, products, name2datasets, data_path
):
# 5) Check for missing corner coordinates, i.e. the zoom product,
# which is currently not supported
if (data['TiledCornerLongitude'].mask.any() or
data['TiledCornerLatitude'].mask.any()
):
continue
# 6) Clip orbit to grid domain
lon = data['FoV75CornerLongitude']
lat = data['FoV75CornerLatitude']
data = omi.clip_orbit(grid, lon, lat, data, boundary=(2,2))
if data['ColumnAmountNO2Trop'].size == 0:
continue
# 7) Use a self-written function to preprocess the OMI data and
# to create the following arrays MxN:
# - measurement values
# - measurement errors (currently only CVM grids errors)
# - estimate of stddev (used in PSM)
# - weight of each measurement
# (see the function preprocessing for an example)
values, errors, stddev, weights = preprocessing(gridding_method, **data)
missing_values = values.mask.copy()
if np.all(values.mask):
continue
# 8) Grid orbit using PSM or CVM:
print 'time: %s, orbit: %d' % (timestamp, orbit)
if gridding_method == 'psm':
grid = omi.psm_grid(grid,
data['Longitude'], data['Latitude'],
data['TiledCornerLongitude'], data['TiledCornerLatitude'],
values, errors, stddev, weights, missing_values,
data['SpacecraftLongitude'], data['SpacecraftLatitude'],
data['SpacecraftAltitude'],
gamma[data['ColumnIndices']],
rho_est
)
else:
grid = omi.cvm_grid(grid, data['FoV75CornerLongitude'], data['FoV75CornerLatitude'],
values, errors, weights, missing_values)
# 9) The distribution of values and errors has to be normalised
# with the weight.
grid.norm()
# 10) The Level 3 product can be saved as HDF5 file
# or converted to an image (requires matplotlib and basemap)
grid.save_as_he5('test_%s.he5' % gridding_method)
grid.save_as_image('test_%s.png' % gridding_method, vmin=0, vmax=rho_est)
# 11) It is possible to set values, errors and weights to zero.
grid.zero()
def igridding_simple(grid_info,
loncorn,
latcorn,
vals,
errors=None,
weights=None,
is_flag=False):
"""
Interface method that applies CVM gridding to a simple set of data defined in 2D numpy arrays
:param grid_info: a dictionary that can be given to omi.Grid as omi.Grid(**grid_info)
:param loncorn: the longitude corner coordinates of the data to grid as a 3D numpy array; assumes that the corners
are given along the first dimension, i.e. loncorn[:,i,j] is the slice with the corner coordinates for pixel i, j
:param latcorn: the latitude corner coordinate of the data to grid as a 3D numpy array; same format as loncorn.
:param vals: the values to grid as a 2D numpy array
:return: a 2D numpy array with the values gridded to the lon/lat coordinates defined in grid and a 2D numpy array
with the weighting values
"""
if errors is None:
errors = np.zeros_like(vals)
if weights is None:
weights = np.ones_like(vals)
if vals.ndim == 3:
print('Doing 3D gridding')
n_levels = vals.shape[2]
for i_lev in range(n_levels):
grid_val_i, grid_wt_i = igridding_simple(grid_info,
loncorn,
latcorn,
vals[:, :, i_lev],
errors=errors[:, :,
i_lev],
weights=weights[:, :,
i_lev],
is_flag=is_flag)
if i_lev == 0:
grid_values = np.full(grid_val_i.shape + (n_levels, ),
np.nan,
dtype=grid_val_i.dtype)
grid_weights = np.full(grid_wt_i.shape + (n_levels, ),
np.nan,
dtype=grid_val_i.dtype)
grid_values[:, :, i_lev] = grid_val_i
grid_weights[:, :, i_lev] = grid_wt_i
else:
vals, errors, weights, mask = simple_preprocessing(
vals, errors, weights)
grid_in = omi.Grid(**grid_info)
grid = omi.cvm_grid(grid_in,
loncorn,
latcorn,
vals,
errors,
weights,
mask,
is_flag=is_flag)
# For now, unlike the normal gridding method, we will retain NaNs in the gridded values and weights
grid_weights = grid.weights
grid.norm()
grid_values = grid.values
return grid_values, grid_weights
def grid_orbit(data,
grid_info,
gridded_quantity,
gridding_method='psm',
preproc_method='generic',
verbosity=0):
# Input checking
if not isinstance(data, dict):
raise TypeError('data must be a dict')
elif gridded_quantity not in data.keys():
raise KeyError(
'data does not have a key matching the gridded_quantity "{}"'.
format(gridded_quantity))
else:
missing_keys = [
k for k in behr_datasets(gridding_method) if k not in data.keys()
]
if len(missing_keys) > 0:
raise KeyError(
'data is missing the following expected keys: {0}'.format(
', '.join(missing_keys)))
grid = make_grid(grid_info)
wgrid = make_grid(grid_info)
# 5) Check for missing corner coordinates, i.e. the zoom product,
# which is currently not supported
if data['TiledCornerLongitude'].mask.any(
) or data['TiledCornerLatitude'].mask.any():
return None, None
# 6) Clip orbit to grid domain
lon = data['FoV75CornerLongitude']
lat = data['FoV75CornerLatitude']
data = omi.clip_orbit(grid, lon, lat, data, boundary=(2, 2))
if data['BEHRColumnAmountNO2Trop'].size == 0:
return None, None
# Preprocess the BEHR data to create the following arrays MxN:
# - measurement values
# - measurement errors (used in CVM, not PSM)
# - estimate of stddev (used in PSM)
# - weight of each measurement
if verbosity > 1:
print(' Doing {} preprocessing for {}'.format(
preproc_method, gridded_quantity))
if preproc_method.lower() == 'behr':
values, errors, stddev, weights = behr_preprocessing(
gridding_method, **data)
elif preproc_method.lower() == 'sp':
values, errors, stddev, weights = sp_preprocessing(
gridding_method, **data)
elif preproc_method.lower() == 'generic':
# I'm copying the values before passing them because I intend this to be able to called multiple times to grid
# different fields, and I don't want to risk values in data being altered.
values, errors, stddev, weights = generic_preprocessing(
gridding_method, gridded_quantity, data[gridded_quantity].copy(),
**data)
else:
raise NotImplementedError(
'No preprocessing option for column_product={}'.format(
gridded_quantity))
missing_values = values.mask.copy()
if np.all(values.mask):
return None, None # two outputs expected, causes a "None object not iterable" error if only one given
new_weight = weights # JLL 9 Aug 2017 - Seems unnecessary now
if verbosity > 1:
print(' Gridding {}'.format(gridded_quantity))
is_flag_field = gridded_quantity in flag_fields
if gridding_method == 'psm':
gamma = omi.compute_smoothing_parameter(40.0, 40.0)
rho_est = np.max(new_weight) * 1.2
if verbosity > 1:
print(' Gridding weights')
try:
wgrid = omi.psm_grid(wgrid, data['Longitude'], data['Latitude'],
data['TiledCornerLongitude'],
data['TiledCornerLatitude'], new_weight,
errors, new_weight * 0.9, weights,
missing_values, data['SpacecraftLongitude'],
data['SpacecraftLatitude'],
data['SpacecraftAltitude'],
gamma[data['ColumnIndices']], rho_est)
# The 90% of new_weight = std. dev. is a best guess comparing uncertainty
# over land and sea
except QhullError as err:
print("Cannot interpolate, QhullError: {0}".format(err.args[0]))
return None, None
rho_est = np.max(values) * 1.2
gamma = omi.compute_smoothing_parameter(1.0, 10.0)
try:
grid = omi.psm_grid(grid, data['Longitude'], data['Latitude'],
data['TiledCornerLongitude'],
data['TiledCornerLatitude'], values, errors,
stddev, weights, missing_values,
data['SpacecraftLongitude'],
data['SpacecraftLatitude'],
data['SpacecraftAltitude'],
gamma[data['ColumnIndices']], rho_est)
except QhullError as err:
print("Cannot interpolate, QhullError: {0}".format(err.args[0]))
return None, None
grid.norm(
) # divides by the weights (at this point, the values in the grid are multiplied by the weights)
# Replace by new weights in a bit
wgrid.norm() # in the new version, wgrid is also normalized.
# Clip the weights so that only positive weights are allowed. Converting things back to np.array removes any
# masking
wgrid.values = np.clip(np.array(wgrid.values), 0.01,
np.max(np.array(wgrid.values)))
wgrid_values = np.array(wgrid.values)
grid_values = np.array(grid.values)
elif gridding_method == 'cvm':
try:
grid = omi.cvm_grid(grid,
data['FoV75CornerLongitude'],
data['FoV75CornerLatitude'],
values,
errors,
weights,
missing_values,
is_flag=is_flag_field)
except QhullError as err:
print("Cannot interpolate, QhullError: {0}".format(err.args[0]))
return None, None
if not is_flag_field:
wgrid_values = grid.weights
grid.norm()
else:
wgrid_values = np.ones_like(grid.values)
grid_values = grid.values
else:
raise NotImplementedError(
'gridding method {0} not understood'.format(gridding_method))
# At this point, grid.values is a numpy array, not a masked array. Using nan_to_num should also automatically set
# the value to 0, but that doesn't guarantee that the same values in weights will be made 0. So we manually multiply
# the weights by 0 for any invalid values in the NO2 grid. This prevents reducing the final column when we divide by
# the total weights outside this function.
good_values = ~np.ma.masked_invalid(grid_values).mask
grid_values = np.nan_to_num(grid_values)
wgrid_values = np.nan_to_num(wgrid_values) * good_values
return grid_values, wgrid_values
def main(start_date, end_date, gridding_method, grid_name, data_path):
# 1. Define a grid
# (a) by giving lower-left and upper-right corner
#grid_name = "NewAsia"
#grid = omi.Grid(llcrnrlat=40.0, urcrnrlat=55.0,llcrnrlon=-5.0, urcrnrlon=20.0, resolution=0.002); grid_name = 'Germany'#7500*12500
#grid = omi.Grid(llcrnrlat= 17.8 , urcrnrlat=53.6 ,llcrnrlon=96.9 , urcrnrlon= 106.8, resolution=0.01); #grid_name = 'Northamerica'#6000*4000
grid = omi.Grid(llcrnrlat=25,
urcrnrlat=50.05,
llcrnrlon=-125,
urcrnrlon=-64.95,
resolution=0.05)
#grid_name = 'Northamerica'#6000*4000
# (b) or by reading this data from a JSON file
# (the default file can be found in omi/data/gridds.json)
#grid = omi.Grid.by_name(grid_name)
# 2. Define parameter for PSM
# - gamma (smoothing parameter)
# - rho_est (typical maximum value of distribution)
rho_est = 4e16
if gridding_method == 'psm':
# gamma is computed as function of pixel overlap
gamma = omi.compute_smoothing_parameter(1.0, 10.0)
# 3. Define a mapping which maps a key to the path in the
# HDF file. The function
# >>> omi.he5.create_name2dataset(path, list_of_dataset_names, dict)
# can be helpful (see above).
name2datasets = [NAME2DATASET_NO2, NAME2DATASET_PIXEL]
# 4a) data in OMI files can be read by
# >>> data = omi.he5.read_datasets(filename, name2dataset)
# 4b) or by iterating over orbits from start to end date at the following
# location:
# os.path.join(data_path, product, 'level2', year, doy, '*.he5')
#
# (see omi.he5 module for details)
products = ['OMNO2.003', 'OMPIXCOR.003'] #part of the path
for timestamp, orbit, data in omi.he5.iter_orbits(start_date, end_date,
products, name2datasets,
data_path):
# debugging check: this loop is occasionally getting what I consider night time swaths that just barely cross
# the top of the domain. I deliberately remove those, even though they may be illuminated
print 'timestamp =', timestamp
if timestamp.hour < 15:
continue
# 5) Check for missing corner coordinates, i.e. the zoom product,
# which is currently not supported
if (data['TiledCornerLongitude'].mask.any()
or data['TiledCornerLatitude'].mask.any()):
continue
# 6) Clip orbit to grid domain
lon = data['FoV75CornerLongitude']
lat = data['FoV75CornerLatitude']
data = omi.clip_orbit(grid, lon, lat, data, boundary=(2, 2))
if data['ColumnAmountNO2Trop'].size == 0:
continue
# 7) Use a self-written function to preprocess the OMI data and
# to create the following arrays MxN:
# - measurement values
# - measurement errors (currently only CVM grids errors)
# - estimate of stddev (used in PSM)
# - weight of each measurement
# (see the function preprocessing for an example)
values, errors, stddev, weights = preprocessing(
gridding_method, **data)
missing_values = values.mask.copy()
if np.all(values.mask):
continue
# 8) Grid orbit using PSM or CVM:
print 'time: %s, orbit: %d' % (timestamp, orbit)
if gridding_method == 'psm':
grid = omi.psm_grid(grid, data['Longitude'], data['Latitude'],
data['TiledCornerLongitude'],
data['TiledCornerLatitude'], values, errors,
stddev, weights, missing_values,
data['SpacecraftLongitude'],
data['SpacecraftLatitude'],
data['SpacecraftAltitude'],
gamma[data['ColumnIndices']], rho_est)
else:
grid = omi.cvm_grid(grid, data['FoV75CornerLongitude'],
data['FoV75CornerLatitude'], values, errors,
weights, missing_values)
# 9) The distribution of values and errors has to be normalised
# with the weight.
grid.norm()
# 10) The Level 3 product can be saved as HDF5 file
# or converted to an image (requires matplotlib and basemap
grid.save_as_he5('%s_%s_%s_%s_%s.he5' %
(grid_name, str(start_date)[8:10], str(start_date)[5:7],
str(start_date)[0:4], gridding_method))
grid.save_as_image('%s_%s_%s_%s_%s.png' %
(grid_name, str(start_date)[8:10], str(start_date)[5:7],
str(start_date)[0:4], gridding_method),
vmin=0,
vmax=1e16)
#grid.save_as_image('%s_%s_%s_%s_%s.he5' % ( str(start_date)[8:10], str(start_date)[5:7], str(start_date)[0:4], grid_name, gridding_method), vmin=0, vmax=rho_est)
# 11) It is possible to set values, errors and weights to zero.
grid.zero()