def main(start_date, end_date, gridding_method, grid_name, data_path):
# 1. Define a grid
# (a) by giving lower-left and upper-right corner
grid = omi.Grid(
llcrnrlat=19.6, urcrnrlat=25.6,
llcrnrlon=108.8, urcrnrlon=117.6, resolution=0.01
)
# (b) or by reading this data from a JSON file
# (the default file can be found in omi/data/gridds.json)
grid = omi.Grid.by_name(grid_name)
# 2. Define parameter for PSM
# - gamma (smoothing parameter)
# - rho_est (typical maximum value of distribution)
rho_est = 4e16
if gridding_method == 'psm':
# gamma is computed as function of pixel overlap
gamma = omi.compute_smoothing_parameter(1.0, 10.0)
# 3. Define a mapping which maps a key to the path in the
# HDF file. The function
# >>> omi.he5.create_name2dataset(path, list_of_dataset_names, dict)
# can be helpful (see above).
name2datasets = [NAME2DATASET_NO2, NAME2DATASET_PIXEL]
# 4a) data in OMI files can be read by
# >>> data = omi.he5.read_datasets(filename, name2dataset)
# 4b) or by iterating over orbits from start to end date at the following
# location:
# os.path.join(data_path, product, 'level2', year, doy, '*.he5')
#
# (see omi.he5 module for details)
products = ['OMNO2.003', 'OMPIXCOR.003']
for timestamp, orbit, data in omi.he5.iter_orbits(
start_date, end_date, products, name2datasets, data_path
):
# 5) Check for missing corner coordinates, i.e. the zoom product,
# which is currently not supported
if (data['TiledCornerLongitude'].mask.any() or
data['TiledCornerLatitude'].mask.any()
):
continue
# 6) Clip orbit to grid domain
lon = data['FoV75CornerLongitude']
lat = data['FoV75CornerLatitude']
data = omi.clip_orbit(grid, lon, lat, data, boundary=(2,2))
if data['ColumnAmountNO2Trop'].size == 0:
continue
# 7) Use a self-written function to preprocess the OMI data and
# to create the following arrays MxN:
# - measurement values
# - measurement errors (currently only CVM grids errors)
# - estimate of stddev (used in PSM)
# - weight of each measurement
# (see the function preprocessing for an example)
values, errors, stddev, weights = preprocessing(gridding_method, **data)
missing_values = values.mask.copy()
if np.all(values.mask):
continue
# 8) Grid orbit using PSM or CVM:
print 'time: %s, orbit: %d' % (timestamp, orbit)
if gridding_method == 'psm':
grid = omi.psm_grid(grid,
data['Longitude'], data['Latitude'],
data['TiledCornerLongitude'], data['TiledCornerLatitude'],
values, errors, stddev, weights, missing_values,
data['SpacecraftLongitude'], data['SpacecraftLatitude'],
data['SpacecraftAltitude'],
gamma[data['ColumnIndices']],
rho_est
)
else:
grid = omi.cvm_grid(grid, data['FoV75CornerLongitude'], data['FoV75CornerLatitude'],
values, errors, weights, missing_values)
# 9) The distribution of values and errors has to be normalised
# with the weight.
grid.norm()
# 10) The Level 3 product can be saved as HDF5 file
# or converted to an image (requires matplotlib and basemap)
grid.save_as_he5('test_%s.he5' % gridding_method)
grid.save_as_image('test_%s.png' % gridding_method, vmin=0, vmax=rho_est)
# 11) It is possible to set values, errors and weights to zero.
grid.zero()
def main(start_date, end_date, grid_name, data_path, save_path, x_convolution_number, y_convolution_number, x_convolution_mid, y_convolution_mid):
# 1. Define a grid
grid = gridname2grid(grid_name)
gridcoll = gridname2grid(grid_name)
wgrid = gridname2grid(grid_name)
grid.values = grid.values * 0.0
wgrid.values = wgrid.values * 0.0
gridcoll.values = gridcoll.values * 0.0
filename = generate_filename(save_path, start_date,grid_name)
#fname = '%s.he5' % (filename)
fname = '%s.he5' % (filename)
if os.path.isfile(fname) == True:
print 'Existing file: ', fname
else:
#print x_convolution_number, y_convolution_number, x_convolution_mid, y_convolution_mid
try:
print filename
# 2. Define parameter for PSM
# - gamma (smoothing parameter)
# - rho_est (typical maximum value of distribution)
rho_est = 4e16
# gamma is computed as function of pixel overlap
gamma = omi.compute_smoothing_parameter(1.0, 10.0)
# 3. Define a mapping which maps a key to the path in the
# HDF file. The function
# >>> omi.he5.create_name2dataset(path, list_of_dataset_names, dict)
# can be helpful (see above).
name2datasets = [NAME2DATASET_NO2, NAME2DATASET_PIXEL]
# 4a) data in OMI files can be read by
# >>> data = omi.he5.read_datasets(filename, name2dataset)
# 4b) or by iterating over orbits from start to end date at the following
# location:
# os.path.join(data_path, product, 'level2', year, doy, '*.he5')
#
# (see omi.he5 module for details)
products = ['OMNO2.003', 'OMPIXCOR.003']
for timestamp, orbit, data in omi.he5.iter_orbits(
start_date, end_date, products, name2datasets, data_path
):
print 'time: %s, orbit: %d' % (timestamp, orbit)
grid = gridname2grid(grid_name)
wgrid = gridname2grid(grid_name)
# 5) Check for missing corner coordinates, i.e. the zoom product,
# which is currently not supported
if (data['TiledCornerLongitude'].mask.any() or
data['TiledCornerLatitude'].mask.any()
):
continue
# 6) Clip orbit to grid domain
lon = data['FoV75CornerLongitude']
lat = data['FoV75CornerLatitude']
data = omi.clip_orbit(grid, lon, lat, data, boundary=(2,2))
if data['ColumnAmountNO2Trop'].size == 0:
continue
values, errors, stddev, weights, cf = preprocessing(**data)
missing_values = values.mask.copy()
#np.save('values.npy',values)
values.dump('values.npy')
weights.dump('weights.npy')
# JLL 9 Aug 2017: for the Gaussian smoothing, it's necessary that all values being smoothed are valid.
# From Annette: use a value that is typical of the area in question. So I will probably look at the
# background average for the US to find an appropriate value here.
#
# This matters because the unmasked data is passed to the convolution, redo_Convolution(A.data, ...)
# Afterwards the previously masked values are remasked, so these values should only be used in the
# Gaussian smoothing.
values.data[values.data<-1e29]=1e15
values.data[values.data==np.nan] = 1e15
valuesmask = values.mask
meanvalue = np.nanmean(values)
#print 'mask', np.shape(valuesmask)
if np.all(values.mask):
continue
b = np.where(values >4*np.std(values))
try:
# JLL 08 Aug 2017: For each value that is above the threshold, find the 3x3 grid of values around it
#
for i in range(len(b[0])):
#print i
#print b[0][i], b[1][i], values[b[0][i]][b[1][i]]
m = b[0][i]
n = b[1][i]
B = values[m-1:m+2,n-1:n+2]
#print B ,m,n
B0 = B*1.0 # JLL 08 Aug 2017: I'm assuming this is to make B0 independent of B?
B0[1][1] = np.nan
#print B[1][1]/np.nanmean(B[0])
if B[1][1]/np.nanmean(B[0])>= 30:
pdb.set_trace()
A = values[m-8:m+9,n-1:n+2]
amean = np.nanmean(A)
replace = redo_Convolution(A.data, x_convolution_number, y_convolution_number, x_convolution_mid, y_convolution_mid)
replace2 = replace*(amean/np.nanmean(replace))
values[m-8:m+9,n-1:n+2] = replace2
except:
print 'no std to high'
values = ma.array(values, mask = valuesmask)
new_weight = weights
#mask0 = valuesmask
#mask0 |= values.data ==np.nan
#values = ma.array(values, mask = valuesmask*mask0)
#meanconvalues = np.nanmean(values)
#values = ma.array(values, mask = valuesmask*mask0)*(meanvalue/meanconvalues)
#print np.shape(values), type(values)
#values.dump('values3.npy')
#print 'mean', np.nanmean(values), meanvalue
#print np.max(weights), np.min(weights), np.max(values), np.min(values)
print 'time: %s, orbit: %d' % (timestamp, orbit)
gamma = omi.compute_smoothing_parameter(40.0, 40.0)
#rho_est = 0.01
rho_est = np.max(new_weight)*1.2
wgrid = omi.psm_grid(wgrid,
data['Longitude'], data['Latitude'],
data['TiledCornerLongitude'], data['TiledCornerLatitude'],
new_weight, errors,new_weight*0.9, weights, missing_values,
data['SpacecraftLongitude'], data['SpacecraftLatitude'],
data['SpacecraftAltitude'],
gamma[data['ColumnIndices']],
rho_est
)
print 'wgrid vorbei'
#rho_est = 4e16
rho_est = np.max(values)*1.2
gamma = omi.compute_smoothing_parameter(1.0, 10.0)
grid = omi.psm_grid(grid,
data['Longitude'], data['Latitude'],
data['TiledCornerLongitude'], data['TiledCornerLatitude'],
values, errors, stddev, weights, missing_values,
data['SpacecraftLongitude'], data['SpacecraftLatitude'],
data['SpacecraftAltitude'],
gamma[data['ColumnIndices']],
rho_est
)
print 'grid vorbei'
grid.norm()
wgrid.norm()
wgrid.values = np.clip(np.array(wgrid.values), 0.01, np.max(np.array(wgrid.values)))
pdb.set_trace()
mask = ~np.ma.masked_invalid(grid.values).mask
gridcoll.values += np.nan_to_num(grid.values)*np.nan_to_num(wgrid.values)*mask
gridcoll.weights += np.nan_to_num(wgrid.values)*mask
grid.zero()
wgrid.zero()
print 'gridzero, wgridzero'
# 9) The distribution of values and errors has to be normalised
# with the weight.
print 'doof'
gridcoll.norm()
print 'doof2'
print filename
gridcoll.save_as_he5('%s_clip_end.he5' % (filename))
#gridcoll.save_as_image('%s.png' % (filename), vmin=0, vmax=rho_est)
print 'geschafft'
gridcoll.zero()
except:
print 'No datas available at following day:', start_date
def main(start_date, end_date, grid_name, data_path, save_path):
# 1. Define a grid
#grid = omi.Grid(llcrnrlat=40.0, urcrnrlat=55.0,llcrnrlon=-5.0, urcrnrlon=20.0, resolution=0.002); grid_name = 'Germany'#7500*12500
#grid = omi.Grid.by_name(grid_name)
#grid = gridname2grid(grid_name)
grid = gridname2grid(grid_name)
gridcoll = gridname2grid(grid_name)
wgrid = gridname2grid(grid_name)
grid.values = grid.values * 0.0
wgrid.values = wgrid.values * 0.0
gridcoll.values = gridcoll.values * 0.0
gridcoll.weights = gridcoll.weights * 0.0
filename = generate_filename(save_path, start_date, grid_name)
fname = '%s.he5' % (filename)
if os.path.isfile(fname) == True:
print('Existing file: ', fname)
else:
try:
# 2. Define parameter for PSM
# - gamma (smoothing parameter)
# - rho_est (typical maximum value of distribution)
rho_est = 4e16
# gamma is computed as function of pixel overlap
gamma = omi.compute_smoothing_parameter(1.0, 10.0)
# 3. Define a mapping which maps a key to the path in the
# HDF file. The function
# >>> omi.he5.create_name2dataset(path, list_of_dataset_names, dict)
# can be helpful (see above).
name2datasets = [NAME2DATASET_NO2, NAME2DATASET_PIXEL]
# 4a) data in OMI files can be read by
# >>> data = omi.he5.read_datasets(filename, name2dataset)
# 4b) or by iterating over orbits from start to end date at the following
# location:
# os.path.join(data_path, product, 'level2', year, doy, '*.he5')
#
# (see omi.he5 module for details)
#products = ['OMNO2.003', 'OMPIXCOR.003']
products = ['BEHR-PSM', 'BEHR-PSM']
pdb.set_trace()
for timestamp, orbit, data in omi.he5.iter_orbits(
start_date, end_date, products, name2datasets, data_path):
print('time: %s, orbit: %d' % (timestamp, orbit))
grid = gridname2grid(grid_name)
wgrid = gridname2grid(grid_name)
#print '1'
# 5) Check for missing corner coordinates, i.e. the zoom product,
# which is currently not supported
if (data['TiledCornerLongitude'].mask.any()
or data['TiledCornerLatitude'].mask.any()):
continue
# 6) Clip orbit to grid domain
lon = data['FoV75CornerLongitude']
lat = data['FoV75CornerLatitude']
data = omi.clip_orbit(grid, lon, lat, data, boundary=(2, 2))
if data['ColumnAmountNO2Trop'].size == 0:
continue
#print '2'
# 7) Use a self-written function to preprocess the OMI data and
# to create the following arrays MxN:
# - measurement values
# - measurement errors (currently only CVM grids errors)
# - estimate of stddev (used in PSM)
# - weight of each measurement
# (see the function preprocessing for an example)
values, errors, stddev, weights = preprocessing(**data)
missing_values = values.mask.copy()
if np.all(values.mask):
continue
#new_weight = 1/np.sqrt(np.abs((errors/1e15) * (1+2*data['CloudRadianceFraction']**2)))#**(1.0/2.0)
new_weight = weights / np.sqrt((np.abs(
(errors / 1e15) *
(1 + 2 * data['CloudRadianceFraction']**2)))) #**(1.0/2.0)
#print 'time: %s, orbit: %d' % (timestamp, orbit)
#print '-----------------------------'
rho_est = 4e16
gamma = omi.compute_smoothing_parameter(1.0, 10.0)
grid = omi.psm_grid(grid, data['Longitude'], data['Latitude'],
data['TiledCornerLongitude'],
data['TiledCornerLatitude'], values,
errors, stddev, weights, missing_values,
data['SpacecraftLongitude'],
data['SpacecraftLatitude'],
data['SpacecraftAltitude'],
gamma[data['ColumnIndices']], rho_est)
#print '3'
gamma = omi.compute_smoothing_parameter(40.0, 40.0)
rho_est = 4
wgrid = omi.psm_grid(
wgrid, data['Longitude'], data['Latitude'],
data['TiledCornerLongitude'], data['TiledCornerLatitude'],
new_weight, errors, new_weight * 0.9, weights,
missing_values, data['SpacecraftLongitude'],
data['SpacecraftLatitude'], data['SpacecraftAltitude'],
gamma[data['ColumnIndices']], rho_est)
# The 90% of new_weight = std. dev. is a best guess comparing uncertainty
# over land and sea
#print '4'
grid.norm(
) # divides by the weights (at this point, the values in the grid are multiplied by the weights)
# Replace by the new weights later
#wgrid.norm() # if you normalize wgrid the data is not as smooth as it could be
wgrid.values = np.nan_to_num(np.array(wgrid.values))
grid.values = np.nan_to_num(np.array(grid.values))
#grid.values = grid.values/grid.weights
#wgrid.values = wgrid.values/wgrid.weights
#print 'counter = ', counter, ':', np.max(gridcoll.values), np.max(grid.values), np.max(wgrid.values)
gridcoll.values += np.nan_to_num(grid.values) * np.nan_to_num(
wgrid.values)
gridcoll.weights += wgrid.values
grid.zero()
wgrid.zero()
# 9) The distribution of values and errors has to be normalised
# with the weight.
gridcoll.norm()
#grid.norm()
# 10) The Level 3 product can be saved as HDF5 file
# or converted to an image (requires matplotlib and basemap
rho_est = 4e16
gridcoll.save_as_he5('%s.he5' % (filename))
#gridcoll.save_as_image('%s.png' % (filename), vmin=0, vmax=rho_est)
except:
print('No datas available at following day:', start_date)
#grid.save_as_he5('%s_%s_%s_%s_%s.he5' % (grid_name, str(start_date)[8:10], str(start_date)[5:7], str(start_date)[0:4]))
#grid.save_as_image('%s_%s_%s_%s_%s.png' % (grid_name, str(start_date)[8:10],str(start_date)[5:7], str(start_date)[0:4]), vmin=0, vmax=rho_est)
#grid.save_as_image('%s_%s_%s_%s_%s.he5' % ( str(start_date)[8:10], str(start_date)[5:7], str(start_date)[0:4], grid_name, gridding_method), vmin=0, vmax=rho_est)
# 11) It is possible to set values, errors and weights to zero.
grid.zero()
def main(start_date, end_date, gridding_method, grid_name, data_path):
# 1. Define a grid
# (a) by giving lower-left and upper-right corner
#grid_name = "NewAsia"
#grid = omi.Grid(llcrnrlat=40.0, urcrnrlat=55.0,llcrnrlon=-5.0, urcrnrlon=20.0, resolution=0.002); grid_name = 'Germany'#7500*12500
#grid = omi.Grid(llcrnrlat= 17.8 , urcrnrlat=53.6 ,llcrnrlon=96.9 , urcrnrlon= 106.8, resolution=0.01); #grid_name = 'Northamerica'#6000*4000
grid = omi.Grid(llcrnrlat=25,
urcrnrlat=50.05,
llcrnrlon=-125,
urcrnrlon=-64.95,
resolution=0.05)
#grid_name = 'Northamerica'#6000*4000
# (b) or by reading this data from a JSON file
# (the default file can be found in omi/data/gridds.json)
#grid = omi.Grid.by_name(grid_name)
# 2. Define parameter for PSM
# - gamma (smoothing parameter)
# - rho_est (typical maximum value of distribution)
rho_est = 4e16
if gridding_method == 'psm':
# gamma is computed as function of pixel overlap
gamma = omi.compute_smoothing_parameter(1.0, 10.0)
# 3. Define a mapping which maps a key to the path in the
# HDF file. The function
# >>> omi.he5.create_name2dataset(path, list_of_dataset_names, dict)
# can be helpful (see above).
name2datasets = [NAME2DATASET_NO2, NAME2DATASET_PIXEL]
# 4a) data in OMI files can be read by
# >>> data = omi.he5.read_datasets(filename, name2dataset)
# 4b) or by iterating over orbits from start to end date at the following
# location:
# os.path.join(data_path, product, 'level2', year, doy, '*.he5')
#
# (see omi.he5 module for details)
products = ['OMNO2.003', 'OMPIXCOR.003'] #part of the path
for timestamp, orbit, data in omi.he5.iter_orbits(start_date, end_date,
products, name2datasets,
data_path):
# debugging check: this loop is occasionally getting what I consider night time swaths that just barely cross
# the top of the domain. I deliberately remove those, even though they may be illuminated
print 'timestamp =', timestamp
if timestamp.hour < 15:
continue
# 5) Check for missing corner coordinates, i.e. the zoom product,
# which is currently not supported
if (data['TiledCornerLongitude'].mask.any()
or data['TiledCornerLatitude'].mask.any()):
continue
# 6) Clip orbit to grid domain
lon = data['FoV75CornerLongitude']
lat = data['FoV75CornerLatitude']
data = omi.clip_orbit(grid, lon, lat, data, boundary=(2, 2))
if data['ColumnAmountNO2Trop'].size == 0:
continue
# 7) Use a self-written function to preprocess the OMI data and
# to create the following arrays MxN:
# - measurement values
# - measurement errors (currently only CVM grids errors)
# - estimate of stddev (used in PSM)
# - weight of each measurement
# (see the function preprocessing for an example)
values, errors, stddev, weights = preprocessing(
gridding_method, **data)
missing_values = values.mask.copy()
if np.all(values.mask):
continue
# 8) Grid orbit using PSM or CVM:
print 'time: %s, orbit: %d' % (timestamp, orbit)
if gridding_method == 'psm':
grid = omi.psm_grid(grid, data['Longitude'], data['Latitude'],
data['TiledCornerLongitude'],
data['TiledCornerLatitude'], values, errors,
stddev, weights, missing_values,
data['SpacecraftLongitude'],
data['SpacecraftLatitude'],
data['SpacecraftAltitude'],
gamma[data['ColumnIndices']], rho_est)
else:
grid = omi.cvm_grid(grid, data['FoV75CornerLongitude'],
data['FoV75CornerLatitude'], values, errors,
weights, missing_values)
# 9) The distribution of values and errors has to be normalised
# with the weight.
grid.norm()
# 10) The Level 3 product can be saved as HDF5 file
# or converted to an image (requires matplotlib and basemap
grid.save_as_he5('%s_%s_%s_%s_%s.he5' %
(grid_name, str(start_date)[8:10], str(start_date)[5:7],
str(start_date)[0:4], gridding_method))
grid.save_as_image('%s_%s_%s_%s_%s.png' %
(grid_name, str(start_date)[8:10], str(start_date)[5:7],
str(start_date)[0:4], gridding_method),
vmin=0,
vmax=1e16)
#grid.save_as_image('%s_%s_%s_%s_%s.he5' % ( str(start_date)[8:10], str(start_date)[5:7], str(start_date)[0:4], grid_name, gridding_method), vmin=0, vmax=rho_est)
# 11) It is possible to set values, errors and weights to zero.
grid.zero()