def add(self,key,sid,datum,resolution=None):
if resolution is not None:
sidtest = gd.spatial_clear_to_resolution(gd.spatial_coerce_resolution(sid,resolution))
elif self.resolution is not None:
sidtest = gd.spatial_clear_to_resolution(gd.spatial_coerce_resolution(sid,self.resolution))
else:
sidtest = sid
if sidtest not in self.sdict.keys():
self.sdict[sidtest] = catalog_entry(sidtest) # construct with relevant resolution
self.sdict[sidtest].add(key,data_entry(sid,datum))
return
def add_to_entry(self,key,sid,datum):
"Add data to entry if it's there."
if self.resolution is not None:
sid_test = gd.spatial_clear_to_resolution(gd.spatial_coerce_resolution(sid,self.resolution))
else:
sid_test = sid
# print('testing ',hex(sid_test), hex(sid))
entry_open = sid_test in self.sdict.keys()
if entry_open:
# print(key,' adding ',data,' to ',hex(sid))
self.add(key,sid,datum)
return entry_open
def build_coarser_level(fnames):
sares = [sare_from_fname(i) for i in fnames]
resolutions = [gd.spatial_resolution(i) for i in sares]
levels = set(resolutions)
# print('levels: ',levels)
if len(levels) != 1:
raise ValueError('Filenames indicate multiple resolutions.')
resolution = resolutions[0]
coarser = {}
for i in fnames:
si = gd.spatial_clear_to_resolution(
gd.spatial_coerce_resolution(sare_from_fname(i), resolution - 1))
if si not in coarser.keys():
coarser[si] = []
coarser[si].append(i)
return coarser
def main():
###########################################################################
# Data source
dataPath="/home/mrilee/data/"
###########################################################################
# MODIS
modis_base = "MOD05_L2."
# modis_item = "A2005349.2120.061.2017294065852"
# modis_time_start = "2005-12-15T21:20:00"
modis_item = "A2005349.2125.061.2017294065400"
modis_time_start = "2005-12-15T21:25:00"
modis_suffix = ".hdf"
modis_filename = modis_base+modis_item+modis_suffix
fmt_suffix = ".h5"
workFileName = "sketchG."+modis_base+modis_item+fmt_suffix
print('loading ',workFileName)
workFile = h5.File(workFileName,'r')
sids = workFile['/image']['stare_spatial']
lat = workFile['/image']['Latitude']
lon = workFile['/image']['Longitude']
data = workFile['/image']['Water_Vapor_Near_Infrared']
workFile.close()
modis_min = np.amin(data)
modis_max = np.amax(data)
sids = sids-1
###########################################################################
# MODIS HDF
hdf = SD(dataPath+modis_filename,SDC.READ)
# ds_wv_nir = hdf.select('Water_Vapor_Near_Infrared')
modis_hdf_resolution = 7
ntri_max = 1000
# hull = ps.to_hull_range_from_latlon(gring_lat[gring_seq],gring_lon[gring_seq],resolution,ntri_max)
modis_hdf_hull = gd.modis_cover_from_gring(hdf,modis_hdf_resolution,ntri_max)
hdf.end()
modis_hdf_lons,modis_hdf_lats,modis_hdf_intmat = ps.triangulate_indices(modis_hdf_hull)
modis_hdf_triang = tri.Triangulation(modis_hdf_lons,modis_hdf_lats,modis_hdf_intmat)
###########################################################################
# GOES
goes_file='sketch9.2005.349.213015.h5'
workFileName = goes_file
workFile = h5.File(workFileName,'r')
goes_sids = workFile['/image']['stare_spatial']
goes_data = workFile['/image']['goes_b3']
workFile.close()
print('goes mnmx: ',np.amin(goes_data),np.amax(goes_data))
goes_min = np.amin(goes_data)
goes_max = np.amax(goes_data)
goes_sids = goes_sids-1
goes_sids_005 = np.unique(gd.spatial_clear_to_resolution((goes_sids[np.where(goes_sids>0)] & ~31)+5))
goes_sids_005_geom = sid_geometry(goes_sids_005)
goes_sids_005_triang = goes_sids_005_geom.triang()
###########################################################################
# Plotting an ROI
cover_rads =[2.0,0,0, 0.125,0,0]
circle_color=[ 'Lime' ,'lightgrey' ,'White' ,'navajowhite' ,'khaki' ,'White' ]
###########################################################################
# HI 28.5N 177W
cover_resolution = 11
cover_lat = 19.5-0.375
cover_lon = -155.5+0.375
###########################################################################
# Plotting
nrows = 1
ncols = 1
# proj = ccrs.PlateCarree()
proj = ccrs.PlateCarree(central_longitude=-160.0)
# proj = ccrs.Mollweide()
# proj = ccrs.Mollweide(central_longitude=-160.0)
transf = ccrs.Geodetic()
fig,axs = plt.subplots(nrows=nrows,ncols=ncols,subplot_kw={'projection': proj})
subplot_title = ['GOES and MODIS Granules']
ax = axs
distribute_to_nodes = False
if True:
iter = 0
ax.set_xlim(-160-45,160+45)
if subplot_title[iter] is not None:
ax.set_title(subplot_title[iter])
if False:
ax.set_global()
if True:
ax.coastlines()
if distribute_to_nodes:
vmin=0; vmax=16
# colors=np.arange(goes_sids_005_triang.x.shape[0]/3,dtype=np.int64)%(vmax-vmin)
colors=np.random.uniform(vmin,vmax,size=int(goes_sids_005_triang.x.shape[0]/3)).astype(np.int64)
ax.tripcolor(goes_sids_005_triang
,facecolors=colors
,edgecolors='k',lw=0
,shading='flat'
,vmin=vmin,vmax=vmax,cmap='rainbow',alpha=0.65
,transform=transf)
ax.triplot(goes_sids_005_triang ,'r-',transform=transf,lw=1.0,markersize=3,alpha=0.5)
ax.triplot(modis_hdf_triang ,'b-',transform=transf,lw=1.0,markersize=3,alpha=0.5)
if True:
phi=np.linspace(0,2*np.pi,64)
cover_rad = cover_rads[iter]
rad=cover_rad
# rad=0.125
ax.plot(cover_lon+rad*np.cos(phi),cover_lat+rad*np.sin(phi),transform=transf,color=circle_color[iter],lw=2)
if iter == 0:
x0 = 0.05
y0 = 0.025; dy = 0.025
plt.figtext(x0,y0+0*dy
,"MODIS (blue): "+"sketchG."+modis_base+modis_item+fmt_suffix+', cover from GRING metadata, max resolution = 7'
,fontsize=10)
k=0;
while goes_sids[k]<0:
k=k+1
plt.figtext(x0,y0+1*dy
,"GOES (red): "+goes_file+', Northern Hemisphere, resolution = 5'
,fontsize=10)
plt.figtext(x0,y0+2*dy
,"Region of Interest near Hawaii, ROI (%s): radius = %0.2f degrees, center = 0x%016x"%(circle_color[iter],cover_rads[0],ps.from_latlon(npf64([cover_lat]),npf64([cover_lon]),cover_resolution)[0])
,fontsize=10)
if distribute_to_nodes:
plt.figtext(x0,y0+3*dy
,"Color tiles show distribution across 16 nodes."
,fontsize=10)
plt.show()
def join_goes_and_m2_to_h5(goes_datapath, goes_filenames, m2_datapath,
m2_file_name, workFileName):
##### HDF5 Data types for output
image_dtype = np.dtype([('stare_spatial', np.int64),
('stare_temporal', np.int64),
('goes_src_coord', np.int64),
('goes_b3', np.int64), ('goes_b4', np.int64),
('goes_b5', np.int64),
('merra2_src_coord', np.int64),
('merra2_tpw', np.int64)])
image_description_dtype = np.dtype([('nx', np.int), ('ny', np.int)])
m2_description_dtype = np.dtype([('nx', np.int), ('ny', np.int),
('tpw_offset', np.double),
('tpw_scale', np.double)])
goes_bandnames = {
"BAND_03": "goes_b3",
"BAND_04": "goes_b4",
"BAND_05": "goes_b5"
}
goes_filenames_valid = []
for i in goes_filenames:
goes_band = i.split('.')[4]
if goes_band in goes_bandnames.keys():
goes_filenames_valid.append(i)
if len(goes_filenames_valid) == 0:
print("*ERROR* Join and save -- no valid GOES filenames. Returning.")
return
# print('goes_filenames: ',goes_filenames)
# print('goes_filenames_valid: ',goes_filenames_valid)
###########################################################################
##### MERRA-2
m2_dLat = 0.5
m2_dLon = 5.0 / 8.0
m2_dLatkm = m2_dLat * gd.re_km / gd.deg_per_rad
m2_dLonkm = m2_dLon * gd.re_km / gd.deg_per_rad
m2_ds = Dataset(m2_datapath + m2_file_name)
m2_lat, m2_lon = np.meshgrid(m2_ds['lat'], m2_ds['lon'])
m2_lat = m2_lat.flatten()
m2_lon = m2_lon.flatten()
m2_idx_ij = np.arange(m2_lat.size, dtype=np.int64)
m2_resolution = int(gd.resolution(m2_dLonkm * 2))
m2_indices = ps.from_latlon(m2_lat, m2_lon, m2_resolution)
m2_term = gd.spatial_terminator(m2_indices)
m2_tid = gd.merra2_stare_time(m2_ds)
###########################################################################
##### GOES
igoes = 0
goes_band = goes_filenames_valid[igoes].split('.')[4]
goes_bandname = goes_bandnames[goes_band]
goes_ds = Dataset(goes_datapath + goes_filenames_valid[igoes])
goes_tid = gd.goes10_img_stare_time(goes_ds)
##### MERRA-2 at the GOES time
fine_match = ps.cmp_temporal(np.array(goes_tid, dtype=np.int64), m2_tid)
m2_ifm = np.nonzero(fine_match)[0]
m2_dataDayI = m2_ds['TQI'][m2_ifm, :, :]
m2_dataDayL = m2_ds['TQL'][m2_ifm, :, :]
m2_dataDayV = m2_ds['TQV'][m2_ifm, :, :]
m2_dataDay = m2_dataDayI + m2_dataDayL + m2_dataDayV
if m2_ifm.size > 1:
print('multiple hits on 2m')
m2_dataDay = np.mean(m2_dataDay, axis=1)
print('m2_dataDay shape: ', n2_dataDay.shape)
m2_data = m2_dataDay[:, :].T
m2_data_flat = m2_data.flatten()
g_lat = goes_ds['lat'][:, :].flatten()
g_lon = goes_ds['lon'][:, :].flatten()
g_idx_valid = np.where((g_lat >= -90.0) & (g_lat <= 90.0))
g_idx_invalid = np.where(((g_lat < -90.0) | (g_lat > 90.0)))
goes_indices = np.full(g_lat.shape, -1, dtype=np.int64)
goes_indices[g_idx_valid] = ps.from_latlon(
g_lat[g_idx_valid], g_lon[g_idx_valid],
int(gd.resolution(goes_ds['elemRes'][0])))
##### Allocate MERRA-2 arrays co-aligned with GOES
m2_src_coord_h5 = np.full(g_lat.shape, -1, dtype=np.int64)
m2_tpw_h5 = np.full(g_lat.shape, -1, dtype=np.int64)
#####
join_resolution = m2_resolution
join = SortedDict()
ktr = 0
for k in range(len(g_idx_valid[0])):
id = g_idx_valid[0][k]
jk = gd.spatial_clear_to_resolution(
gd.spatial_coerce_resolution(goes_indices[id], join_resolution))
if jk not in join.keys():
join[jk] = join_value()
join[jk].add(goes_bandname, id)
ktr = ktr + 1
# if ktr > 10:
# break
# # exit();
for k in range(len(m2_indices)):
jk = gd.spatial_clear_to_resolution(m2_indices[k])
if jk not in join.keys():
join[jk] = join_value()
join[jk].add('m2', k)
###########################################################################
tpw_scale = 0.001
tpw_offset = 0
###########################################################################
##### JOIN
jkeys = join.keys()
ktr = 0
nktr = len(jkeys) # gd_idx_valid is a tuple with one element
dktr = nktr / 10.0
elements_pushed = 0
print('Push joined m2 data into the dataset n = ', nktr)
for k in range(nktr):
ktr = ktr + 1
if (ktr % int(dktr)) == 0:
print(int((10.0 * ktr) / dktr), '% complete, ', elements_pushed,
' elements pushed.')
sid = jkeys[k]
if join[sid].contains(goes_bandname):
if join[sid].contains('m2'):
m2s = join[sid].get('m2')[0] # Grab the first one
m2_src_coord_h5[join[sid].get(goes_bandname)] = m2s
# m2_tpw_h5[join[sid].get(goes_bandname)] = (m2_data_flat[m2s]-tpw_offset)/tpw_scale
avg = (np.mean(m2_data_flat[join[sid].get('m2')]) -
tpw_offset) / tpw_scale
m2_tpw_h5[join[sid].get(goes_bandname)] = avg
elements_pushed = elements_pushed + len(
join[sid].get(goes_bandname))
###########################################################################
##### HDF5 SAVE DATASET
workFile = h5.File(workFileName, 'w')
image_ds = workFile.create_dataset('image', [goes_ds['data'].size],
dtype=image_dtype)
image_description_ds = workFile.create_dataset(
'image_description', [], dtype=image_description_dtype)
m2_description_ds = workFile.create_dataset('merra2_description', [],
dtype=m2_description_dtype)
workFile['/image']['stare_spatial'] = goes_indices[:]
workFile['/image']['stare_temporal'] = gd.goes10_img_stare_time(goes_ds)[0]
workFile['/image']['goes_src_coord'] = np.arange(g_lat.size,
dtype=np.int64)
workFile['/image']['merra2_src_coord'] = m2_src_coord_h5.flatten()
workFile['/image']['merra2_tpw'] = m2_tpw_h5.flatten()
# oops
# workFile['/image_description']['nx'] = goes_ds['data'].shape[1]
# workFile['/image_description']['ny'] = goes_ds['data'].shape[2]
workFile['/image_description']['nx'] = goes_ds['data'].shape[2]
workFile['/image_description']['ny'] = goes_ds['data'].shape[1]
print('image nx,ny: ', goes_ds['data'].shape[2], goes_ds['data'].shape[1])
workFile['/merra2_description']['nx'] = 576
workFile['/merra2_description']['ny'] = 361
workFile['/merra2_description']['tpw_offset'] = tpw_offset
workFile['/merra2_description']['tpw_scale'] = tpw_scale
# workFile['/image']['goes_b3'] = goes_ds['data'][0,:,:].flatten()
# workFile['/image']['goes_b4'] = goes_ds['data'][0,:,:].flatten()
while igoes < len(goes_filenames_valid):
print(i, ' saving ', goes_bandname, ' from file ',
goes_filenames_valid[igoes])
workFile['/image'][goes_bandname] = goes_ds['data'][0, :, :].flatten()
goes_ds.close()
igoes = igoes + 1
# Assume remaining GOES bands have the same image sizes and locations.
if igoes < len(goes_filenames_valid):
goes_band = goes_filenames_valid[igoes].split('.')[4]
goes_ds = Dataset(goes_datapath + goes_filenames_valid[igoes])
goes_bandname = goes_bandnames[goes_band]
workFile.close()
return
def main():
###########################################################################
# Data source
dataPath = "/home/mrilee/data/"
###########################################################################
# MODIS
modis_base = "MOD05_L2."
# modis_item = "A2005349.2120.061.2017294065852"
# modis_time_start = "2005-12-15T21:20:00"
modis_item = "A2005349.2125.061.2017294065400"
modis_time_start = "2005-12-15T21:25:00"
modis_suffix = ".hdf"
modis_filename = modis_base + modis_item + modis_suffix
# hdf = SD(dataPath+modis_filename,SDC.READ)
# ds_wv_nir = hdf.select('Water_Vapor_Near_Infrared')
fmt_suffix = ".h5"
workFileName = "sketchG." + modis_base + modis_item + fmt_suffix
print('loading ', workFileName)
workFile = h5.File(workFileName, 'r')
sids = workFile['/image']['stare_spatial']
lat = workFile['/image']['Latitude']
lon = workFile['/image']['Longitude']
data = workFile['/image']['Water_Vapor_Near_Infrared']
workFile.close()
modis_min = np.amin(data)
modis_max = np.amax(data)
sids = sids - 1
###########################################################################
# GOES
goes_file = 'sketch9.2005.349.213015.h5'
workFileName = goes_file
workFile = h5.File(workFileName, 'r')
goes_sids = workFile['/image']['stare_spatial']
goes_data = workFile['/image']['goes_b3']
workFile.close()
print('goes mnmx: ', np.amin(goes_data), np.amax(goes_data))
goes_min = np.amin(goes_data)
goes_max = np.amax(goes_data)
goes_sids = goes_sids - 1
###########################################################################
# Plotting
nrows = 2
ncols = 3
nrows = 1
ncols = 1
proj = ccrs.PlateCarree()
# proj = ccrs.Mollweide()
# proj = ccrs.Mollweide(central_longitude=-160.0)
transf = ccrs.Geodetic()
# https://stackoverflow.com/questions/33942233/how-do-i-change-matplotlibs-subplot-projection-of-an-existing-axis
# plt.figure()
fig, axs = plt.subplots(nrows=nrows,
ncols=ncols,
subplot_kw={'projection': proj})
# axs.set_facecolor('k')
# axs.patch.set_facecolor('black')
# axs.set_facecolor('black')
if nrows * ncols == 1:
fig = [fig]
axs = [axs]
goes_line = [False, False, False]
modis_line = [False, False, False]
cover_plot = [True, True, True]
goes_plot_1 = [True, False, True]
goes_plot_1_points = [True, False, True]
modis_plot_1 = [False, True, True]
plt_show_1 = [False, False, True]
goes_line = [False, False, False, True, False, True]
modis_line = [False, False, False, False, True, True]
cover_plot = [False, False, False, False, False, False]
goes_plot_1 = [True, False, True, True, False, True]
goes_plot_1_points = [False, False, False, True, False, True]
modis_plot_1 = [False, True, True, False, True, True]
modis_plot_1_points = [False, False, False, False, True, True]
plt_show_1 = [False, False, True, False, False, True]
irow = [0, 0, 0, 1, 1, 1]
icol = [0, 1, 2, 0, 1, 2]
coastline_color = 'black'
coastline_color = 'black'
# blend
blend_tripcolor_1 = True
blend_tripcolor_1_res = 10
# blend_tripcolor_1_res = 9 # FFE
# blend_tripcolor_1_res = 6 # Test
blend_tripcolor_1_cmap = None
blend_tripcolor_1_alpha = 1
blend_tripcolor_1_gamma_g = 0.65
blend_tripcolor_1_gamma_m = 0.65
if blend_tripcolor_1:
goes_plot_1 = [False] * 6
modis_plot_1 = [False] * 6
# coastline_color = 'white'
coastline_color = 'black'
# 2020-0125 pix 1
# goes_plot_1_res = 9
# modis_plot_1_res = 9
#
# goes_plot_1_res = 6
# modis_plot_1_res = 6
#
# plot_1_res = 9 # FFE
plot_1_res = 6
goes_plot_1_res = plot_1_res
modis_plot_1_res = plot_1_res
# Colors
goes_plot_1_tripcolor = 'Reds'
modis_plot_1_tripcolor = 'Blues'
#
common_alpha = 0.7
goes_plot_1_alpha = common_alpha
modis_plot_1_alpha = common_alpha
# recalculate=[True,False,False,True,False,False]
recalculate = [True, False, True, True, False, True]
cover_rads = [2.0, 0, 2, 0.125, 0, 0.125]
# cover_rads =[2.0,0,0, 0.125,0,0]
circle_plot = [False, False, False, False, False, False]
circle_color = [
'White', 'lightgrey', 'White', 'navajowhite', 'khaki', 'White'
]
modis_scatter_color = [
'darkcyan', 'darkcyan', 'darkcyan', 'darkcyan', 'cyan', 'cyan'
]
nodes_cover = [1, 2, 1, 1, 2, 1] # 1 == goes, 2 == modis, 0 == None
# nodes_cover=[0,0,0,0,0,0]
subplot_title = [
"ROI+GOES", "ROI+MODIS", "ROI+GOES+MODIS", None, None, None
]
# for iter in range(6):
# for iter in [2,5]:
if True:
iter = 2
###########################################################################
if recalculate[iter]:
print('recalculating iter = ', iter)
###########################################################################
cover_resolution = 11
# cover_resolution = 12
cover_type = 'circular'
# cover_resolution = 6
#+ cover_resolution = 5
#+ cover_type = 'bounding_box'
if cover_type == 'circular':
###########################################################################
# HI 28.5N 177W
# Near the Big Island
cover_lat = 19.5 - 0.375
cover_lon = -155.5 + 0.375
# Midway Island
# cover_lat = 28.2
# cover_lon = -177.35
# Ni'ihau
# cover_lat = 21.9
# cover_lon = -160.17
cover_rad = cover_rads[iter]
cover = ps.to_circular_cover(cover_lat, cover_lon, cover_rad,
cover_resolution)
# ,range_size_limit=2000)
elif cover_type == 'bounding_box':
# Set cover to "bounding box."
cover_lat = np.array([15, 15, 38, 38], dtype=np.float)
cover_lon = np.array([-174, -145, -145, -174], dtype=np.float)
cover = ps.to_hull_range_from_latlon(cover_lat, cover_lon,
cover_resolution)
cover_cat = catalog(resolution=cover_resolution, sids=cover)
cover_sids_min = np.amin(cover)
cover_sids_max = np.amax(cover) # Need to convert to terminator
cover_sids_max = gd.spatial_terminator(cover_sids_max)
# for k in list(cover_cat.sdict.keys()):
# print('cc: ',hex(k))
###########################################################################
#
gm_cat_resolution = 5
gm_catalog = catalog(resolution=gm_cat_resolution)
k = 0
for i in range(10):
while (goes_sids[k] < 0):
k = k + 1
# print('adding: ','0x%016x'%goes_sids[k],k)
gm_catalog.add('goes', goes_sids[k], goes_data[k])
k = k + 1
for i in range(10):
# print('adding: ','0x%016x'%sids[i])
gm_catalog.add('modis', sids[i], data[i])
k = 0
# for i in range(10):
idx = np.arange(
goes_sids.size)[np.where((goes_sids > cover_sids_min)
& (goes_sids < cover_sids_max))]
for k in range(len(idx)):
# while(goes_sids[k]<0):
# k=k+1
if goes_sids[idx[k]] > 0:
cover_cat.add_to_entry('goes', goes_sids[idx[k]],
goes_data[idx[k]])
# k=k+1
idx = np.arange(sids.size)[np.where((sids > cover_sids_min)
& (sids < cover_sids_max))]
for k in range(len(idx)):
if sids[idx[k]] > 0:
cover_cat.add_to_entry('modis', sids[idx[k]], data[idx[k]])
# print(yaml.dump(gm_catalog))
# exit()
#
###########################################################################
print('plotting iter ', iter)
if nrows * ncols == 1:
ax = axs[0]
else:
ax = axs[irow[iter], icol[iter]]
if subplot_title[iter] is not None:
ax.set_title(subplot_title[iter])
if False:
ax.set_global()
if True:
ax.coastlines(color=coastline_color)
if iter == 0:
x0 = 0.05
y0 = 0.025
dy = 0.025
plt.figtext(x0,
y0 + 0 * dy,
"MODIS: " + "sketchG." + modis_base + modis_item +
fmt_suffix +
', Water_Vapor_Near_Infrared, resolution = %i' %
(sids[10000] & 31),
fontsize=10)
k = 0
while goes_sids[k] < 0:
k = k + 1
plt.figtext(x0,
y0 + 1 * dy,
"GOES: " + goes_file +
' BAND_3 (6.7mu), resolution = %i' %
(goes_sids[k] & 31),
fontsize=10)
if cover_type == 'circular':
plt.figtext(
x0,
y0 + 2 * dy,
"ROI Cover: resolution = %d, radius = %0.2f (upper) %0.3f (lower) degrees, center = 0x%016x"
% (cover_resolution, cover_rads[0], cover_rads[3],
ps.from_latlon(npf64([cover_lat]), npf64([cover_lon]),
cover_resolution)[0]),
fontsize=10)
# plt.show()
# exit()
if False:
lli = ps.triangulate_indices(cover)
ax.triplot(tri.Triangulation(lli[0], lli[1], lli[2]),
'g-',
transform=transf,
lw=1,
markersize=3)
if True:
if goes_plot_1[iter]:
cc_data = cover_cat.get_all_data('goes')
csids, sdat = zip(*[cd.as_tuple() for cd in cc_data])
if goes_plot_1_points[iter]:
glat, glon = ps.to_latlon(csids)
# csids_at_res = list(map(gd.spatial_clear_to_resolution,csids))
# cc_data_accum = dict()
# for cs in csids_at_res:
# cc_data_accum[cs] = []
# for ics in range(len(csids_at_res)):
# cc_data_accum[csids_at_res[ics]].append(sdat[ics])
# for cs in cc_data_accum.keys():
# if len(cc_data_accum[cs]) > 1:
# cc_data_accum[cs] = [sum(cc_data_accum[cs])/(1.0*len(cc_data_accum[cs]))]
# tmp_values = np.array(list(cc_data_accum.values()))
# vmin = np.amin(tmp_values)
# vmax = np.amax(np.array(tmp_values))
cc_data_accum, vmin, vmax = gd.simple_collect(
csids, sdat, force_resolution=goes_plot_1_res)
# print('a100: ',cc_data)
# print('cc_data type: ',type(cc_data))
# print('cc_data[0] type: ',type(cc_data[0]))
for cs in cc_data_accum.keys():
# print('item: ',hex(cs),cc_data_accum[cs])
lli = ps.triangulate_indices([cs])
triang = tri.Triangulation(lli[0], lli[1], lli[2])
cd_plt = np.array(cc_data_accum[cs])
# print('cd_plt type ',type(cd_plt))
# print('cd_plt shape ',cd_plt.shape)
# print('cd_plt type ',type(cd_plt[0]))
if goes_line[iter]:
ax.triplot(triang,
'r-',
transform=transf,
lw=1.5,
markersize=3,
alpha=0.5)
# ax.tripcolor(triang,facecolors=cd_plt,vmin=goes_min,vmax=goes_max,cmap='Reds',alpha=0.4)
ax.tripcolor(triang,
facecolors=cd_plt,
edgecolors='k',
lw=0,
shading='flat',
vmin=vmin,
vmax=vmax,
cmap=goes_plot_1_tripcolor,
alpha=goes_plot_1_alpha)
# for cd in cc_data:
# lli = ps.triangulate_indices([cd.sid])
# triang = tri.Triangulation(lli[0],lli[1],lli[2])
# cd_plt = np.array([cd.datum])
# if goes_line[iter]:
# ax.triplot(triang,'r-',transform=transf,lw=3,markersize=3,alpha=0.5)
# ax.tripcolor(triang,facecolors=cd_plt,vmin=goes_min,vmax=goes_max,cmap='Reds',alpha=0.4)
if modis_plot_1[iter]:
cc_data_m = cover_cat.get_all_data('modis')
csids, sdat = zip(*[cd.as_tuple() for cd in cc_data_m])
# mlat,mlon = ps.to_latlon(csids)
cc_data_m_accum, vmin, vmax = gd.simple_collect(
csids, sdat, force_resolution=modis_plot_1_res)
for cs in cc_data_m_accum.keys():
lli = ps.triangulate_indices([cs])
triang = tri.Triangulation(lli[0], lli[1], lli[2])
cd_plt = np.array(cc_data_m_accum[cs])
# print('lli[0] len ',len(lli[0]))
# print('cd_plt len ', len(cd_plt))
# print('cd_plt type ',type(cd_plt))
# print('cd_plt shape ',cd_plt.shape)
# print('cd_plt type ',type(cd_plt[0]))
if modis_line[iter]:
ax.triplot(triang,
'b-',
transform=transf,
lw=1.5,
markersize=3,
alpha=0.5)
# ax.tripcolor(triang,facecolors=cd_plt,vmin=goes_min,vmax=goes_max,cmap='Blues',alpha=0.4)
ax.tripcolor(triang,
facecolors=cd_plt,
edgecolors='k',
lw=0,
shading='flat',
vmin=vmin,
vmax=vmax,
cmap=modis_plot_1_tripcolor,
alpha=modis_plot_1_alpha)
# for cd in cc_data_m:
# lli = ps.triangulate_indices([cd.sid])
# triang = tri.Triangulation(lli[0],lli[1],lli[2])
# cd_plt = np.array([cd.datum])
# if modis_line[iter]:
# ax.triplot(triang,'b-',transform=transf,lw=1,markersize=3,alpha=0.5)
# ax.tripcolor(triang,facecolors=cd_plt,vmin=modis_min,vmax=modis_max,cmap='Blues',alpha=0.4)
if modis_plot_1_points[iter]:
mlat, mlon = ps.to_latlon(csids)
ax.scatter(mlon, mlat, s=8, c=modis_scatter_color[iter])
# ax.scatter(mlon,mlat,s=8,c='cyan')
# ax.scatter(mlon,mlat,s=8,c='darkcyan')
# blend_tripcolor_1 = False
# blend_tripcolor_res_1 = 6
# blend_tripcolor_1_cmap = None
# blend_tripcolor_1_alpha = 1
if blend_tripcolor_1:
cc_data = cover_cat.get_all_data('goes')
csids, sdat = zip(*[cd.as_tuple() for cd in cc_data])
cc_data_accum, vmin, vmax = gd.simple_collect(
csids, sdat, force_resolution=blend_tripcolor_1_res)
cc_data_m = cover_cat.get_all_data('modis')
csids_m, sdat_m = zip(*[cd.as_tuple() for cd in cc_data_m])
cc_data_m_accum, vmin_m, vmax_m = gd.simple_collect(
csids_m, sdat_m, force_resolution=blend_tripcolor_1_res)
data_accum_keys = set()
for cs in cc_data_accum.keys():
data_accum_keys.add(cs)
for cs in cc_data_m_accum.keys():
data_accum_keys.add(cs)
for cs in data_accum_keys:
# print('item: ',hex(cs),cc_data_accum[cs])
lli = ps.triangulate_indices([cs])
triang = tri.Triangulation(lli[0], lli[1], lli[2])
try:
cd_plt_g = (np.array(cc_data_accum[cs]) -
vmin) / (vmax - vmin)
cd_plt_g = cd_plt_g**blend_tripcolor_1_gamma_g
except:
cd_plt_g = np.array([0])
try:
cd_plt_m = (np.array(cc_data_m_accum[cs]) -
vmin_m) / (vmax_m - vmin_m)
cd_plt_m = cd_plt_m**blend_tripcolor_1_gamma_m
except:
cd_plt_m = np.array([0])
######
# blend 1 & 2
# cd_plt = np.array([[cd_plt_g,0,cd_plt_m]])
######
# blend 3
# print('len: ',cd_plt_g.shape,cd_plt_m.shape)
cd_plt = np.array([[
cd_plt_g[0], 0.5 * (cd_plt_g + cd_plt_m)[0],
cd_plt_m[0]
]])
cd_cmp = colors_to_cmap(cd_plt)
zs = np.asarray(range(3), dtype=np.float) / 2.0
ax.tripcolor(
triang,
zs,
cmap=cd_cmp
# ,facecolors=cd_plt
,
edgecolors='k',
lw=0,
shading='gouraud'
# ,shading='flat'
# ,vmin=vmin,vmax=vmax
# ,cmap=blend_tripcolor_1_cmap
,
alpha=blend_tripcolor_1_alpha)
# ,vmin=vmin,vmax=vmax,cmap=blend_tripcolor_1_cmap,alpha=blend_tripcolor_1_alpha)
if goes_plot_1[iter]:
if goes_plot_1_points[iter]:
ax.scatter(glon, glat, s=8, c='black')
if nodes_cover[iter] > 0:
if nodes_cover[iter] == 1:
cc_data_ = cover_cat.get_all_data('goes')
else:
cc_data_ = cover_cat.get_all_data('modis')
sids_, dat_ = zip(*[cd.as_tuple() for cd in cc_data_])
# print('sids_ len: ',len(sids_))
sids_test = gd.spatial_clear_to_resolution(
npi64([
gd.spatial_coerce_resolution(s, gm_cat_resolution)
for s in sids_
]))
# print('sids_tlen: ',len(sids_test))
if cover_type == 'circular':
print('cover: 0x%016x' %
ps.from_latlon(npf64([cover_lat]), npf64(
[cover_lon]), cover_resolution)[0])
geom_test = sid_geometry(sids_test)
for s in geom_test.triangles.keys():
print(iter, ' 0x%016x' % s)
triang_test = geom_test.triang()
# ax.triplot(triang_test,'g-',transform=transf,lw=1.0,markersize=3,alpha=0.75)
ax.triplot(triang_test,
'k-',
transform=transf,
lw=1.0,
markersize=3,
alpha=0.5)
if False:
for i in range(0, 10):
k = cover_cat.sdict.peekitem(i)[0]
triang = cover_cat.sdict[k].geometry.triang()
ax.triplot(triang,
'b-',
transform=transf,
lw=1,
markersize=3,
alpha=0.5)
if cover_plot[iter]:
# lli = ps.triangulate_indices(ps.expand_intervals(cover,9,result_size_limit=2048))
lli = ps.triangulate_indices(cover)
ax.triplot(tri.Triangulation(lli[0], lli[1], lli[2]),
'k-',
transform=transf,
lw=1,
markersize=3,
alpha=0.5)
# ,'g-',transform=transf,lw=1,markersize=3)
if False:
# k = gm_catalog.sdict.keys()[0]
# for k in gm_catalog.sdict.keys():
for i in range(0, 3):
k = gm_catalog.sdict.peekitem(i)[0]
triang = gm_catalog.sdict[k].geometry.triang()
ax.triplot(triang,
'r-',
transform=transf,
lw=1,
markersize=3)
if circle_plot[iter]:
phi = np.linspace(0, 2 * np.pi, 64)
# rad=cover_rad
rad = 0.125
ax.plot(cover_lon + rad * np.cos(phi),
cover_lat + rad * np.sin(phi),
transform=transf,
color=circle_color[iter])
# ax.set_facecolor('k')
if plt_show_1[iter]:
plt.show()
###########################################################################
#
# if False:
# sw_timer.stamp('triangulating')
# print('triangulating')
# client = Client()
# for lli_ in slam(client,ps.triangulate_indices,sids):
# sw_timer.stamp('slam iteration')
# print('lli_ type: ',type(lli_))
# lli = lli_.result()
# sw_timer.stamp('slam result')
# print('lli type: ',type(lli))
# triang = tri.Triangulation(lli[0],lli[1],lli[2])
# sw_timer.stamp('slam triang')
# plt.triplot(triang,'r-',transform=transf,lw=1.5,markersize=3,alpha=0.5)
# sw_timer.stamp('slam triplot')
#
# sw_timer.stamp('plt show')
# # lons,lats,intmat=ps.triangulate_indices(sids)
# # triang = tri.Triangulation(lons,lats,intmat)
# # plt.triplot(triang,'r-',transform=transf,lw=1.5,markersize=3)
#
# plt.show()
# client.close()
print(sw_timer.report_all())
def hex16(i):
return "0x%016x" % i
start2 = timer()
print('join prep done, time12 = ', start2 - start1)
join_resolution = m2_resolution
join = SortedDict()
ktr = 0
print('add goes b5 to join')
for k in range(len(g_idx_valid[0])):
id = g_idx_valid[0][k]
jk = gd.spatial_clear_to_resolution(
gd.spatial_coerce_resolution(goes_b5_indices[id], join_resolution))
# print('id,jk: ',id,hex16(jk))
# print('keys: ',[hex16(i) for i in join.keys()])
if jk not in join.keys():
join[jk] = join_value()
join[jk].add('goes_b5', id)
ktr = ktr + 1
# if ktr > 10:
# break
# # exit();
start2a = timer()
print('add goes b5 to join, time = ', start2a - start2)
print('add MERRA-2 to join')
for k in range(len(m2_indices)):
def join(self):
"Join goes files with a particular m2_file."
sw_timer.stamp('join_goes_and_m2-join-start')
###########################################################################
##### MERRA-2
sw_timer.stamp('join_goes_and_m2-start-merra2-start')
m2_dLat = 0.5
m2_dLon = 5.0 / 8.0
m2_dLatkm = m2_dLat * gd.re_km / gd.deg_per_rad
m2_dLonkm = m2_dLon * gd.re_km / gd.deg_per_rad
sw_timer.stamp('join_goes_and_m2-start-merra2-dataset-start')
m2_ds = Dataset(self.m2_datapath + self.m2_file_name)
sw_timer.stamp('join_goes_and_m2-start-merra2-dataset-end')
m2_lat, m2_lon = np.meshgrid(m2_ds['lat'], m2_ds['lon'])
m2_lat = m2_lat.flatten()
m2_lon = m2_lon.flatten()
m2_idx_ij = np.arange(m2_lat.size, dtype=np.int64)
m2_resolution = int(gd.resolution(m2_dLonkm * 2))
m2_indices = ps.from_latlon(m2_lat, m2_lon, m2_resolution)
m2_term = gd.spatial_terminator(m2_indices)
m2_tid = gd.merra2_stare_time(m2_ds)
sw_timer.stamp('join_goes_and_m2-start-merra2-end')
###########################################################################
##### GOES
sw_timer.stamp('join_goes_and_m2-start-goes-start')
self.igoes = 0
self.goes_band = self.goes_filenames_valid[self.igoes].split('.')[4]
self.goes_bandname = self.goes_bandnames[self.goes_band]
sw_timer.stamp('join_goes_and_m2-start-goes-start-dataset-start')
self.goes_ds = Dataset(self.goes_datapath +
self.goes_filenames_valid[self.igoes])
sw_timer.stamp('join_goes_and_m2-start-goes-start-dataset-end')
goes_tid = gd.goes10_img_stare_time(self.goes_ds)
##### MERRA-2 at the GOES time
fine_match = ps.cmp_temporal(np.array(goes_tid, dtype=np.int64),
m2_tid)
# print('fine_match: ',fine_match)
# print('gtid: ',[hex16(i) for i in goes_tid])
# print('m2_tid: ',[hex16(i) for i in m2_tid])
m2_ifm = np.nonzero(fine_match)[0]
# print('m2_ifm: ',m2_ifm)
if m2_ifm.size == 1:
m2_dataDayI = m2_ds['TQI'][m2_ifm, :, :]
m2_dataDayL = m2_ds['TQL'][m2_ifm, :, :]
m2_dataDayV = m2_ds['TQV'][m2_ifm, :, :]
else:
m2_dataDayI = np.mean(m2_ds['TQI'][m2_ifm, :, :], 0)
m2_dataDayL = np.mean(m2_ds['TQL'][m2_ifm, :, :], 0)
m2_dataDayV = np.mean(m2_ds['TQV'][m2_ifm, :, :], 0)
m2_dataDay = m2_dataDayI + m2_dataDayL + m2_dataDayV
# print('m2_dataDay.shape: ',m2_dataDay.shape)
m2_data = m2_dataDay[:, :].T
# print('m2_data.shape: ',m2_data.shape)
m2_data_flat = m2_data.flatten()
# print('m2_data.flat.sh: ',m2_data_flat.shape)
g_lat = self.goes_ds['lat'][:, :].flatten()
self.g_lat_size = g_lat.size
# print('gds lat shape: ',self.goes_ds['lat'][:,:].shape)
# print('gds lat fltsh: ',g_lat.shape)
g_lon = self.goes_ds['lon'][:, :].flatten()
g_idx_valid = np.where((g_lat >= -90.0) & (g_lat <= 90.0))
g_idx_invalid = np.where(((g_lat < -90.0) | (g_lat > 90.0)))
self.goes_indices = np.full(g_lat.shape, -1, dtype=np.int64)
sw_timer.stamp('join_goes_and_m2-start-goes-from_latlon-start')
self.goes_indices[g_idx_valid] = ps.from_latlon(
g_lat[g_idx_valid], g_lon[g_idx_valid],
int(gd.resolution(self.goes_ds['elemRes'][0])))
self.goes_indices = self.goes_indices
sw_timer.stamp('join_goes_and_m2-start-goes-from_latlon-end')
###########################################################################
##### Allocate MERRA-2 arrays co-aligned with GOES
self.m2_src_coord_h5 = np.full(g_lat.shape, -1, dtype=np.int64)
self.m2_tpw_h5 = np.full(g_lat.shape, -1, dtype=np.int64)
sw_timer.stamp('join_goes_and_m2-start-join')
#####
join_resolution = m2_resolution
join = SortedDict()
ktr = 0
for k in range(len(g_idx_valid[0])):
id = g_idx_valid[0][k]
jk = gd.spatial_clear_to_resolution(
gd.spatial_coerce_resolution(self.goes_indices[id],
join_resolution))
if jk not in join.keys():
join[jk] = join_value()
join[jk].add(self.goes_bandname, id)
ktr = ktr + 1
# if ktr > 10:
# break
# # exit();
for k in range(len(m2_indices)):
jk = gd.spatial_clear_to_resolution(m2_indices[k])
if jk not in join.keys():
join[jk] = join_value()
join[jk].add('m2', k)
###########################################################################
self.tpw_scale = 0.001
self.tpw_offset = 0
###########################################################################
##### JOIN
# TODO Add metadata for traceability.
jkeys = join.keys()
ktr = 0
nktr = len(jkeys) # gd_idx_valid is a tuple with one element
ktr_max = nktr
elements_pushed = 0
print('Push joined m2 data into the dataset n = ', nktr)
for k in range(nktr):
ktr = ktr + 1
if self.verbose_progress:
if int(100.0 * ktr / ktr_max) % 5 == 0 or int(
100 * ktr / ktr_max) < 2:
print(
'join_goes_merra2: %2d%% complete, %d elements pushed.'
% (int(100 * ktr / ktr_max), elements_pushed),
end='\r',
flush=True)
sid = jkeys[k]
if join[sid].contains(self.goes_bandname):
if join[sid].contains('m2'):
m2s = join[sid].get('m2')[0] # Grab the first one
self.m2_src_coord_h5[join[sid].get(
self.goes_bandname)] = m2s
# self.m2_tpw_h5[join[sid].get(self.goes_bandname)] = (m2_data_flat[m2s]-self.tpw_offset)/self.tpw_scale
avg = (np.mean(m2_data_flat[join[sid].get('m2')]) -
self.tpw_offset) / self.tpw_scale
self.m2_tpw_h5[join[sid].get(self.goes_bandname)] = avg
elements_pushed = elements_pushed + len(join[sid].get(
self.goes_bandname))
print('join_goes_merra2: done, %d elements pushed. ' %
(elements_pushed),
flush=True)
print('')
sw_timer.stamp('join_goes_and_m2-join-end')
sw_timer.stamp('join_goes_and_m2-start-goes-end')
return self
