def add(self,key,sid,datum,resolution=None): if resolution is not None: sidtest = gd.spatial_clear_to_resolution(gd.spatial_coerce_resolution(sid,resolution)) elif self.resolution is not None: sidtest = gd.spatial_clear_to_resolution(gd.spatial_coerce_resolution(sid,self.resolution)) else: sidtest = sid if sidtest not in self.sdict.keys(): self.sdict[sidtest] = catalog_entry(sidtest) # construct with relevant resolution self.sdict[sidtest].add(key,data_entry(sid,datum)) return
def add_to_entry(self,key,sid,datum): "Add data to entry if it's there." if self.resolution is not None: sid_test = gd.spatial_clear_to_resolution(gd.spatial_coerce_resolution(sid,self.resolution)) else: sid_test = sid # print('testing ',hex(sid_test), hex(sid)) entry_open = sid_test in self.sdict.keys() if entry_open: # print(key,' adding ',data,' to ',hex(sid)) self.add(key,sid,datum) return entry_open
def build_coarser_level(fnames): sares = [sare_from_fname(i) for i in fnames] resolutions = [gd.spatial_resolution(i) for i in sares] levels = set(resolutions) # print('levels: ',levels) if len(levels) != 1: raise ValueError('Filenames indicate multiple resolutions.') resolution = resolutions[0] coarser = {} for i in fnames: si = gd.spatial_clear_to_resolution( gd.spatial_coerce_resolution(sare_from_fname(i), resolution - 1)) if si not in coarser.keys(): coarser[si] = [] coarser[si].append(i) return coarser
def join_goes_and_m2_to_h5(goes_datapath, goes_filenames, m2_datapath, m2_file_name, workFileName): ##### HDF5 Data types for output image_dtype = np.dtype([('stare_spatial', np.int64), ('stare_temporal', np.int64), ('goes_src_coord', np.int64), ('goes_b3', np.int64), ('goes_b4', np.int64), ('goes_b5', np.int64), ('merra2_src_coord', np.int64), ('merra2_tpw', np.int64)]) image_description_dtype = np.dtype([('nx', np.int), ('ny', np.int)]) m2_description_dtype = np.dtype([('nx', np.int), ('ny', np.int), ('tpw_offset', np.double), ('tpw_scale', np.double)]) goes_bandnames = { "BAND_03": "goes_b3", "BAND_04": "goes_b4", "BAND_05": "goes_b5" } goes_filenames_valid = [] for i in goes_filenames: goes_band = i.split('.')[4] if goes_band in goes_bandnames.keys(): goes_filenames_valid.append(i) if len(goes_filenames_valid) == 0: print("*ERROR* Join and save -- no valid GOES filenames. Returning.") return # print('goes_filenames: ',goes_filenames) # print('goes_filenames_valid: ',goes_filenames_valid) ########################################################################### ##### MERRA-2 m2_dLat = 0.5 m2_dLon = 5.0 / 8.0 m2_dLatkm = m2_dLat * gd.re_km / gd.deg_per_rad m2_dLonkm = m2_dLon * gd.re_km / gd.deg_per_rad m2_ds = Dataset(m2_datapath + m2_file_name) m2_lat, m2_lon = np.meshgrid(m2_ds['lat'], m2_ds['lon']) m2_lat = m2_lat.flatten() m2_lon = m2_lon.flatten() m2_idx_ij = np.arange(m2_lat.size, dtype=np.int64) m2_resolution = int(gd.resolution(m2_dLonkm * 2)) m2_indices = ps.from_latlon(m2_lat, m2_lon, m2_resolution) m2_term = gd.spatial_terminator(m2_indices) m2_tid = gd.merra2_stare_time(m2_ds) ########################################################################### ##### GOES igoes = 0 goes_band = goes_filenames_valid[igoes].split('.')[4] goes_bandname = goes_bandnames[goes_band] goes_ds = Dataset(goes_datapath + goes_filenames_valid[igoes]) goes_tid = gd.goes10_img_stare_time(goes_ds) ##### MERRA-2 at the GOES time fine_match = ps.cmp_temporal(np.array(goes_tid, dtype=np.int64), m2_tid) m2_ifm = np.nonzero(fine_match)[0] m2_dataDayI = m2_ds['TQI'][m2_ifm, :, :] m2_dataDayL = m2_ds['TQL'][m2_ifm, :, :] m2_dataDayV = m2_ds['TQV'][m2_ifm, :, :] m2_dataDay = m2_dataDayI + m2_dataDayL + m2_dataDayV if m2_ifm.size > 1: print('multiple hits on 2m') m2_dataDay = np.mean(m2_dataDay, axis=1) print('m2_dataDay shape: ', n2_dataDay.shape) m2_data = m2_dataDay[:, :].T m2_data_flat = m2_data.flatten() g_lat = goes_ds['lat'][:, :].flatten() g_lon = goes_ds['lon'][:, :].flatten() g_idx_valid = np.where((g_lat >= -90.0) & (g_lat <= 90.0)) g_idx_invalid = np.where(((g_lat < -90.0) | (g_lat > 90.0))) goes_indices = np.full(g_lat.shape, -1, dtype=np.int64) goes_indices[g_idx_valid] = ps.from_latlon( g_lat[g_idx_valid], g_lon[g_idx_valid], int(gd.resolution(goes_ds['elemRes'][0]))) ##### Allocate MERRA-2 arrays co-aligned with GOES m2_src_coord_h5 = np.full(g_lat.shape, -1, dtype=np.int64) m2_tpw_h5 = np.full(g_lat.shape, -1, dtype=np.int64) ##### join_resolution = m2_resolution join = SortedDict() ktr = 0 for k in range(len(g_idx_valid[0])): id = g_idx_valid[0][k] jk = gd.spatial_clear_to_resolution( gd.spatial_coerce_resolution(goes_indices[id], join_resolution)) if jk not in join.keys(): join[jk] = join_value() join[jk].add(goes_bandname, id) ktr = ktr + 1 # if ktr > 10: # break # # exit(); for k in range(len(m2_indices)): jk = gd.spatial_clear_to_resolution(m2_indices[k]) if jk not in join.keys(): join[jk] = join_value() join[jk].add('m2', k) ########################################################################### tpw_scale = 0.001 tpw_offset = 0 ########################################################################### ##### JOIN jkeys = join.keys() ktr = 0 nktr = len(jkeys) # gd_idx_valid is a tuple with one element dktr = nktr / 10.0 elements_pushed = 0 print('Push joined m2 data into the dataset n = ', nktr) for k in range(nktr): ktr = ktr + 1 if (ktr % int(dktr)) == 0: print(int((10.0 * ktr) / dktr), '% complete, ', elements_pushed, ' elements pushed.') sid = jkeys[k] if join[sid].contains(goes_bandname): if join[sid].contains('m2'): m2s = join[sid].get('m2')[0] # Grab the first one m2_src_coord_h5[join[sid].get(goes_bandname)] = m2s # m2_tpw_h5[join[sid].get(goes_bandname)] = (m2_data_flat[m2s]-tpw_offset)/tpw_scale avg = (np.mean(m2_data_flat[join[sid].get('m2')]) - tpw_offset) / tpw_scale m2_tpw_h5[join[sid].get(goes_bandname)] = avg elements_pushed = elements_pushed + len( join[sid].get(goes_bandname)) ########################################################################### ##### HDF5 SAVE DATASET workFile = h5.File(workFileName, 'w') image_ds = workFile.create_dataset('image', [goes_ds['data'].size], dtype=image_dtype) image_description_ds = workFile.create_dataset( 'image_description', [], dtype=image_description_dtype) m2_description_ds = workFile.create_dataset('merra2_description', [], dtype=m2_description_dtype) workFile['/image']['stare_spatial'] = goes_indices[:] workFile['/image']['stare_temporal'] = gd.goes10_img_stare_time(goes_ds)[0] workFile['/image']['goes_src_coord'] = np.arange(g_lat.size, dtype=np.int64) workFile['/image']['merra2_src_coord'] = m2_src_coord_h5.flatten() workFile['/image']['merra2_tpw'] = m2_tpw_h5.flatten() # oops # workFile['/image_description']['nx'] = goes_ds['data'].shape[1] # workFile['/image_description']['ny'] = goes_ds['data'].shape[2] workFile['/image_description']['nx'] = goes_ds['data'].shape[2] workFile['/image_description']['ny'] = goes_ds['data'].shape[1] print('image nx,ny: ', goes_ds['data'].shape[2], goes_ds['data'].shape[1]) workFile['/merra2_description']['nx'] = 576 workFile['/merra2_description']['ny'] = 361 workFile['/merra2_description']['tpw_offset'] = tpw_offset workFile['/merra2_description']['tpw_scale'] = tpw_scale # workFile['/image']['goes_b3'] = goes_ds['data'][0,:,:].flatten() # workFile['/image']['goes_b4'] = goes_ds['data'][0,:,:].flatten() while igoes < len(goes_filenames_valid): print(i, ' saving ', goes_bandname, ' from file ', goes_filenames_valid[igoes]) workFile['/image'][goes_bandname] = goes_ds['data'][0, :, :].flatten() goes_ds.close() igoes = igoes + 1 # Assume remaining GOES bands have the same image sizes and locations. if igoes < len(goes_filenames_valid): goes_band = goes_filenames_valid[igoes].split('.')[4] goes_ds = Dataset(goes_datapath + goes_filenames_valid[igoes]) goes_bandname = goes_bandnames[goes_band] workFile.close() return
def main(): ########################################################################### # Data source dataPath = "/home/mrilee/data/" ########################################################################### # MODIS modis_base = "MOD05_L2." # modis_item = "A2005349.2120.061.2017294065852" # modis_time_start = "2005-12-15T21:20:00" modis_item = "A2005349.2125.061.2017294065400" modis_time_start = "2005-12-15T21:25:00" modis_suffix = ".hdf" modis_filename = modis_base + modis_item + modis_suffix # hdf = SD(dataPath+modis_filename,SDC.READ) # ds_wv_nir = hdf.select('Water_Vapor_Near_Infrared') fmt_suffix = ".h5" workFileName = "sketchG." + modis_base + modis_item + fmt_suffix print('loading ', workFileName) workFile = h5.File(workFileName, 'r') sids = workFile['/image']['stare_spatial'] lat = workFile['/image']['Latitude'] lon = workFile['/image']['Longitude'] data = workFile['/image']['Water_Vapor_Near_Infrared'] workFile.close() modis_min = np.amin(data) modis_max = np.amax(data) sids = sids - 1 ########################################################################### # GOES goes_file = 'sketch9.2005.349.213015.h5' workFileName = goes_file workFile = h5.File(workFileName, 'r') goes_sids = workFile['/image']['stare_spatial'] goes_data = workFile['/image']['goes_b3'] workFile.close() print('goes mnmx: ', np.amin(goes_data), np.amax(goes_data)) goes_min = np.amin(goes_data) goes_max = np.amax(goes_data) goes_sids = goes_sids - 1 ########################################################################### # Plotting nrows = 2 ncols = 3 nrows = 1 ncols = 1 proj = ccrs.PlateCarree() # proj = ccrs.Mollweide() # proj = ccrs.Mollweide(central_longitude=-160.0) transf = ccrs.Geodetic() # https://stackoverflow.com/questions/33942233/how-do-i-change-matplotlibs-subplot-projection-of-an-existing-axis # plt.figure() fig, axs = plt.subplots(nrows=nrows, ncols=ncols, subplot_kw={'projection': proj}) # axs.set_facecolor('k') # axs.patch.set_facecolor('black') # axs.set_facecolor('black') if nrows * ncols == 1: fig = [fig] axs = [axs] goes_line = [False, False, False] modis_line = [False, False, False] cover_plot = [True, True, True] goes_plot_1 = [True, False, True] goes_plot_1_points = [True, False, True] modis_plot_1 = [False, True, True] plt_show_1 = [False, False, True] goes_line = [False, False, False, True, False, True] modis_line = [False, False, False, False, True, True] cover_plot = [False, False, False, False, False, False] goes_plot_1 = [True, False, True, True, False, True] goes_plot_1_points = [False, False, False, True, False, True] modis_plot_1 = [False, True, True, False, True, True] modis_plot_1_points = [False, False, False, False, True, True] plt_show_1 = [False, False, True, False, False, True] irow = [0, 0, 0, 1, 1, 1] icol = [0, 1, 2, 0, 1, 2] coastline_color = 'black' coastline_color = 'black' # blend blend_tripcolor_1 = True blend_tripcolor_1_res = 10 # blend_tripcolor_1_res = 9 # FFE # blend_tripcolor_1_res = 6 # Test blend_tripcolor_1_cmap = None blend_tripcolor_1_alpha = 1 blend_tripcolor_1_gamma_g = 0.65 blend_tripcolor_1_gamma_m = 0.65 if blend_tripcolor_1: goes_plot_1 = [False] * 6 modis_plot_1 = [False] * 6 # coastline_color = 'white' coastline_color = 'black' # 2020-0125 pix 1 # goes_plot_1_res = 9 # modis_plot_1_res = 9 # # goes_plot_1_res = 6 # modis_plot_1_res = 6 # # plot_1_res = 9 # FFE plot_1_res = 6 goes_plot_1_res = plot_1_res modis_plot_1_res = plot_1_res # Colors goes_plot_1_tripcolor = 'Reds' modis_plot_1_tripcolor = 'Blues' # common_alpha = 0.7 goes_plot_1_alpha = common_alpha modis_plot_1_alpha = common_alpha # recalculate=[True,False,False,True,False,False] recalculate = [True, False, True, True, False, True] cover_rads = [2.0, 0, 2, 0.125, 0, 0.125] # cover_rads =[2.0,0,0, 0.125,0,0] circle_plot = [False, False, False, False, False, False] circle_color = [ 'White', 'lightgrey', 'White', 'navajowhite', 'khaki', 'White' ] modis_scatter_color = [ 'darkcyan', 'darkcyan', 'darkcyan', 'darkcyan', 'cyan', 'cyan' ] nodes_cover = [1, 2, 1, 1, 2, 1] # 1 == goes, 2 == modis, 0 == None # nodes_cover=[0,0,0,0,0,0] subplot_title = [ "ROI+GOES", "ROI+MODIS", "ROI+GOES+MODIS", None, None, None ] # for iter in range(6): # for iter in [2,5]: if True: iter = 2 ########################################################################### if recalculate[iter]: print('recalculating iter = ', iter) ########################################################################### cover_resolution = 11 # cover_resolution = 12 cover_type = 'circular' # cover_resolution = 6 #+ cover_resolution = 5 #+ cover_type = 'bounding_box' if cover_type == 'circular': ########################################################################### # HI 28.5N 177W # Near the Big Island cover_lat = 19.5 - 0.375 cover_lon = -155.5 + 0.375 # Midway Island # cover_lat = 28.2 # cover_lon = -177.35 # Ni'ihau # cover_lat = 21.9 # cover_lon = -160.17 cover_rad = cover_rads[iter] cover = ps.to_circular_cover(cover_lat, cover_lon, cover_rad, cover_resolution) # ,range_size_limit=2000) elif cover_type == 'bounding_box': # Set cover to "bounding box." cover_lat = np.array([15, 15, 38, 38], dtype=np.float) cover_lon = np.array([-174, -145, -145, -174], dtype=np.float) cover = ps.to_hull_range_from_latlon(cover_lat, cover_lon, cover_resolution) cover_cat = catalog(resolution=cover_resolution, sids=cover) cover_sids_min = np.amin(cover) cover_sids_max = np.amax(cover) # Need to convert to terminator cover_sids_max = gd.spatial_terminator(cover_sids_max) # for k in list(cover_cat.sdict.keys()): # print('cc: ',hex(k)) ########################################################################### # gm_cat_resolution = 5 gm_catalog = catalog(resolution=gm_cat_resolution) k = 0 for i in range(10): while (goes_sids[k] < 0): k = k + 1 # print('adding: ','0x%016x'%goes_sids[k],k) gm_catalog.add('goes', goes_sids[k], goes_data[k]) k = k + 1 for i in range(10): # print('adding: ','0x%016x'%sids[i]) gm_catalog.add('modis', sids[i], data[i]) k = 0 # for i in range(10): idx = np.arange( goes_sids.size)[np.where((goes_sids > cover_sids_min) & (goes_sids < cover_sids_max))] for k in range(len(idx)): # while(goes_sids[k]<0): # k=k+1 if goes_sids[idx[k]] > 0: cover_cat.add_to_entry('goes', goes_sids[idx[k]], goes_data[idx[k]]) # k=k+1 idx = np.arange(sids.size)[np.where((sids > cover_sids_min) & (sids < cover_sids_max))] for k in range(len(idx)): if sids[idx[k]] > 0: cover_cat.add_to_entry('modis', sids[idx[k]], data[idx[k]]) # print(yaml.dump(gm_catalog)) # exit() # ########################################################################### print('plotting iter ', iter) if nrows * ncols == 1: ax = axs[0] else: ax = axs[irow[iter], icol[iter]] if subplot_title[iter] is not None: ax.set_title(subplot_title[iter]) if False: ax.set_global() if True: ax.coastlines(color=coastline_color) if iter == 0: x0 = 0.05 y0 = 0.025 dy = 0.025 plt.figtext(x0, y0 + 0 * dy, "MODIS: " + "sketchG." + modis_base + modis_item + fmt_suffix + ', Water_Vapor_Near_Infrared, resolution = %i' % (sids[10000] & 31), fontsize=10) k = 0 while goes_sids[k] < 0: k = k + 1 plt.figtext(x0, y0 + 1 * dy, "GOES: " + goes_file + ' BAND_3 (6.7mu), resolution = %i' % (goes_sids[k] & 31), fontsize=10) if cover_type == 'circular': plt.figtext( x0, y0 + 2 * dy, "ROI Cover: resolution = %d, radius = %0.2f (upper) %0.3f (lower) degrees, center = 0x%016x" % (cover_resolution, cover_rads[0], cover_rads[3], ps.from_latlon(npf64([cover_lat]), npf64([cover_lon]), cover_resolution)[0]), fontsize=10) # plt.show() # exit() if False: lli = ps.triangulate_indices(cover) ax.triplot(tri.Triangulation(lli[0], lli[1], lli[2]), 'g-', transform=transf, lw=1, markersize=3) if True: if goes_plot_1[iter]: cc_data = cover_cat.get_all_data('goes') csids, sdat = zip(*[cd.as_tuple() for cd in cc_data]) if goes_plot_1_points[iter]: glat, glon = ps.to_latlon(csids) # csids_at_res = list(map(gd.spatial_clear_to_resolution,csids)) # cc_data_accum = dict() # for cs in csids_at_res: # cc_data_accum[cs] = [] # for ics in range(len(csids_at_res)): # cc_data_accum[csids_at_res[ics]].append(sdat[ics]) # for cs in cc_data_accum.keys(): # if len(cc_data_accum[cs]) > 1: # cc_data_accum[cs] = [sum(cc_data_accum[cs])/(1.0*len(cc_data_accum[cs]))] # tmp_values = np.array(list(cc_data_accum.values())) # vmin = np.amin(tmp_values) # vmax = np.amax(np.array(tmp_values)) cc_data_accum, vmin, vmax = gd.simple_collect( csids, sdat, force_resolution=goes_plot_1_res) # print('a100: ',cc_data) # print('cc_data type: ',type(cc_data)) # print('cc_data[0] type: ',type(cc_data[0])) for cs in cc_data_accum.keys(): # print('item: ',hex(cs),cc_data_accum[cs]) lli = ps.triangulate_indices([cs]) triang = tri.Triangulation(lli[0], lli[1], lli[2]) cd_plt = np.array(cc_data_accum[cs]) # print('cd_plt type ',type(cd_plt)) # print('cd_plt shape ',cd_plt.shape) # print('cd_plt type ',type(cd_plt[0])) if goes_line[iter]: ax.triplot(triang, 'r-', transform=transf, lw=1.5, markersize=3, alpha=0.5) # ax.tripcolor(triang,facecolors=cd_plt,vmin=goes_min,vmax=goes_max,cmap='Reds',alpha=0.4) ax.tripcolor(triang, facecolors=cd_plt, edgecolors='k', lw=0, shading='flat', vmin=vmin, vmax=vmax, cmap=goes_plot_1_tripcolor, alpha=goes_plot_1_alpha) # for cd in cc_data: # lli = ps.triangulate_indices([cd.sid]) # triang = tri.Triangulation(lli[0],lli[1],lli[2]) # cd_plt = np.array([cd.datum]) # if goes_line[iter]: # ax.triplot(triang,'r-',transform=transf,lw=3,markersize=3,alpha=0.5) # ax.tripcolor(triang,facecolors=cd_plt,vmin=goes_min,vmax=goes_max,cmap='Reds',alpha=0.4) if modis_plot_1[iter]: cc_data_m = cover_cat.get_all_data('modis') csids, sdat = zip(*[cd.as_tuple() for cd in cc_data_m]) # mlat,mlon = ps.to_latlon(csids) cc_data_m_accum, vmin, vmax = gd.simple_collect( csids, sdat, force_resolution=modis_plot_1_res) for cs in cc_data_m_accum.keys(): lli = ps.triangulate_indices([cs]) triang = tri.Triangulation(lli[0], lli[1], lli[2]) cd_plt = np.array(cc_data_m_accum[cs]) # print('lli[0] len ',len(lli[0])) # print('cd_plt len ', len(cd_plt)) # print('cd_plt type ',type(cd_plt)) # print('cd_plt shape ',cd_plt.shape) # print('cd_plt type ',type(cd_plt[0])) if modis_line[iter]: ax.triplot(triang, 'b-', transform=transf, lw=1.5, markersize=3, alpha=0.5) # ax.tripcolor(triang,facecolors=cd_plt,vmin=goes_min,vmax=goes_max,cmap='Blues',alpha=0.4) ax.tripcolor(triang, facecolors=cd_plt, edgecolors='k', lw=0, shading='flat', vmin=vmin, vmax=vmax, cmap=modis_plot_1_tripcolor, alpha=modis_plot_1_alpha) # for cd in cc_data_m: # lli = ps.triangulate_indices([cd.sid]) # triang = tri.Triangulation(lli[0],lli[1],lli[2]) # cd_plt = np.array([cd.datum]) # if modis_line[iter]: # ax.triplot(triang,'b-',transform=transf,lw=1,markersize=3,alpha=0.5) # ax.tripcolor(triang,facecolors=cd_plt,vmin=modis_min,vmax=modis_max,cmap='Blues',alpha=0.4) if modis_plot_1_points[iter]: mlat, mlon = ps.to_latlon(csids) ax.scatter(mlon, mlat, s=8, c=modis_scatter_color[iter]) # ax.scatter(mlon,mlat,s=8,c='cyan') # ax.scatter(mlon,mlat,s=8,c='darkcyan') # blend_tripcolor_1 = False # blend_tripcolor_res_1 = 6 # blend_tripcolor_1_cmap = None # blend_tripcolor_1_alpha = 1 if blend_tripcolor_1: cc_data = cover_cat.get_all_data('goes') csids, sdat = zip(*[cd.as_tuple() for cd in cc_data]) cc_data_accum, vmin, vmax = gd.simple_collect( csids, sdat, force_resolution=blend_tripcolor_1_res) cc_data_m = cover_cat.get_all_data('modis') csids_m, sdat_m = zip(*[cd.as_tuple() for cd in cc_data_m]) cc_data_m_accum, vmin_m, vmax_m = gd.simple_collect( csids_m, sdat_m, force_resolution=blend_tripcolor_1_res) data_accum_keys = set() for cs in cc_data_accum.keys(): data_accum_keys.add(cs) for cs in cc_data_m_accum.keys(): data_accum_keys.add(cs) for cs in data_accum_keys: # print('item: ',hex(cs),cc_data_accum[cs]) lli = ps.triangulate_indices([cs]) triang = tri.Triangulation(lli[0], lli[1], lli[2]) try: cd_plt_g = (np.array(cc_data_accum[cs]) - vmin) / (vmax - vmin) cd_plt_g = cd_plt_g**blend_tripcolor_1_gamma_g except: cd_plt_g = np.array([0]) try: cd_plt_m = (np.array(cc_data_m_accum[cs]) - vmin_m) / (vmax_m - vmin_m) cd_plt_m = cd_plt_m**blend_tripcolor_1_gamma_m except: cd_plt_m = np.array([0]) ###### # blend 1 & 2 # cd_plt = np.array([[cd_plt_g,0,cd_plt_m]]) ###### # blend 3 # print('len: ',cd_plt_g.shape,cd_plt_m.shape) cd_plt = np.array([[ cd_plt_g[0], 0.5 * (cd_plt_g + cd_plt_m)[0], cd_plt_m[0] ]]) cd_cmp = colors_to_cmap(cd_plt) zs = np.asarray(range(3), dtype=np.float) / 2.0 ax.tripcolor( triang, zs, cmap=cd_cmp # ,facecolors=cd_plt , edgecolors='k', lw=0, shading='gouraud' # ,shading='flat' # ,vmin=vmin,vmax=vmax # ,cmap=blend_tripcolor_1_cmap , alpha=blend_tripcolor_1_alpha) # ,vmin=vmin,vmax=vmax,cmap=blend_tripcolor_1_cmap,alpha=blend_tripcolor_1_alpha) if goes_plot_1[iter]: if goes_plot_1_points[iter]: ax.scatter(glon, glat, s=8, c='black') if nodes_cover[iter] > 0: if nodes_cover[iter] == 1: cc_data_ = cover_cat.get_all_data('goes') else: cc_data_ = cover_cat.get_all_data('modis') sids_, dat_ = zip(*[cd.as_tuple() for cd in cc_data_]) # print('sids_ len: ',len(sids_)) sids_test = gd.spatial_clear_to_resolution( npi64([ gd.spatial_coerce_resolution(s, gm_cat_resolution) for s in sids_ ])) # print('sids_tlen: ',len(sids_test)) if cover_type == 'circular': print('cover: 0x%016x' % ps.from_latlon(npf64([cover_lat]), npf64( [cover_lon]), cover_resolution)[0]) geom_test = sid_geometry(sids_test) for s in geom_test.triangles.keys(): print(iter, ' 0x%016x' % s) triang_test = geom_test.triang() # ax.triplot(triang_test,'g-',transform=transf,lw=1.0,markersize=3,alpha=0.75) ax.triplot(triang_test, 'k-', transform=transf, lw=1.0, markersize=3, alpha=0.5) if False: for i in range(0, 10): k = cover_cat.sdict.peekitem(i)[0] triang = cover_cat.sdict[k].geometry.triang() ax.triplot(triang, 'b-', transform=transf, lw=1, markersize=3, alpha=0.5) if cover_plot[iter]: # lli = ps.triangulate_indices(ps.expand_intervals(cover,9,result_size_limit=2048)) lli = ps.triangulate_indices(cover) ax.triplot(tri.Triangulation(lli[0], lli[1], lli[2]), 'k-', transform=transf, lw=1, markersize=3, alpha=0.5) # ,'g-',transform=transf,lw=1,markersize=3) if False: # k = gm_catalog.sdict.keys()[0] # for k in gm_catalog.sdict.keys(): for i in range(0, 3): k = gm_catalog.sdict.peekitem(i)[0] triang = gm_catalog.sdict[k].geometry.triang() ax.triplot(triang, 'r-', transform=transf, lw=1, markersize=3) if circle_plot[iter]: phi = np.linspace(0, 2 * np.pi, 64) # rad=cover_rad rad = 0.125 ax.plot(cover_lon + rad * np.cos(phi), cover_lat + rad * np.sin(phi), transform=transf, color=circle_color[iter]) # ax.set_facecolor('k') if plt_show_1[iter]: plt.show() ########################################################################### # # if False: # sw_timer.stamp('triangulating') # print('triangulating') # client = Client() # for lli_ in slam(client,ps.triangulate_indices,sids): # sw_timer.stamp('slam iteration') # print('lli_ type: ',type(lli_)) # lli = lli_.result() # sw_timer.stamp('slam result') # print('lli type: ',type(lli)) # triang = tri.Triangulation(lli[0],lli[1],lli[2]) # sw_timer.stamp('slam triang') # plt.triplot(triang,'r-',transform=transf,lw=1.5,markersize=3,alpha=0.5) # sw_timer.stamp('slam triplot') # # sw_timer.stamp('plt show') # # lons,lats,intmat=ps.triangulate_indices(sids) # # triang = tri.Triangulation(lons,lats,intmat) # # plt.triplot(triang,'r-',transform=transf,lw=1.5,markersize=3) # # plt.show() # client.close() print(sw_timer.report_all())
def hex16(i): return "0x%016x" % i start2 = timer() print('join prep done, time12 = ', start2 - start1) join_resolution = m2_resolution join = SortedDict() ktr = 0 print('add goes b5 to join') for k in range(len(g_idx_valid[0])): id = g_idx_valid[0][k] jk = gd.spatial_clear_to_resolution( gd.spatial_coerce_resolution(goes_b5_indices[id], join_resolution)) # print('id,jk: ',id,hex16(jk)) # print('keys: ',[hex16(i) for i in join.keys()]) if jk not in join.keys(): join[jk] = join_value() join[jk].add('goes_b5', id) ktr = ktr + 1 # if ktr > 10: # break # # exit(); start2a = timer() print('add goes b5 to join, time = ', start2a - start2) print('add MERRA-2 to join') for k in range(len(m2_indices)):
def join(self): "Join goes files with a particular m2_file." sw_timer.stamp('join_goes_and_m2-join-start') ########################################################################### ##### MERRA-2 sw_timer.stamp('join_goes_and_m2-start-merra2-start') m2_dLat = 0.5 m2_dLon = 5.0 / 8.0 m2_dLatkm = m2_dLat * gd.re_km / gd.deg_per_rad m2_dLonkm = m2_dLon * gd.re_km / gd.deg_per_rad sw_timer.stamp('join_goes_and_m2-start-merra2-dataset-start') m2_ds = Dataset(self.m2_datapath + self.m2_file_name) sw_timer.stamp('join_goes_and_m2-start-merra2-dataset-end') m2_lat, m2_lon = np.meshgrid(m2_ds['lat'], m2_ds['lon']) m2_lat = m2_lat.flatten() m2_lon = m2_lon.flatten() m2_idx_ij = np.arange(m2_lat.size, dtype=np.int64) m2_resolution = int(gd.resolution(m2_dLonkm * 2)) m2_indices = ps.from_latlon(m2_lat, m2_lon, m2_resolution) m2_term = gd.spatial_terminator(m2_indices) m2_tid = gd.merra2_stare_time(m2_ds) sw_timer.stamp('join_goes_and_m2-start-merra2-end') ########################################################################### ##### GOES sw_timer.stamp('join_goes_and_m2-start-goes-start') self.igoes = 0 self.goes_band = self.goes_filenames_valid[self.igoes].split('.')[4] self.goes_bandname = self.goes_bandnames[self.goes_band] sw_timer.stamp('join_goes_and_m2-start-goes-start-dataset-start') self.goes_ds = Dataset(self.goes_datapath + self.goes_filenames_valid[self.igoes]) sw_timer.stamp('join_goes_and_m2-start-goes-start-dataset-end') goes_tid = gd.goes10_img_stare_time(self.goes_ds) ##### MERRA-2 at the GOES time fine_match = ps.cmp_temporal(np.array(goes_tid, dtype=np.int64), m2_tid) # print('fine_match: ',fine_match) # print('gtid: ',[hex16(i) for i in goes_tid]) # print('m2_tid: ',[hex16(i) for i in m2_tid]) m2_ifm = np.nonzero(fine_match)[0] # print('m2_ifm: ',m2_ifm) if m2_ifm.size == 1: m2_dataDayI = m2_ds['TQI'][m2_ifm, :, :] m2_dataDayL = m2_ds['TQL'][m2_ifm, :, :] m2_dataDayV = m2_ds['TQV'][m2_ifm, :, :] else: m2_dataDayI = np.mean(m2_ds['TQI'][m2_ifm, :, :], 0) m2_dataDayL = np.mean(m2_ds['TQL'][m2_ifm, :, :], 0) m2_dataDayV = np.mean(m2_ds['TQV'][m2_ifm, :, :], 0) m2_dataDay = m2_dataDayI + m2_dataDayL + m2_dataDayV # print('m2_dataDay.shape: ',m2_dataDay.shape) m2_data = m2_dataDay[:, :].T # print('m2_data.shape: ',m2_data.shape) m2_data_flat = m2_data.flatten() # print('m2_data.flat.sh: ',m2_data_flat.shape) g_lat = self.goes_ds['lat'][:, :].flatten() self.g_lat_size = g_lat.size # print('gds lat shape: ',self.goes_ds['lat'][:,:].shape) # print('gds lat fltsh: ',g_lat.shape) g_lon = self.goes_ds['lon'][:, :].flatten() g_idx_valid = np.where((g_lat >= -90.0) & (g_lat <= 90.0)) g_idx_invalid = np.where(((g_lat < -90.0) | (g_lat > 90.0))) self.goes_indices = np.full(g_lat.shape, -1, dtype=np.int64) sw_timer.stamp('join_goes_and_m2-start-goes-from_latlon-start') self.goes_indices[g_idx_valid] = ps.from_latlon( g_lat[g_idx_valid], g_lon[g_idx_valid], int(gd.resolution(self.goes_ds['elemRes'][0]))) self.goes_indices = self.goes_indices sw_timer.stamp('join_goes_and_m2-start-goes-from_latlon-end') ########################################################################### ##### Allocate MERRA-2 arrays co-aligned with GOES self.m2_src_coord_h5 = np.full(g_lat.shape, -1, dtype=np.int64) self.m2_tpw_h5 = np.full(g_lat.shape, -1, dtype=np.int64) sw_timer.stamp('join_goes_and_m2-start-join') ##### join_resolution = m2_resolution join = SortedDict() ktr = 0 for k in range(len(g_idx_valid[0])): id = g_idx_valid[0][k] jk = gd.spatial_clear_to_resolution( gd.spatial_coerce_resolution(self.goes_indices[id], join_resolution)) if jk not in join.keys(): join[jk] = join_value() join[jk].add(self.goes_bandname, id) ktr = ktr + 1 # if ktr > 10: # break # # exit(); for k in range(len(m2_indices)): jk = gd.spatial_clear_to_resolution(m2_indices[k]) if jk not in join.keys(): join[jk] = join_value() join[jk].add('m2', k) ########################################################################### self.tpw_scale = 0.001 self.tpw_offset = 0 ########################################################################### ##### JOIN # TODO Add metadata for traceability. jkeys = join.keys() ktr = 0 nktr = len(jkeys) # gd_idx_valid is a tuple with one element ktr_max = nktr elements_pushed = 0 print('Push joined m2 data into the dataset n = ', nktr) for k in range(nktr): ktr = ktr + 1 if self.verbose_progress: if int(100.0 * ktr / ktr_max) % 5 == 0 or int( 100 * ktr / ktr_max) < 2: print( 'join_goes_merra2: %2d%% complete, %d elements pushed.' % (int(100 * ktr / ktr_max), elements_pushed), end='\r', flush=True) sid = jkeys[k] if join[sid].contains(self.goes_bandname): if join[sid].contains('m2'): m2s = join[sid].get('m2')[0] # Grab the first one self.m2_src_coord_h5[join[sid].get( self.goes_bandname)] = m2s # self.m2_tpw_h5[join[sid].get(self.goes_bandname)] = (m2_data_flat[m2s]-self.tpw_offset)/self.tpw_scale avg = (np.mean(m2_data_flat[join[sid].get('m2')]) - self.tpw_offset) / self.tpw_scale self.m2_tpw_h5[join[sid].get(self.goes_bandname)] = avg elements_pushed = elements_pushed + len(join[sid].get( self.goes_bandname)) print('join_goes_merra2: done, %d elements pushed. ' % (elements_pushed), flush=True) print('') sw_timer.stamp('join_goes_and_m2-join-end') sw_timer.stamp('join_goes_and_m2-start-goes-end') return self