def make_composite_stats(k0_file, scat_file, impulse_file, res=40, \ t_window=20, dt_fine=0.05): k0_field_dict={'location':['latitude','longitude', 'elevation'], 'both':['K0']} catalog=setup(scat_file, impulse_file) D_in=pc.data().from_h5(k0_file, field_dict=k0_field_dict) D_in.get_xy(EPSG=3031) # remove bogus points (lat=lon=0) D_in.index(np.abs(D_in.x)>0) pt_dict=pc.bin_rows(np.c_[np.round(D_in.x/res)*res, np.round(D_in.y/res)*res]) D_out=pc.data().from_dict( {field:np.zeros(len(pt_dict))+np.NaN for field in \ ['t_ctr', 't_med','t_sigma', 't_sigma_r','N','x','y', 'z_sigma', 'z_slope_mag']}) TX=resample_wf(catalog[0], dt_fine) TX_mean, TX_med = composite_stats(TX, t_window)[0:2] for ii, ctr in enumerate(pt_dict): N=len(pt_dict[ctr]) if N < 10: continue WF0 = make_composite_wf(D_in.K0[pt_dict[ctr]], catalog)[0] WF0 = resample_wf(WF0, 0.05) D_out.t_ctr[ii], D_out.t_med[ii], D_out.t_sigma[ii], D_out.t_sigma_r[ii]=composite_stats(WF0, t_window) D_out.z_sigma[ii], D_out.z_slope_mag[ii] = plane_fit_R(D_in, pt_dict[ctr]) D_out.x[ii] = ctr[0] D_out.y[ii] = ctr[1] D_out.N[ii] = len(pt_dict[ctr]) return D_out, TX_mean, TX_med
def get_xovers(self,invalid_to_nan=True): rgt=self.attrs['ReferenceGroundTrack'] xo={'ref':{},'crossing':{},'both':{}} n_cycles=self.ROOT.h_corr.shape[1] zz=np.zeros(n_cycles) for field in ['delta_time','h_corr','h_corr_sigma','h_corr_sigma_systematic', 'ref_pt','rgt','atl06_quality_summary','latitude','longitude','cycle_number','x_atc','y_atc','fit_quality']: xo['ref'][field]=[] xo['crossing'][field]=[] #if field in ['delta_time','h_corr','h_corr_sigma','h_corr_sigma_systematic']: # xo['ref'][field].append([]) # xo['ref'][field].append([]) xo['crossing']['along_track_rss']=[] if hasattr(self,'x'): for field in ['x','y']: xo['ref'][field]=[] for i1, ref_pt in enumerate(self.crossing_track_data.ref_pt): i0=np.flatnonzero(self.ROOT.ref_pt==ref_pt)[0] # fill vectors for field in ['latitude','longitude']: xo['ref'][field] += [getattr(self.ROOT, field)[i0]+zz] for field in ['x_atc','y_atc']: xo['ref'][field] += [getattr(self.ref_surf, field)[i0]+zz] xo['ref']['ref_pt'] += [self.ROOT.ref_pt[i0]+zz] xo['ref']['rgt'] += [rgt+zz] xo['ref']['fit_quality'] += [self.ref_surf.fit_quality[i0]+zz] for field in ['delta_time','h_corr','h_corr_sigma','h_corr_sigma_systematic', 'ref_pt','rgt','atl06_quality_summary', 'cycle_number', 'along_track_rss' ]: xo['crossing'][field] += [getattr(self.crossing_track_data, field)[i1]+zz] # fill vectors for each cycle for field in ['delta_time', 'h_corr','h_corr_sigma','h_corr_sigma_systematic']: # vars that are N_pts x N_cycles xo['ref'][field] += [getattr(self.ROOT, field)[i0,:]] xo['ref']['atl06_quality_summary'] += [self.cycle_stats.atl06_summary_zero_count[i0,:] > 0] xo['ref']['cycle_number'] += [getattr(self.ROOT,'cycle_number')] if hasattr(self, 'x'): for field in ['x','y']: xo['ref'][field] += [getattr(self, field)[i0]+zz] xo['crossing']['latitude']=xo['ref']['latitude'] xo['crossing']['longitude']=xo['ref']['longitude'] xo['crossing']['x_atc']=xo['ref']['x_atc'] xo['crossing']['y_atc']=xo['ref']['y_atc'] xo['crossing']['fit_quality'] = xo['ref']['fit_quality'] for field in xo['crossing']: xo['crossing'][field]=np.concatenate(xo['crossing'][field], axis=0) for field in xo['ref']: xo['ref'][field]=np.concatenate(xo['ref'][field], axis=0) ref=pc.data().from_dict(xo['ref']) crossing=pc.data().from_dict(xo['crossing']) delta={} delta['h_corr']=crossing.h_corr-ref.h_corr delta['delta_time']=crossing.delta_time-ref.delta_time delta['h_corr_sigma']=np.sqrt(crossing.h_corr_sigma**2+ref.h_corr_sigma**2) delta['latitude']=ref.latitude.copy() delta['longitude']=ref.longitude.copy() delta=pc.data().from_dict(delta) return ref, crossing, delta
def save_L_interp(L_interps, file): D_list = [] for L_interp in L_interps: pt0 = np.array(list(L_interp.keys())) D = {} for field in L_interp[pt0[0]]: D[field] = np.array([L_interp[pt][field] for pt in pt0]) D_list += [pc.data().from_dict(D)] pc.data().from_list(D_list).to_h5(file)
def as_points(self, keep_all=False): """ Return a pointCollection.data object containing the points in the grid """ x,y=np.meshgrid(self.x, self.y) if keep_all: result = pc.data(filename=self.filename).\ from_dict({'x':x.ravel(),'y':y.ravel(),'z':self.z.ravel()}) else: good=np.isfinite(self.z).ravel() result = pc.data(filename=self.filename).\ from_dict({'x':x.ravel()[good],'y':y.ravel()[good],'z':self.z.ravel()[good]}) if self.time is not None: result.assign({'time':self.time+np.zeros_like(self.z)}) return result
def read_CS2_data(xy0, W, index_files, apply_filters=True, DEM_file=None, dem_tol=50): if DEM_file is not None: DEM=pc.grid.data().from_geotif(DEM_file, bounds=[xy0[0]+np.array([-W/2, W/2]), \ xy0[1]+np.array([-W/2, W/2])]) else: DEM = None D = [] D = [read_poca_data( xy0, W, file, apply_filters=apply_filters, DEM=DEM) \ for file in index_files['POCA']] D += [ read_swath_data(xy0, W, file, apply_filters=apply_filters)\ for file in index_files['swath']] D = pc.data().from_list(D) if DEM_file is not None: D.DEM = DEM.interp(D.x, D.y) D.index(np.abs(D.h - D.DEM) < dem_tol) if apply_filters: with open(os.path.join(Path(__file__).parent.absolute(), \ 'CS2_known_bad_orbits.json'), 'r') as fh: bad_orbits = json.load(fh)[1] #['bad orbits'] D.index(~np.in1d(D.abs_orbit.astype(int), np.array(bad_orbits, dtype='int'))) return D
def read_poca_data(xy0, W, index_file, apply_filters=True, DEM=None): fields=['x','y','time','h', 'power','coherence','AD','error_composite', \ 'ambiguity','abs_orbit','phase','range_surf'] D=pc.geoIndex().from_file(index_file).query_xy_box(xy0[0]+np.array([-W/2, W/2]), \ xy0[1]+np.array([-W/2, W/2]), fields=fields) D = pc.data().from_list(D) if D is None: return D D.time = matlab_to_year(D.time) #D.assign({'burst':D.pulse_num, D.assign({'swath': np.zeros_like(D.x, dtype=bool)}) if DEM is not None: gx, gy = np.gradient(DEM.z, DEM.x, DEM.y) temp = DEM.copy() temp.z = np.abs(gx + 1j * gy) DEM_slope_mag = temp.interp(D.x, D.y) D.assign({ 'sigma': np.maximum( 0.5, 50 * DEM_slope_mag + np.maximum(0, -0.64 * (np.log10(D.power) + 14))) }) else: # if no DEM is specified, use a typical value of 0.01 for the slope D.assign({ 'sigma': 50 * 0.01 + np.maximum(0, -0.64 * (np.log10(D.power) + 14)) }) if apply_filters: D.index((D.power > 1e-16) & (D.error_composite == 0) & (D.power < 1e-12)) return D
def read_swath_data(xy0, W, index_file, apply_filters=True, baseline_num=None): fields = ['x','y','time','h', 'power','coherence',\ 'R_POCA', 'ambiguity','abs_orbit', 'block_h_spread',\ 'count','phase','R_offnadir','range_surf','seg_ind'] D=pc.geoIndex().from_file(index_file).query_xy_box(xy0[0]+np.array([-W/2, W/2]), \ xy0[1]+np.array([-W/2, W/2]), fields=fields) if D is not None: D = pc.data().from_list(D) if D is None: return D if baseline_num is not None: D.assign({'baseline': np.zeros_like(D.x) + baseline_num}) if baseline_num == 2: D.power /= 2. D.index(D.power > 0.) D.time = matlab_to_year(D.time) #D.assign({'burst':D.pulse_num, D.assign({'swath': np.ones_like(D.x, dtype=bool)}) D.assign({'sigma':np.minimum(5, np.maximum(1, 0.95 -.4816*(np.log10(D.power)+14.3) +\ 1.12*np.sqrt(D.block_h_spread)))}) if apply_filters: if np.any(D.count > 1): D.index( (D.power > 1e-17) & (D.power < 1e-13) & \ (D.count > 3) & (D.block_h_spread < 15)) else: D.index((D.power > 1e-17) & (D.power < 1e-13)) return D
def __init__(self, D, map_file, index_file, datatype): self.files=[] self.datatype=datatype self.map_file=map_file self.index_file=index_file print(f"TrackPicker: index_file is {index_file}") srs_proj4=pc.geoIndex().from_file(index_file).attrs['SRS_proj4'] if datatype == 'ATL06': for ii, Di in enumerate(D): Di.get_xy(srs_proj4) Di.assign({'file_num':np.zeros_like(Di.x)+ii}) self.files += [Di.filename] self.D_all=pc.data().from_list(D) elif datatype == 'ATL11': D_list=[] for ii, Di in enumerate(D): self.files += [Di.filename] Di.get_xy(srs_proj4) D_list.append(pc.data().from_dict({ 'x':Di.x, 'y':Di.y, 'file_num':np.zeros_like(Di.x)+ii, 'h_corr':Di.corrected_h.h_corr[:,-1].ravel()})) self.D_all=pc.data().from_list(D_list) else: self.D_all=pc.data().from_list(D) self.D=D XR=[np.nanmin(self.D_all.x), np.nanmax(self.D_all.x)] YR=[np.nanmin(self.D_all.y), np.nanmax(self.D_all.y)] self.fig=plt.figure() if map_file is not None: get_map(map_file, bounds=[XR, YR]) #for Di in D: # plt.plot(Di.x, Di.y,'.') if self.datatype=='ATL06': plt.scatter(self.D_all.x, self.D_all.y, 6, c=self.D_all.r_eff, linewidth=0, vmax=1, vmin=0) elif self.datatype=='ATL11': plt.scatter(self.D_all.x, self.D_all.y, 6, c=self.D_all.h_corr); plt.colorbar() elif self.datatype=='ATM_waveform_fit': plt.scatter(self.D_all.x, self.D_all.y, 6, c=np.log10(self.D_all.K0)) elif self.datatype=='CS2': plt.scatter(self.D_all.x, self.D_all.y, 6, c=self.D_all.h); plt.colorbar() hax=plt.axes([0.7, 0.05, 0.25, 0.25]) hax.hist((self.D_all.time-730486)/365.25+2000, 100) self.cid=self.fig.canvas.mpl_connect('button_press_event', self) plt.show()
def main(): # test code thefile='Volumes/ice2/ben/ATL14_test/Jako_d2zdt2=5000_d3z=0.00001_d2zdt2=1500_RACMO//centers/E480_N-1200.h5' D=pc.data().from_h5(thefile, group='/data/') D.index(D.sensor>=4) xy0=np.array([ 480000., -1200000.]) best_sensors=subset_DEM_stack(D, xy0, 4.e4)
def calc_cell_area(grid): xg, yg = np.meshgrid(grid.ctrs[1], grid.ctrs[0]) if grid.srs_proj4 is not None: lat = pc.data().from_dict({ 'x': xg, 'y': yg }).get_latlon(proj4_string=grid.srs_proj4).latitude return pc.ps_scale_for_lat(lat)**2 * grid.delta[0] * grid.delta[1] else: return np.ones(grid.shape[0:2]) * grid.delta[0] * grid.delta[1]
def main(): import glob #gI_file='/Volumes/insar10/ben/IS2_tiles/GL/GeoIndex.h5' gI_file=glob.glob('/Volumes/ice2/ben/scf/GL_06/latest/tiles/*/GeoIndex.h5')[0] xy0=[-170000.0, -2280000.0] dI=pc.data().from_list(read_ICESat2(xy0, {'x':4.e4, 'y':4.e4}, gI_file, cplx_accept_threshold=0.25)) import matplotlib.pyplot as plt plt.scatter(dI.x, dI.y, c=dI.z, linewidth=0); plt.colorbar() plt.axis('equal')
def main(): import matplotlib.pyplot as plt import pointCollection as pc t=np.arange(2002, 2022, 0.1) xy_jako=(-170000, -2280000) corrections = ['RACMO_fac', 'RACMO_fac_smb', 'RACMO'] for correction in corrections: data=pc.data().from_dict({'time':t, 'x':np.ones_like(t)*xy_jako[0], 'y':np.ones_like(t)*xy_jako[1], 'z':np.zeros_like(t)}) assign_firn_correction(data, correction, 1) plt.plot(data.time, data.z) plt.legend(corrections)
def apply_bin_fn(D_pt, res, fn=None, field='z'): pt_dict = bin_rows(np.c_[np.round(D_pt.x / res) * res, np.round(D_pt.y / res) * res]) keys = list(pt_dict.keys()) z = np.array([fn(D_pt, pt_dict[key]) for key in keys]) N = np.array([len(pt_dict[key]) for key in keys]) keys = np.array(keys) return pc.data(filename=D_pt.filename)\ .from_dict({'x':keys[:,0], 'y':keys[:,1], field:z,'count':N})
def main(): W = {'x': 1.e4, 'y': 200, 't': 2} x = np.arange(-W['x'] / 2, W['x'] / 2, 100) D=pc.data().from_dict({'x':x, 'y':np.zeros_like(x),'z':np.sin(2*np.pi*x/2000.),\ 'time':np.zeros_like(x)-0.5, 'sigma':np.zeros_like(x)+0.1}) D1 = D D1.t = np.ones_like(x) data = pc.data().from_list( [D, D.copy().assign({'time': np.zeros_like(x) + 0.5})]) E_d3zdx2dt = 0.0001 E_d2z0dx2 = 0.006 E_d2zdt2 = 5 E_RMS = { 'd2z0_dx2': E_d2z0dx2, 'dz0_dx': E_d2z0dx2 * 1000, 'd3z_dx2dt': E_d3zdx2dt, 'd2z_dxdt': E_d3zdx2dt * 1000, 'd2z_dt2': E_d2zdt2 } ctr = {'x': 0., 'y': 0., 't': 0.} SRS_proj4 = '+proj=stere +lat_0=90 +lat_ts=70 +lon_0=-45 +k=1 +x_0=0 +y_0=0 +datum=WGS84 +units=m +no_defs ' spacing = {'z0': 50, 'dz': 50, 'dt': 0.25} S = smooth_xytb_fit(data=data, ctr=ctr, W=W, spacing=spacing, E_RMS=E_RMS, reference_epoch=2, N_subset=None, compute_E=False, max_iterations=2, srs_proj4=SRS_proj4, VERBOSE=True, dzdt_lags=[1]) return S
def write(self, out_dir, fields=None, append=True, ind_fields=['x', 'y', 'time']): if self.D is None: return out_file = out_dir + '/E%d_N%d.h5' % (self.xy0[0], self.xy0[1]) n_source_files = 0 if fields is None: field_dict = self.__default_field_dict__() fields = [] for key in field_dict.keys(): fields += field_dict[key] if os.path.isfile(out_file): with h5py.File(out_file, 'r') as h5f: if 'source_files' in h5f: grp = h5f['source_files'] n_source_files = len(h5f['source_files'].attrs.keys()) file_dict = {} for file_num, Di in enumerate(self.D): if not hasattr(Di, 'x'): Di.get_xy(self.SRS_proj4) Di.assign({ 'source_file_num': np.zeros_like(Di.x, dtype=int) + file_num + n_source_files }) Di.ravel_fields() Di.index(np.isfinite(getattr(Di, self.z_field))) file_dict[file_num] = Di.filename out_fields = list(set(fields + ['x', 'y', 'source_file_num'])) D_all = pc.data(fields=out_fields).from_list(self.D) if D_all.size == 0: return pc.indexedH5.data(filename=out_file, bin_W=self.bin_W).to_file(D_all, \ out_file, time_field=self.time_field, append=append,\ ind_fields=ind_fields) with h5py.File(out_file, 'r+') as h5f: if 'source_files' in h5f: grp = h5f['source_files'] else: grp = h5f.create_group("source_files") for key in file_dict: grp.attrs['file_%d' % key] = file_dict[key]
def from_h5(self, filename, pair=None): if pair is None: pair = self.pair else: self.pair_name = f'pt{int(self.pair)}' D_at = pc.ATL11.data().from_h5(filename, pair=pair) D_xo = pc.data().from_h5(filename, group=self.pair_name + '/crossing_track_data', field_dict=self.__default_XO_field_dict__()) D_xo.index(np.isfinite(D_xo.ref_pt) & np.isfinite(D_xo.cycle_number)) with h5py.File(filename, 'r') as h5f: rgt = int(h5f[self.pair_name].attrs['ReferenceGroundTrack']) D_at.assign({'rgt': np.zeros_like(D_at.delta_time) + rgt}) n_cycles = D_at.cycle_number[-1, -1] u_pt_xo = np.unique(D_xo.ref_pt) D_at.index(np.in1d(D_at.ref_pt[:, 0], u_pt_xo)) theshape = (u_pt_xo.size, n_cycles, 2) self.assign( {field: np.zeros(theshape) + np.NaN for field in D_xo.fields}) row = np.searchsorted(u_pt_xo, D_at.ref_pt.ravel()) col = D_at.cycle_number.ravel().astype(int) - 1 for field in self.fields: if field not in D_at.fields: continue getattr(self, field).flat[np.ravel_multi_index( (row, col, np.zeros_like(row, dtype=int)), theshape)] = getattr(D_at, field).ravel() row = np.searchsorted(u_pt_xo, D_xo.ref_pt) col = D_xo.cycle_number.astype(int) - 1 for field in self.fields: getattr(self, field).flat[np.ravel_multi_index( (row, col, 1 + np.zeros_like(row, dtype=int)), theshape)] = getattr(D_xo, field) self.__update_size_and_shape__() return self
def make_plot(k0_file, scat_file, impulse_file, xy0=None, res=40, t_window=20): #res_test(impulse_file) k0_field_dict={'location':['latitude','longitude'], 'both':['K0']} catalog=setup(scat_file, impulse_file) D_in=pc.data().from_h5(k0_file, field_dict=k0_field_dict) D_in.get_xy(EPSG=3031) pt_dict=pc.bin_rows(np.c_[np.round(D_in.x/res)*res, np.round(D_in.y/res)*res]) xy_fits=np.concatenate([np.array(key).reshape([1,2]) for key in pt_dict], axis=0) best=np.argmin(np.abs((xy_fits[:,0]-xy0[0])+1j*(xy_fits[:,1]-xy0[1]))) WF_composite, WFs=make_composite_wf(D_in.K0[pt_dict[tuple(xy_fits[best,:])]], catalog) TX=resample_wf(catalog[0], 0.05) WF_composite=resample_wf(WF_composite, 0.05) bar0, med0, sigma0, sigmar_0=composite_stats(TX, t_window) barc, medc, sigmac, sigmar_c=composite_stats(WF_composite, t_window) TX_corr_mean_c, TX_corr_med_c=TX_corr(TX, np.max([6*sigmar_c, t_window]), sigmar_c, TX_sigma=sigma0) TX_corr_mean_0, TX_corr_med_0=TX_corr(TX, np.max([6*sigmar_0, t_window]), sigmar_0, TX_sigma=sigma0) plt.figure() z0 = TX.centroid()*-0.15 for WF in WFs: #WFi = resample_wf(WF, 0.01) plt.plot(WF.p, WF.t*-.15-z0, linewidth=0.5, color='gray') plt.plot(TX.p, TX.t*-.15-z0, 'k', linewidth=2) plt.plot(WF_composite.p.ravel(), WF_composite.t.ravel()*-.15-z0, 'b', linewidth=2) plt.gca().axhline(-bar0*.15-z0, color='k', linewidth=2, label='TX mean') plt.gca().axhline(-barc*.15-z0, color='b', linewidth=2, label='RX mean') plt.gca().axhline(-med0*.15-z0, linestyle='--', color='k', linewidth=2, label='TX med') plt.gca().axhline(-medc*.15-z0, linestyle='--', color='b', linewidth=2, label='RX med') plt.legend() plt.xlabel('power') plt.ylabel('elevation WRT surface, m') plt.show()
def make_test_data(): ''' make test data for CS2 inversions inputs: None (reads test_CS2_data.h5) outputs: z: simulated elevation data bounds: a dict containing the range of the x, y, and time data z_func: a function that gives heights as a function of x, y, time and (binary) swath ''' data = pc.data().from_h5('CS2_test_data.h5', group='/') data.time = matlab_to_year(data.time) XR = np.array( [np.floor(data.x.min() / 1.e4), np.ceil(data.x.max()) / 1.e4]) * 1.e4 YR = np.array( [np.floor(data.y.min() / 1.e4), np.ceil(data.y.max()) / 1.e4]) * 1.e4 TR = np.array([np.floor(data.time.min()), np.ceil(data.time.max())]) def z_func(x, y, t, swath, XR, YR, TR): if swath is None: swath = np.zeros_like(x, dtype=bool) dem = (x - XR[0]) * 0.05 + 20 * np.sin(2 * np.pi * (y - YR[0]) / (np.diff(YR) / 10)) t0 = TR[0] + np.array([0, 1 / 3, 2 / 3, 1]) * np.diff(TR) t_func = 3 * np.interp(t, t0, np.array([0, 0, 1, 1])) delta_z = 2 * t_func * np.sin(2 * np.pi * (x - XR[0]) / (np.diff(XR) / 4)) swath_bias = 1.5 + np.exp(-((x - np.mean(XR))**2 + (y - np.mean(YR))**2) / 2 / (np.diff(XR) / 3)**2) return dem + delta_z + swath_bias * swath return data, z_func(data.x, data.y, data.time, data.swath, XR, YR, TR), [XR, YR, TR], z_func
def append_SMB_ATL11(input_file, base_dir, REGION, MODEL): # read input file field_dict = {None: ('delta_time', 'h_corr', 'x', 'y')} D11 = pc.data().from_h5(input_file, field_dict=field_dict) # check if running crossover or along-track ATL11 if (D11.h_corr.ndim == 3): nseg, ncycle, ncross = D11.shape else: nseg, ncycle = D11.shape # get projection of input coordinates EPSG = set_projection(REGION) # available models models = dict(AA={}, GL={}) # MAR models['GL']['MAR'] = [] # models['GL']['MAR'].append('MARv3.9-ERA') # models['GL']['MAR'].append('MARv3.10-ERA') # models['GL']['MAR'].append('MARv3.11-NCEP') models['GL']['MAR'].append('MARv3.11-ERA') models['GL']['MAR'].append('MARv3.11.2-ERA-6km') models['GL']['MAR'].append('MARv3.11.2-ERA-7.5km') models['GL']['MAR'].append('MARv3.11.2-ERA-10km') models['GL']['MAR'].append('MARv3.11.2-ERA-15km') models['GL']['MAR'].append('MARv3.11.2-ERA-20km') models['GL']['MAR'].append('MARv3.11.2-NCEP-20km') # RACMO models['GL']['RACMO'] = [] # models['GL']['RACMO'].append('RACMO2.3-XGRN11') # models['GL']['RACMO'].append('RACMO2.3p2-XGRN11') models['GL']['RACMO'].append('RACMO2.3p2-FGRN055') # MERRA2-hybrid models['GL']['MERRA2-hybrid'] = [] # models['GL']['MERRA2-hybrid'].append('GSFC-fdm-v0') # models['GL']['MERRA2-hybrid'].append('GSFC-fdm-v1') models['GL']['MERRA2-hybrid'].append('GSFC-fdm-v1.1') models['AA']['MERRA2-hybrid'] = [] # models['AA']['MERRA2-hybrid'].append('GSFC-fdm-v0') models['AA']['MERRA2-hybrid'].append('GSFC-fdm-v1') for model_version in models[REGION][MODEL]: if (MODEL == 'MAR'): match_object = re.match(r'(MARv\d+\.\d+(.\d+)?)', model_version) MAR_VERSION = match_object.group(0) MAR_REGION = dict(GL='Greenland', AA='Antarctic')[REGION] # model subdirectories SUBDIRECTORY = dict(AA={}, GL={}) SUBDIRECTORY['GL']['MARv3.9-ERA'] = [ 'ERA_1958-2018_10km', 'daily_10km' ] SUBDIRECTORY['GL']['MARv3.10-ERA'] = [ 'ERA_1958-2019-15km', 'daily_15km' ] SUBDIRECTORY['GL']['MARv3.11-NCEP'] = [ 'NCEP1_1948-2020_20km', 'daily_20km' ] SUBDIRECTORY['GL']['MARv3.11-ERA'] = [ 'ERA_1958-2019-15km', 'daily_15km' ] SUBDIRECTORY['GL']['MARv3.11.2-ERA-6km'] = ['6km_ERA5'] SUBDIRECTORY['GL']['MARv3.11.2-ERA-7.5km'] = ['7.5km_ERA5'] SUBDIRECTORY['GL']['MARv3.11.2-ERA-10km'] = ['10km_ERA5'] SUBDIRECTORY['GL']['MARv3.11.2-ERA-15km'] = ['15km_ERA5'] SUBDIRECTORY['GL']['MARv3.11.2-ERA-20km'] = ['20km_ERA5'] SUBDIRECTORY['GL']['MARv3.11.2-NCEP-20km'] = ['20km_NCEP1'] MAR_MODEL = SUBDIRECTORY[REGION][model_version] DIRECTORY = os.path.join(base_dir, 'MAR', MAR_VERSION, MAR_REGION, *MAR_MODEL) # variable coordinates KWARGS = dict(AA={}, GL={}) KWARGS['GL']['MARv3.9-ERA'] = dict(XNAME='X10_153', YNAME='Y21_288') KWARGS['GL']['MARv3.10-ERA'] = dict(XNAME='X10_105', YNAME='Y21_199') KWARGS['GL']['MARv3.11-NCEP'] = dict(XNAME='X12_84', YNAME='Y21_155') KWARGS['GL']['MARv3.11-ERA'] = dict(XNAME='X10_105', YNAME='Y21_199') KWARGS['GL']['MARv3.11.2-ERA-6km'] = dict(XNAME='X12_251', YNAME='Y20_465') KWARGS['GL']['MARv3.11.2-ERA-7.5km'] = dict(XNAME='X12_203', YNAME='Y20_377') KWARGS['GL']['MARv3.11.2-ERA-10km'] = dict(XNAME='X10_153', YNAME='Y21_288') KWARGS['GL']['MARv3.11.2-ERA-15km'] = dict(XNAME='X10_105', YNAME='Y21_199') KWARGS['GL']['MARv3.11.2-ERA-20km'] = dict(XNAME='X12_84', YNAME='Y21_155') KWARGS['GL']['MARv3.11.2-NCEP-20km'] = dict(XNAME='X12_84', YNAME='Y21_155') MAR_KWARGS = KWARGS[REGION][model_version] # output variable keys for both direct and derived fields KEYS = ['zsurf', 'zfirn', 'zmelt', 'zsmb', 'zaccum'] # HDF5 longname attributes for each variable LONGNAME = {} LONGNAME['zsurf'] = "Snow Height Change" LONGNAME['zfirn'] = "Snow Height Change due to Compaction" LONGNAME['zmelt'] = "Snow Height Change due to Surface Melt" LONGNAME['zsmb'] = "Snow Height Change due to Surface Mass Balance" LONGNAME[ 'zaccum'] = "Snow Height Change due to Surface Accumulation" elif (MODEL == 'RACMO'): RACMO_VERSION, RACMO_MODEL = model_version.split('-') # output variable keys KEYS = ['zsurf'] # HDF5 longname attributes for each variable LONGNAME = {} LONGNAME['zsurf'] = "Snow Height Change" elif (MODEL == 'MERRA2-hybrid'): merra2_regex = re.compile(r'GSFC-fdm-((v\d+)(\.\d+)?)$') # get MERRA-2 version and major version MERRA2_VERSION = merra2_regex.match(model_version).group(1) MERRA2_MAJOR_VERSION = merra2_regex.match(model_version).group(2) # MERRA-2 hybrid directory DIRECTORY = os.path.join(base_dir, 'MERRA2_hybrid', MERRA2_VERSION) MERRA2_REGION = dict(AA='ais', GL='gris')[REGION] # output variable keys for both direct and derived fields KEYS = ['zsurf', 'zfirn', 'zsmb'] # HDF5 longname attributes for each variable LONGNAME = {} LONGNAME['zsurf'] = "Snow Height Change" LONGNAME['zfirn'] = "Snow Height Change due to Compaction" LONGNAME['zsmb'] = "Snow Height Change due to Surface Mass Balance" # check if running crossover or along track if (D11.h_corr.ndim == 3): # allocate for output height for crossover data OUTPUT = {} for key in KEYS: OUTPUT[key] = np.ma.zeros((nseg, ncycle, ncross), fill_value=np.nan) OUTPUT[key].mask = np.ones((nseg, ncycle, ncross), dtype=np.bool) OUTPUT[key].interpolation = np.zeros((nseg, ncycle, ncross), dtype=np.uint8) # for each cycle of ICESat-2 ATL11 data for c in range(ncycle): # check that there are valid crossovers cross = [ xo for xo in range(ncross) if np.any(np.isfinite(D11.delta_time[:, c, xo])) ] # for each valid crossing for xo in cross: # find valid crossovers i, = np.nonzero(np.isfinite(D11.delta_time[:, c, xo])) # convert from delta time to decimal-years tdec = convert_delta_time(D11.delta_time[i, c, xo])['decimal'] if (MODEL == 'MAR'): # read and interpolate daily MAR outputs ZN4 = SMBcorr.interpolate_mar_daily(DIRECTORY, EPSG, MAR_VERSION, tdec, D11.x[i, c, xo], D11.y[i, c, xo], VARIABLE='ZN4', SIGMA=1.5, FILL_VALUE=np.nan, **MAR_KWARGS) ZN5 = SMBcorr.interpolate_mar_daily(DIRECTORY, EPSG, MAR_VERSION, tdec, D11.x[i, c, xo], D11.y[i, c, xo], VARIABLE='ZN5', SIGMA=1.5, FILL_VALUE=np.nan, **MAR_KWARGS) ZN6 = SMBcorr.interpolate_mar_daily(DIRECTORY, EPSG, MAR_VERSION, tdec, D11.x[i, c, xo], D11.y[i, c, xo], VARIABLE='ZN6', SIGMA=1.5, FILL_VALUE=np.nan, **MAR_KWARGS) # set attributes to output for iteration OUTPUT['zfirn'].data[i, c, xo] = np.copy(ZN4.data) OUTPUT['zfirn'].mask[i, c, xo] = np.copy(ZN4.mask) OUTPUT['zfirn'].interpolation[i, c, xo] = np.copy( ZN4.interpolation) OUTPUT['zsurf'].data[i, c, xo] = np.copy(ZN6.data) OUTPUT['zsurf'].mask[i, c, xo] = np.copy(ZN6.mask) OUTPUT['zsurf'].interpolation[i, c, xo] = np.copy( ZN6.interpolation) OUTPUT['zmelt'].data[i, c, xo] = np.copy(ZN5.data) OUTPUT['zmelt'].mask[i, c, xo] = np.copy(ZN5.mask) OUTPUT['zmelt'].interpolation[i, c, xo] = np.copy( ZN5.interpolation) # calculate derived fields OUTPUT['zsmb'].data[i, c, xo] = ZN6.data - ZN4.data OUTPUT['zsmb'].mask[i, c, xo] = ZN4.mask | ZN6.mask OUTPUT['zaccum'].data[ i, c, xo] = ZN6.data - ZN4.data - ZN5.data OUTPUT['zaccum'].mask[ i, c, xo] = ZN4.mask | ZN5.mask | ZN6.mask elif (MODEL == 'RACMO'): # read and interpolate daily RACMO outputs hgtsrf = SMBcorr.interpolate_racmo_daily( base_dir, EPSG, RACMO_MODEL, tdec, D11.x[i, c, xo], D11.y[i, c, xo], VARIABLE='hgtsrf', SIGMA=1.5, FILL_VALUE=np.nan) # set attributes to output for iteration OUTPUT['zsurf'].data[i, c, xo] = np.copy(hgtsrf.data) OUTPUT['zsurf'].mask[i, c, xo] = np.copy(hgtsrf.mask) OUTPUT['zsurf'].interpolation[i, c, xo] = np.copy( hgtsrf.interpolation) elif (MODEL == 'MERRA2-hybrid'): # read and interpolate 5-day MERRA2-Hybrid outputs FAC = SMBcorr.interpolate_merra_hybrid( DIRECTORY, EPSG, MERRA2_REGION, tdec, D11.x[i, c, xo], D11.y[i, c, xo], VERSION=MERRA2_MAJOR_VERSION, VARIABLE='FAC', SIGMA=1.5, FILL_VALUE=np.nan) smb = SMBcorr.interpolate_merra_hybrid( DIRECTORY, EPSG, MERRA2_REGION, tdec, D11.x[i, c, xo], D11.y[i, c, xo], VERSION=MERRA2_MAJOR_VERSION, VARIABLE='cum_smb_anomaly', SIGMA=1.5, FILL_VALUE=np.nan) height = SMBcorr.interpolate_merra_hybrid( DIRECTORY, EPSG, MERRA2_REGION, tdec, D11.x[i, c, xo], D11.y[i, c, xo], VERSION=MERRA2_MAJOR_VERSION, VARIABLE='height', SIGMA=1.5, FILL_VALUE=np.nan) # set attributes to output for iteration OUTPUT['zfirn'].data[i, c, xo] = np.copy(FAC.data) OUTPUT['zfirn'].mask[i, c, xo] = np.copy(FAC.mask) OUTPUT['zfirn'].interpolation[i, c, xo] = np.copy( FAC.interpolation) OUTPUT['zsurf'].data[i, c, xo] = np.copy(height.data) OUTPUT['zsurf'].mask[i, c, xo] = np.copy(height.mask) OUTPUT['zsurf'].interpolation[i, c, xo] = np.copy( height.interpolation) OUTPUT['zsmb'].data[i, c, xo] = np.copy(smb.data) OUTPUT['zsmb'].mask[i, c, xo] = np.copy(smb.mask) OUTPUT['zsmb'].interpolation[i, c, xo] = np.copy( smb.interpolation) else: # allocate for output height for along-track data OUTPUT = {} for key in KEYS: OUTPUT[key] = np.ma.zeros((nseg, ncycle), fill_value=np.nan) OUTPUT[key].mask = np.ones((nseg, ncycle), dtype=np.bool) OUTPUT[key].interpolation = np.zeros((nseg, ncycle), dtype=np.uint8) # check that there are valid elevations cycle = [ c for c in range(ncycle) if np.any(np.isfinite(D11.delta_time[:, c])) ] # for each valid cycle of ICESat-2 ATL11 data for c in cycle: # find valid elevations i, = np.nonzero(np.isfinite(D11.delta_time[:, c])) # convert from delta time to decimal-years tdec = convert_delta_time(D11.delta_time[i, c])['decimal'] if (MODEL == 'MAR'): # read and interpolate daily MAR outputs ZN4 = SMBcorr.interpolate_mar_daily(DIRECTORY, EPSG, MAR_VERSION, tdec, D11.x[i, c], D11.y[i, c], VARIABLE='ZN4', SIGMA=1.5, FILL_VALUE=np.nan, **MAR_KWARGS) ZN5 = SMBcorr.interpolate_mar_daily(DIRECTORY, EPSG, MAR_VERSION, tdec, D11.x[i, c], D11.y[i, c], VARIABLE='ZN5', SIGMA=1.5, FILL_VALUE=np.nan, **MAR_KWARGS) ZN6 = SMBcorr.interpolate_mar_daily(DIRECTORY, EPSG, MAR_VERSION, tdec, D11.x[i, c], D11.y[i, c], VARIABLE='ZN6', SIGMA=1.5, FILL_VALUE=np.nan, **MAR_KWARGS) # set attributes to output for iteration OUTPUT['zfirn'].data[i, c] = np.copy(ZN4.data) OUTPUT['zfirn'].mask[i, c] = np.copy(ZN4.mask) OUTPUT['zfirn'].interpolation[i, c] = np.copy( ZN4.interpolation) OUTPUT['zsurf'].data[i, c] = np.copy(ZN6.data) OUTPUT['zsurf'].mask[i, c] = np.copy(ZN6.mask) OUTPUT['zsurf'].interpolation[i, c] = np.copy( ZN6.interpolation) OUTPUT['zmelt'].data[i, c] = np.copy(ZN5.data) OUTPUT['zmelt'].mask[i, c] = np.copy(ZN5.mask) OUTPUT['zmelt'].interpolation[i, c] = np.copy( ZN5.interpolation) # calculate derived fields OUTPUT['zsmb'].data[i, c] = ZN6.data - ZN4.data OUTPUT['zsmb'].mask[i, c] = ZN4.mask | ZN6.mask OUTPUT['zaccum'].data[i, c] = ZN6.data - ZN4.data - ZN5.data OUTPUT['zaccum'].mask[i, c] = ZN4.mask | ZN5.mask | ZN6.mask elif (MODEL == 'RACMO'): # read and interpolate daily RACMO outputs hgtsrf = SMBcorr.interpolate_racmo_daily(base_dir, EPSG, RACMO_MODEL, tdec, D11.x[i, c], D11.y[i, c], VARIABLE='hgtsrf', SIGMA=1.5, FILL_VALUE=np.nan) # set attributes to output for iteration OUTPUT['zsurf'].data[i, c] = np.copy(hgtsrf.data) OUTPUT['zsurf'].mask[i, c] = np.copy(hgtsrf.mask) OUTPUT['zsurf'].interpolation[i, c] = np.copy( hgtsrf.interpolation) elif (MODEL == 'MERRA2-hybrid'): # read and interpolate 5-day MERRA2-Hybrid outputs FAC = SMBcorr.interpolate_merra_hybrid( DIRECTORY, EPSG, MERRA2_REGION, tdec, D11.x[i, c], D11.y[i, c], VERSION=MERRA2_MAJOR_VERSION, VARIABLE='FAC', SIGMA=1.5, FILL_VALUE=np.nan) smb = SMBcorr.interpolate_merra_hybrid( DIRECTORY, EPSG, MERRA2_REGION, tdec, D11.x[i, c], D11.y[i, c], VERSION=MERRA2_MAJOR_VERSION, VARIABLE='cum_smb_anomaly', SIGMA=1.5, FILL_VALUE=np.nan) height = SMBcorr.interpolate_merra_hybrid( DIRECTORY, EPSG, MERRA2_REGION, tdec, D11.x[i, c], D11.y[i, c], VERSION=MERRA2_MAJOR_VERSION, VARIABLE='height', SIGMA=1.5, FILL_VALUE=np.nan) # set attributes to output for iteration OUTPUT['zfirn'].data[i, c] = np.copy(FAC.data) OUTPUT['zfirn'].mask[i, c] = np.copy(FAC.mask) OUTPUT['zfirn'].interpolation[i, c] = np.copy( FAC.interpolation) OUTPUT['zsurf'].data[i, c] = np.copy(height.data) OUTPUT['zsurf'].mask[i, c] = np.copy(height.mask) OUTPUT['zsurf'].interpolation[i, c] = np.copy( height.interpolation) OUTPUT['zsmb'].data[i, c] = np.copy(smb.data) OUTPUT['zsmb'].mask[i, c] = np.copy(smb.mask) OUTPUT['zsmb'].interpolation[i, c] = np.copy( smb.interpolation) # append input HDF5 file with new firn model outputs fileID = h5py.File(os.path.expanduser(input_file), 'a') fileID.create_group(model_version) h5 = {} for key in KEYS: # verify mask values OUTPUT[key].mask |= (OUTPUT[key].data == OUTPUT[key].fill_value) | \ np.isnan(OUTPUT[key].data) OUTPUT[key].data[OUTPUT[key].mask] = OUTPUT[key].fill_value # output variable to HDF5 val = '{0}/{1}'.format(model_version, key) h5[key] = fileID.create_dataset(val, OUTPUT[key].shape, data=OUTPUT[key], dtype=OUTPUT[key].dtype, compression='gzip', fillvalue=OUTPUT[key].fill_value) h5[key].attrs['units'] = "m" h5[key].attrs['long_name'] = LONGNAME[key] h5[key].attrs[ 'coordinates'] = "../delta_time ../latitude ../longitude" h5[key].attrs['model'] = model_version # close the output HDF5 file fileID.close()
#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ Created on Tue Jan 21 15:56:39 2020 @author: ben """ import pointCollection as pc import numpy as np import sys import matplotlib.pyplot as plt plt.figure() for index_file in sys.argv[1:]: xy = pc.geoIndex().from_file(index_file).bins_as_array() plt.plot(xy[0], xy[1], 'o', label=index_file) D = pc.geoIndex().from_file(index_file).query_xy( [[xy[0][0]], [xy[1][0]]], get_data=True, fields=['x', 'y', 'h_li', 'pair', 'cycle_number']) D = pc.data().from_list(D) plt.plot(D.x, D.y, '.', label='first bin of ' + index_file) plt.axis('equal') plt.legend() plt.show()
def main(): parser = argparse.ArgumentParser() parser.add_argument('--gps', '-G', type=str, help="GPS file to run") parser.add_argument('--atm', '-A', type=str, help='ATM directory to run') parser.add_argument('--hemisphere', '-H', type=int, default=-1, help='hemisphere, must be 1 or -1') parser.add_argument('--query', '-Q', type=float, default=100, help='KD-Tree query radius') parser.add_argument('--median', '-M', default=False, action='store_true', help='Run block median') parser.add_argument('--scan', '-S', default=False, action='store_true', help='Run ATM scan fit') parser.add_argument('--verbose', '-v', default=False, action='store_true', help='verbose output of run') args = parser.parse_args() if args.hemisphere == 1: SRS_proj4 = '+proj=stere +lat_0=90 +lat_ts=70 +lon_0=-45 +k=1 +x_0=0 +y_0=0 +datum=WGS84 +units=m +no_defs ' elif args.hemisphere == -1: SRS_proj4 = '+proj=stere +lat_0=-90 +lat_ts=-71 +lon_0=0 +k=1 +x_0=0 +y_0=0 +datum=WGS84 +units=m +no_defs' # tilde expansion of file arguments GPS_file = os.path.expanduser(args.gps) fileBasename, fileExtension = os.path.splitext(GPS_file) ATM_dir = os.path.expanduser(args.atm) print("working on GPS file {0}, ATM directory {1}".format( GPS_file, ATM_dir)) if args.verbose else None # find Qfit files within ATM_dir Qfit_regex = re.compile( r"ATM1B.*_(\d{4})(\d{2})(\d{2})_(\d{2})(\d{2})(\d{2}).*.h5$") Qfit_files = [ os.path.join(ATM_dir, f) for f in os.listdir(ATM_dir) if Qfit_regex.search(f) ] # output directory out_dir = os.path.join(ATM_dir, 'xovers') if not os.path.isdir(out_dir): os.mkdir(out_dir) # output file out_file = 'vs_{0}.h5'.format(os.path.basename(fileBasename)) # check if output file exists if os.path.isfile(os.path.join(out_dir, out_file)): print("found: {0}".format(os.path.join( out_dir, out_file))) if args.verbose else None # read GPS HDF5 file GPS_field_dict = {None: ['latitude', 'longitude', 'z']} GPS = pc.data().from_h5(GPS_file, field_dict=GPS_field_dict).get_xy(SRS_proj4) # run block median over GPS data if args.median: GPS = blockmedian_for_gps(GPS, 5) # read all Qfit files within ATM directory Qlist = list() for f in sorted(Qfit_files): Qlist.append(pc.ATM_Qfit.data().from_h5(f)) # merge the list of ATM data and build the search tree Q_full = pc.data().from_list(Qlist).get_xy(SRS_proj4) # fit scan parameters to an ATM data structure if args.scan: Q_full = fit_ATM_data(Q_full) # run block median for qsub if args.median: Q_full = blockmedian_for_qsub(Q_full, 5) # construct search tree from ATM Qfit coords # pickle Qtree to save computational time for future runs if os.path.isfile(os.path.join(ATM_dir, 'tree.p')): Qtree = pickle.load(open(os.path.join(ATM_dir, 'tree.p'), 'rb')) else: Qtree = KDTree(np.c_[Q_full.x, Q_full.y]) pickle.dump(Qtree, open(os.path.join(ATM_dir, 'tree.p'), 'wb')) # output fields out_fields = [ 'x', 'y', 'z', 'longitude', 'latitude', 't_qfit', 'h_qfit_50m', 'sigma_qfit_50m', 'dz_50m', 'RDE_50m', 'N_50m', 'hbar_20m', 'h_qfit_10m', 'sigma_qfit_10m', 'dz_10m', 'RDE_10m', 'N_10m', 'x_10m_mean', 'y_10m_mean' ] # append scan fields to output template if args.scan: out_fields.extend(['scan_XT_50m', 'scan_XT_10m']) out_template = {f: np.NaN for f in out_fields} out = list() # query the search tree to find points within query radius Qquery = Qtree.query_radius(np.c_[GPS.x, GPS.y], args.query) # indices of GPS points within bin ind, = np.nonzero([np.any(i) for i in Qquery]) # loop over queries in the GPS data for i in ind: GPSsub = GPS.copy_subset(np.array([i])) # grab the Qfit bins around the GPS bin Qdata = Q_full.copy_subset(Qquery[i], by_row=True) Qdata.index( np.isfinite(Qdata.elevation) & np.isfinite(Qdata.latitude) & np.isfinite(Qdata.longitude)) # create output dictionary of GPS and plane-fit ATM comparison this_out = compare_gps_with_qfit(GPSsub, Qdata, out_template) if this_out is not None: out.append(this_out) # if there were overlapping points between the GPS and ATM data if out: D = dict() with h5py.File(os.path.join(out_dir, out_file), 'w') as h5f: for field in out[0].keys(): D[field] = np.array([ii[field] for ii in out]) print(field, D[field].dtype) if args.verbose else None h5f.create_dataset(field, data=D[field])
def append_SMB_averages_ATL11(input_file, base_dir, REGION, MODEL, RANGE=[2000, 2019], VERBOSE=False): # read input file field_dict = {None: ('delta_time', 'h_corr', 'x', 'y')} D11 = pc.data().from_h5(input_file, field_dict=field_dict) # check if running crossover or along-track ATL11 if (D11.h_corr.ndim == 3): nseg, ncycle, ncross = D11.shape else: nseg, ncycle = D11.shape # get projection of input coordinates EPSG = set_projection(REGION) # available models models = dict(AA={}, GL={}) # MAR models['GL']['MAR'] = [] # models['GL']['MAR'].append('MARv3.9-ERA') # models['GL']['MAR'].append('MARv3.10-ERA') # models['GL']['MAR'].append('MARv3.11-NCEP') # models['GL']['MAR'].append('MARv3.11-ERA') # models['GL']['MAR'].append('MARv3.11.2-ERA-6km') # models['GL']['MAR'].append('MARv3.11.2-ERA-7.5km') # models['GL']['MAR'].append('MARv3.11.2-ERA-10km') # models['GL']['MAR'].append('MARv3.11.2-ERA-15km') # models['GL']['MAR'].append('MARv3.11.2-ERA-20km') # models['GL']['MAR'].append('MARv3.11.2-NCEP-20km') # models['GL']['MAR'].append('MARv3.11.5-ERA-6km') models['GL']['MAR'].append('MARv3.11.5-ERA-10km') models['GL']['MAR'].append('MARv3.11.5-ERA-15km') models['GL']['MAR'].append('MARv3.11.5-ERA-20km') # RACMO models['GL']['RACMO'] = [] # models['GL']['RACMO'].append('RACMO2.3-XGRN11') # models['GL']['RACMO'].append('RACMO2.3p2-XGRN11') models['GL']['RACMO'].append('RACMO2.3p2-FGRN055') # MERRA2-hybrid models['GL']['MERRA2-hybrid'] = [] # models['GL']['MERRA2-hybrid'].append('GSFC-fdm-v0') # models['GL']['MERRA2-hybrid'].append('GSFC-fdm-v1') models['GL']['MERRA2-hybrid'].append('GSFC-fdm-v1.1') models['AA']['MERRA2-hybrid'] = [] # models['AA']['MERRA2-hybrid'].append('GSFC-fdm-v0') models['AA']['MERRA2-hybrid'].append('GSFC-fdm-v1') models['AA']['MERRA2-hybrid'].append('GSFC-fdm-v1.1') # for each model to append to ATL11 for model_version in models[REGION][MODEL]: # keyword arguments for all models # add range to input keyword arguments KWARGS = dict(SIGMA=1.5, FILL_VALUE=np.nan, RANGE=RANGE) if (MODEL == 'MAR'): match_object = re.match(r'(MARv\d+\.\d+(.\d+)?)', model_version) MAR_VERSION = match_object.group(0) MAR_REGION = dict(GL='Greenland', AA='Antarctic')[REGION] # model subdirectories SUBDIRECTORY = dict(AA={}, GL={}) SUBDIRECTORY['GL']['MARv3.9-ERA'] = [ 'ERA_1958-2018_10km', 'daily_10km' ] SUBDIRECTORY['GL']['MARv3.10-ERA'] = [ 'ERA_1958-2019-15km', 'daily_15km' ] SUBDIRECTORY['GL']['MARv3.11-NCEP'] = [ 'NCEP1_1948-2020_20km', 'daily_20km' ] SUBDIRECTORY['GL']['MARv3.11-ERA'] = [ 'ERA_1958-2019-15km', 'daily_15km' ] SUBDIRECTORY['GL']['MARv3.11.2-ERA-6km'] = ['6km_ERA5'] SUBDIRECTORY['GL']['MARv3.11.2-ERA-7.5km'] = ['7.5km_ERA5'] SUBDIRECTORY['GL']['MARv3.11.2-ERA-10km'] = ['10km_ERA5'] SUBDIRECTORY['GL']['MARv3.11.2-ERA-15km'] = ['15km_ERA5'] SUBDIRECTORY['GL']['MARv3.11.2-ERA-20km'] = ['20km_ERA5'] SUBDIRECTORY['GL']['MARv3.11.2-NCEP-20km'] = ['20km_NCEP1'] SUBDIRECTORY['GL']['MARv3.11.5-ERA-6km'] = ['6km_ERA5'] SUBDIRECTORY['GL']['MARv3.11.5-ERA-10km'] = ['10km_ERA5'] SUBDIRECTORY['GL']['MARv3.11.5-ERA-15km'] = ['15km_ERA5'] SUBDIRECTORY['GL']['MARv3.11.5-ERA-20km'] = ['20km_ERA5'] MAR_MODEL = SUBDIRECTORY[REGION][model_version] DIRECTORY = os.path.join(base_dir, 'MAR', MAR_VERSION, MAR_REGION, *MAR_MODEL) # keyword arguments for variable coordinates MAR_KWARGS = dict(AA={}, GL={}) MAR_KWARGS['GL']['MARv3.9-ERA'] = dict(XNAME='X10_153', YNAME='Y21_288') MAR_KWARGS['GL']['MARv3.10-ERA'] = dict(XNAME='X10_105', YNAME='Y21_199') MAR_KWARGS['GL']['MARv3.11-NCEP'] = dict(XNAME='X12_84', YNAME='Y21_155') MAR_KWARGS['GL']['MARv3.11-ERA'] = dict(XNAME='X10_105', YNAME='Y21_199') MAR_KWARGS['GL']['MARv3.11.2-ERA-6km'] = dict(XNAME='X12_251', YNAME='Y20_465') MAR_KWARGS['GL']['MARv3.11.2-ERA-7.5km'] = dict(XNAME='X12_203', YNAME='Y20_377') MAR_KWARGS['GL']['MARv3.11.2-ERA-10km'] = dict(XNAME='X10_153', YNAME='Y21_288') MAR_KWARGS['GL']['MARv3.11.2-ERA-15km'] = dict(XNAME='X10_105', YNAME='Y21_199') MAR_KWARGS['GL']['MARv3.11.2-ERA-20km'] = dict(XNAME='X12_84', YNAME='Y21_155') MAR_KWARGS['GL']['MARv3.11.2-NCEP-20km'] = dict(XNAME='X12_84', YNAME='Y21_155') MAR_KWARGS['GL']['MARv3.11.5-ERA-6km'] = dict(XNAME='X12_251', YNAME='Y20_465') MAR_KWARGS['GL']['MARv3.11.5-ERA-10km'] = dict(XNAME='X10_153', YNAME='Y21_288') MAR_KWARGS['GL']['MARv3.11.5-ERA-15km'] = dict(XNAME='X10_105', YNAME='Y21_199') MAR_KWARGS['GL']['MARv3.11.5-ERA-20km'] = dict(XNAME='X12_84', YNAME='Y21_155') KWARGS.update(MAR_KWARGS[REGION][model_version]) # netCDF4 variable names for direct fields VARIABLES = ['ZN6', 'ZN4', 'ZN5'] # output variable keys for both direct and derived fields KEYS = [ 'zsurf_ave', 'zfirn_ave', 'zmelt_ave', 'zsmb_ave', 'zaccum_ave' ] # HDF5 longname attributes for each variable LONGNAME = {} LONGNAME['zsurf_ave'] = "Snow Height Change" LONGNAME['zfirn_ave'] = "Snow Height Change due to Compaction" LONGNAME['zmelt_ave'] = "Snow Height Change due to Surface Melt" LONGNAME[ 'zsmb_ave'] = "Snow Height Change due to Surface Mass Balance" LONGNAME[ 'zaccum_ave'] = "Snow Height Change due to Surface Accumulation" elif (MODEL == 'RACMO'): RACMO_VERSION, RACMO_MODEL = model_version.split('-') # netCDF4 variable names VARIABLES = ['hgtsrf'] # output variable keys KEYS = ['zsurf_ave'] # HDF5 longname attributes for each variable LONGNAME = {} LONGNAME['zsurf_ave'] = "Snow Height Change" elif (MODEL == 'MERRA2-hybrid'): # regular expression pattern for extracting version merra2_regex = re.compile(r'GSFC-fdm-((v\d+)(\.\d+)?)$') # get MERRA-2 version and major version MERRA2_VERSION = merra2_regex.match(model_version).group(1) # MERRA-2 hybrid directory DIRECTORY = os.path.join(base_dir, 'MERRA2_hybrid', MERRA2_VERSION) # MERRA-2 region name from ATL11 region MERRA2_REGION = dict(AA='ais', GL='gris')[REGION] # keyword arguments for MERRA-2 interpolation programs if MERRA2_VERSION in ('v0', 'v1', 'v1.0'): KWARGS['VERSION'] = merra2_regex.match(model_version).group(2) # netCDF4 variable names VARIABLES = ['FAC', 'cum_smb_anomaly', 'height'] # add additional Greenland variables if (MERRA2_REGION == 'gris'): VARIABLES.append('runoff_anomaly') else: KWARGS['VERSION'] = MERRA2_VERSION.replace('.', '_') # netCDF4 variable names VARIABLES = ['FAC', 'SMB_a', 'h_a'] # add additional Greenland variables if (MERRA2_REGION == 'gris'): VARIABLES.append('Me_a') # output variable keys KEYS = ['zsurf_ave', 'zfirn_ave', 'zsmb_ave'] # HDF5 longname attributes for each variable LONGNAME = {} LONGNAME['zsurf_ave'] = "Snow Height Change" LONGNAME['zfirn_ave'] = "Snow Height Change due to Compaction" LONGNAME[ 'zsmb_ave'] = "Snow Height Change due to Surface Mass Balance" # check if running crossover or along track if (D11.h_corr.ndim == 3): # allocate for output height for crossover data OUTPUT = {} for key in KEYS: OUTPUT[key] = np.ma.zeros((nseg, ncycle, ncross), fill_value=np.nan) OUTPUT[key].mask = np.ones((nseg, ncycle, ncross), dtype=bool) # for each cycle of ICESat-2 ATL11 data for c in range(ncycle): # check that there are valid crossovers cross = [ xo for xo in range(ncross) if np.any(np.isfinite(D11.delta_time[:, c, xo])) ] # for each valid crossing for xo in cross: # find valid crossovers i, = np.nonzero(np.isfinite(D11.delta_time[:, c, xo])) # convert from delta time to decimal-years tdec = convert_delta_time(D11.delta_time[i, c, xo])['decimal'] if (MODEL == 'MAR'): for key, var in zip(KEYS, VARIABLES): # read and interpolate daily MAR outputs OUT = SMBcorr.interpolate_mar_daily(DIRECTORY, EPSG, MAR_VERSION, tdec, D11.x[i, c, xo], D11.y[i, c, xo], VARIABLE=var, **KWARGS) # set attributes to output for iteration OUTPUT[key].data[i, c, xo] = np.copy(OUT.data) OUTPUT[key].mask[i, c, xo] = np.copy(OUT.mask) OUTPUT[key].interpolation[i, c, xo] = np.copy( OUT.interpolation) # calculate derived fields OUTPUT['zsmb_ave'].data[i,c,xo] = OUTPUT['zsurf_ave'].data[i,c,xo] - \ OUTPUT['zfirn_ave'].data[i,c,xo] OUTPUT['zsmb_ave'].mask[i,c,xo] = OUTPUT['zsurf_ave'].mask[i,c,xo] | \ OUTPUT['zfirn_ave'].mask[i,c,xo] OUTPUT['zaccum_ave'].data[i,c,xo] = OUTPUT['zsurf_ave'].data[i,c,xo] - \ OUTPUT['zfirn_ave'].data[i,c,xo] - OUTPUT['zmelt_ave'].data[i,c,xo] OUTPUT['zaccum_ave'].mask[i,c,xo] = OUTPUT['zsurf_ave'].mask[i,c,xo] | \ OUTPUT['zfirn_ave'].mask[i,c,xo] | OUTPUT['zmelt_ave'].mask[i,c,xo] elif (MODEL == 'RACMO'): # read and interpolate daily RACMO outputs for key, var in zip(KEYS, VARIABLES): OUT = SMBcorr.interpolate_racmo_daily(base_dir, EPSG, RACMO_MODEL, tdec, D11.x[i, c, xo], D11.y[i, c, xo], VARIABLE=var, **KWARGS) # set attributes to output for iteration OUTPUT[key].data[i, c, xo] = np.copy(OUT.data) OUTPUT[key].mask[i, c, xo] = np.copy(OUT.mask) OUTPUT[key].interpolation[i, c, xo] = np.copy( OUT.interpolation) elif (MODEL == 'MERRA2-hybrid'): # read and interpolate 5-day MERRA2-Hybrid outputs for key, var in zip(KEYS, VARIABLES): OUT = SMBcorr.interpolate_merra_hybrid( DIRECTORY, EPSG, MERRA2_REGION, tdec, D11.x[i, c, xo], D11.y[i, c, xo], VARIABLE=var, **KWARGS) # set attributes to output for iteration OUTPUT[key].data[i, c, xo] = np.copy(OUT.data) OUTPUT[key].mask[i, c, xo] = np.copy(OUT.mask) OUTPUT[key].interpolation[i, c, xo] = np.copy( OUT.interpolation) else: # allocate for output height for along-track data OUTPUT = {} for key in KEYS: OUTPUT[key] = np.ma.zeros((nseg, ncycle), fill_value=np.nan) OUTPUT[key].mask = np.ones((nseg, ncycle), dtype=bool) # check that there are valid elevations cycle = [ c for c in range(ncycle) if np.any(np.isfinite(D11.delta_time[:, c])) ] # for each valid cycle of ICESat-2 ATL11 data for c in cycle: # find valid elevations i, = np.nonzero(np.isfinite(D11.delta_time[:, c])) # convert from delta time to decimal-years tdec = convert_delta_time(D11.delta_time[i, c])['decimal'] if (MODEL == 'MAR'): for key, var in zip(KEYS, VARIABLES): # read and interpolate daily MAR outputs OUT = SMBcorr.interpolate_mar_daily(DIRECTORY, EPSG, MAR_VERSION, tdec, D11.x[i, c], D11.y[i, c], VARIABLE=var, **KWARGS) # set attributes to output for iteration OUTPUT[key].data[i, c] = np.copy(OUT.data) OUTPUT[key].mask[i, c] = np.copy(OUT.mask) OUTPUT[key].interpolation[i, c] = np.copy( OUT.interpolation) # calculate derived fields OUTPUT['zsmb_ave'].data[i,c] = OUTPUT['zsurf_ave'].data[i,c] - \ OUTPUT['zfirn_ave'].data[i,c] OUTPUT['zsmb_ave'].mask[i,c] = OUTPUT['zsurf_ave'].mask[i,c] | \ OUTPUT['zfirn_ave'].mask[i,c] OUTPUT['zaccum'].data[i,c] = OUTPUT['zsurf_ave'].data[i,c] - \ OUTPUT['zfirn_ave'].data[i,c] - OUTPUT['zmelt_ave'].data[i,c] OUTPUT['zaccum'].mask[i,c] = OUTPUT['zsurf_ave'].mask[i,c] | \ OUTPUT['zfirn_ave'].mask[i,c] | OUTPUT['zmelt_ave'].mask[i,c] elif (MODEL == 'RACMO'): # read and interpolate daily RACMO outputs for key, var in zip(KEYS, VARIABLES): OUT = SMBcorr.interpolate_racmo_daily(base_dir, EPSG, RACMO_MODEL, tdec, D11.x[i, c], D11.y[i, c], VARIABLE=var, **KWARGS) # set attributes to output for iteration OUTPUT[key].data[i, c] = np.copy(OUT.data) OUTPUT[key].mask[i, c] = np.copy(OUT.mask) OUTPUT[key].interpolation[i, c] = np.copy( OUT.interpolation) elif (MODEL == 'MERRA2-hybrid'): # read and interpolate 5-day MERRA2-Hybrid outputs for key, var in zip(KEYS, VARIABLES): OUT = SMBcorr.interpolate_merra_hybrid(DIRECTORY, EPSG, MERRA2_REGION, tdec, D11.x[i, c], D11.y[i, c], VARIABLE=var, **KWARGS) # set attributes to output for iteration OUTPUT[key].data[i, c] = np.copy(OUT.data) OUTPUT[key].mask[i, c] = np.copy(OUT.mask) OUTPUT[key].interpolation[i, c] = np.copy( OUT.interpolation) # append input HDF5 file with new firn model outputs fileID = h5py.File(os.path.expanduser(input_file), 'a') print(input_file) if VERBOSE else None # fileID.create_group(model_version) if model_version not in fileID.keys() else None h5 = {} for key in KEYS: print(f'{sys.argv[0]}: writing{key}') if VERBOSE else None # verify mask values OUTPUT[key].mask |= (OUTPUT[key].data == OUTPUT[key].fill_value) | \ np.isnan(OUTPUT[key].data) OUTPUT[key].data[OUTPUT[key].mask] = OUTPUT[key].fill_value # output variable to HDF5 val = '{0}/{1}'.format(model_version, key) if val not in fileID: h5[key] = fileID.create_dataset( val, OUTPUT[key].shape, data=OUTPUT[key], dtype=OUTPUT[key].dtype, compression='gzip', fillvalue=OUTPUT[key].fill_value) else: h5[key] = fileID[val] fileID[val][...] = OUTPUT[key] h5[key].attrs['units'] = "m" h5[key].attrs['long_name'] = LONGNAME[key] h5[key].attrs[ 'coordinates'] = "../delta_time ../latitude ../longitude" h5[key].attrs['model'] = model_version # close the output HDF5 file fileID.close()
@author: ben """ import matplotlib.pyplot as plt import pointCollection as pc import sys from mpl_toolkits.mplot3d import Axes3D import numpy as np D = [] for file in sys.argv[1:]: D.append({\ 'dz':pc.grid.data().from_h5(file, group='dz/', field_mapping={'z':'dz'}), 'count':pc.grid.data().from_h5(file, group='dz/', field_mapping={'z':'count'}), 'z0':pc.grid.data().from_h5(file, group='z0/', field_mapping={'z':'z0'}), 'data':pc.data().from_h5(file, group='data/')}) plt.clf() hax = [] for ii in range(1, 5): hax.append(plt.gcf().add_subplot(2, 2, ii)) t_slice = [0, 2] for Di in D: dt = Di['dz'].t[t_slice[1]] - Di['dz'].t[t_slice[0]] h_im = hax[0].imshow( (Di['dz'].z[:, :, t_slice[1]] - Di['dz'].z[:, :, t_slice[0]]) / dt, extent=Di['dz'].extent, cmap='Spectral', origin='lower',
def read(self, xy_bin, fields=['x', 'y', 'time'], index_range=[[-1], [-1]]): if isinstance(fields, dict): field_list = [] for key in fields: field_list += fields[key] else: field_list = fields.copy() out_data = {field: list() for field in field_list} with h5py.File(self.filename, 'r') as h5f: blank_fields = list() if isinstance(xy_bin, np.ndarray): xy_bin = [xy_bin[:, 0], xy_bin[:, 1]] if index_range[0][0] >= 0: # All the geo bins are together. Use the index_range variable to read for field in field_list: if field not in h5f: blank_fields.append(field) continue # make sure the index range gets iterated over properly. if len(index_range[0]) < 2: if len(h5f[field].shape) > 1: out_data[field].append( np.array(h5f[field][ 0, int(index_range[0]):int(index_range[1])])) else: out_data[field].append( np.array( h5f[field] [int(index_range[0]):int(index_range[1])])) else: for i0_i1 in zip(index_range[0], index_range[1]): if len(h5f[field].shape) > 1: out_data[field].append( np.array(h5f[field][0, i0_i1[0]:i0_i1[1]])) else: out_data[field].append( np.array(h5f[field][i0_i1[0]:i0_i1[1]])) else: # this is a file with distinct bins, each with its own set of datasets for xy in zip(xy_bin[0], xy_bin[1]): bin_name = '%dE_%dN' % xy for field in field_list: if field in h5f: if bin_name in h5f[field]: out_data[field].append( np.array(h5f[field][bin_name]).squeeze()) elif bin_name in h5f: if field in h5f[bin_name]: out_data[field].append( np.array(h5f[bin_name][field]).squeeze()) else: blank_fields.append(field) for field in field_list: if isinstance(out_data[field], list): if len(out_data[field]) > 1 or isinstance( out_data[field], list): try: temp = list() for item in out_data[field]: if item.size > 0 and item.ndim > 0: temp.append(item) elif item.size == 1 and item.ndim == 0: temp.append(np.array([item])) if len(temp) > 1: out_data[field] = np.concatenate(temp) elif len(temp) == 0: out_data[field] = np.zeros(0) elif len(temp) == 1: out_data[field] = temp[0] except ValueError as e: print( "ValueError in read_indexed_h5_file, continuing" ) print(e) else: out_data[field] = np.array(out_data[field]) return pc.data(fields=field_list).from_dict(out_data)
'hbar_20m', 'RDE_50m','t_seg','t_qfit','y_atc', 'x_seg_mean', 'y_seg_mean'] out_template={f:np.NaN for f in out_fields} out=list() #READ IN THE ATM DATA HERE AND BUILD THE QTREE # ALSO RUN BLOCKMEDIAN_FOR_QSUB #plt.figure(1) for bin_name in D6_GI.keys(): #plt.clf() print(bin_name) bin_xy=[int(coord) for coord in bin_name.split('_')] # query the Qfit index to get all the data for the current bin. pad=6 means that we read 6 km on either side of the central bin, covering the 10-km ATL06 bin plus 1 km buffer Qlist=Q_GI.query_xy([[bin_xy[0]], [bin_xy[1]]], pad=6, get_data=True, strict=True, fields=Qfit_fields) Qsub=pc.data().from_list(Qlist).get_xy(SRS_proj4) Qsub=blockmedian_for_qsub(Qsub, 5) #START CUTTING HERE # the geo index works much better (more selective) if the Qfit data are sorted by geobin x0=np.round(Qsub.x/100.)*100 y0=np.round(Qsub.x/100.)*100 iB=np.argsort(x0+(y0-y0.min())/(y0.max()-y0.min())) Qsub.index(iB) # index the sorted Qfit data GI_Qsub=pc.geoIndex(delta=[100, 100]).from_xy([Qsub.x, Qsub.y]) # query ATL06 for the current bin, and index it
def find_best_wxx0(D11, W_domain, mask_file, DEBUG=False): scale_vals = np.array([0.01, 0.03, 0.1, 0.3, 1, 3, 10, 30, 100, 300]) E_d3zdx2dt = 0.0001 E_d2z0dx2 = 0.006 E_d2zdt2 = 5000 data_gap_scale = 2500 # define the domain's width in x, y, and time W = {'x': W_domain, 'y': 200, 't': .2} # define the grid center: XR = np.nanmean(D11.x_atc) + np.array([-1, 1]) * W['x'] / 2 ctr = {'x': XR[0] + W['x'] / 2., 'y': 0., 't': 0.} # define the grid spacing spacing = {'z0': 50, 'dz': 100, 'dt': .1} # reference points are every 60 m, but every third reference point # is returned. dN = np.ceil(W['x'] / 20).astype(int) L_interp = {} for pt0 in np.arange(D11.ref_pt[0, 0] + dN / 2, D11.ref_pt[-1, 0], dN / 2): ii = np.flatnonzero(np.abs(D11.ref_pt[:, 0] - pt0) < 3 * dN / 2) N_good = np.sum(np.isfinite(D11.h_corr[ii, :]), axis=0) if np.max(N_good) < 0.9 * dN: continue max_pts = np.max(N_good) good_enough = np.argwhere(N_good >= max_pts - 5) if len(good_enough) == 1: bc = good_enough[0] else: # choose the smoothest cycle that has enough points sigma_dz = np.nanmedian(np.abs( np.diff(D11.h_corr[ii, good_enough], axis=1)), axis=1) bc = good_enough[np.argmin(sigma_dz)] nb = N_good[bc] xy_ctr = [ np.nanmean(D11.x[ii, bc]), np.nanmean(D11.y[ii, bc]), np.nanmean(D11.h_corr[ii, bc]) ] D=pc.data().from_dict({'x':D11.x_atc[ii,bc], 'y':np.zeros_like(ii, dtype=float),'z':D11.h_corr[ii,bc],\ 'time':np.zeros_like(ii, dtype=float), 'sigma':D11.h_corr_sigma[ii,bc]}) D.index(np.isfinite(D.z) & np.isfinite(D.sigma) & (D.sigma > 0)) S = [] ctr = {'x': np.nanmean(D.x), 'y': 0., 't': 0.} L_curve = {key: [] for key in ['wzz0', 'sigma_hat_s', 'N']} for scale_val in scale_vals: # run the fit E_RMS = { 'd2z0_dx2': E_d2z0dx2 * scale_val, 'dz0_dx': E_d2z0dx2 * data_gap_scale * scale_val, 'd3z_dx2dt': E_d3zdx2dt, 'd2z_dxdt': E_d3zdx2dt * data_gap_scale, 'd2z_dt2': E_d2zdt2 } try: S.append( smooth_xytb_fit(data=D, ctr=ctr, W=W, spacing=spacing, E_RMS=E_RMS, reference_epoch=1, N_subset=None, compute_E=False, max_iterations=5, VERBOSE=False)) except Exception as e: S.append( smooth_xytb_fit(data=D, ctr=ctr, W=W, spacing=spacing, E_RMS=E_RMS, reference_epoch=1, N_subset=None, compute_E=False, max_iterations=5, VERBOSE=False)) d_ed = S[-1]['data'] d_ed.index(d_ed.three_sigma_edit == 1) L_curve['sigma_hat_s'].append( LSsurf.RDE((d_ed.z - d_ed.z_est) / d_ed.sigma)) L_curve['wzz0'].append(E_RMS['d2z0_dx2']) L_curve['N'].append(d_ed.size) for key in L_curve.keys(): L_curve[key] = np.array(L_curve[key]) try: N0 = safe_interp(1, L_curve['sigma_hat_s'], L_curve['N'], loglog=True) if DEBUG and np.nanmin(L_curve['sigma_hat_s'] > 1): return D, ii, bc, pt0, L_curve w_for_r_of_1 = safe_interp(1, L_curve['sigma_hat_s'], L_curve['wzz0'], loglog=True) w_for_r_10pct_above_min = \ safe_interp(1.1*L_curve['sigma_hat_s'].min(), L_curve['sigma_hat_s'], L_curve['wzz0'], loglog=True) L_interp[pt0] = { "w_for_r_of_1": w_for_r_of_1, 'w_for_r_10pct_above_min': w_for_r_10pct_above_min, 'sigma_hat_min': np.nanmin(L_curve['sigma_hat_s']), 'F_data_r_of_1': N0 / d_ed.size, 'x': xy_ctr[0], 'y': xy_ctr[1], 'z': xy_ctr[2], 'ref_pt': pt0 } #print([ L_interp[pt0]['w_for_r_of_1'], np.min(L_curve['sigma_hat_s'])]) except Exception as e: print(e) return L_interp
D11.get_xy(EPSG=3413) ind = np.arange(5, D11.x.shape[0], n_skip) els=(D11.x[ind,0] > MOG.x[0]) & (D11.x[ind,0] < MOG.x[-1]) & \ (D11.y[ind,0] > MOG.y[0]) & (D11.y[ind,0] < MOG.y[-1]) if els.size > 0: ind = ind[els] D11.assign({'file_ind': np.zeros_like(D11.x) + count}) D11.index(ind) xydh[data_count] = D11 data_count += 1 #except: # print("problem with "+file) D = pc.data(columns=D11.shape[1]).from_list(xydh) D.to_h5('/Volumes/ice2/ben/scf/GL_11/rel007_dump_every_4th.h5') if False: D3 = D[np.arange(0, D.shape[0], 3)] gimp_mask = pc.grid.data().from_geotif( '/Volumes/ice1/ben/GimpMasks_v1.1/GimpIceMask_1km.tif').interp( D3.x[:, 0], D3.y[:, 0]) D3 = D3[gimp_mask > 0.1] fig = plt.figure(4) plt.clf() xx = D3.x[:, 0] yy = D3.y[:, 0]
field_dict={'corrected_h':['h_corr','latitude','longitude','delta_time']} if True: MOG=pc.grid.data().from_geotif('/Volumes/ice1/ben/MOG/2005/mog_2005_1km.tif') thedir='/Volumes/ice2/ben/scf/GL_11_alpha' files=glob.glob(thedir+'/ATL11*.h5') xydh=[] for count, file in enumerate(files): for pair in [1, 2, 3]: filePair=(file, pair) D11 = ATL11.data().from_file(filePair[0], pair=filePair[1], field_dict=field_dict).get_xy(None, EPSG=3413) try: ind=np.arange(5, D11.x.size, 10) temp=pc.data().from_dict({'x':D11.x[ind],'y':D11.y[ind], \ 'dh':D11.corrected_h.h_corr[ind, cycles[1]]-D11.corrected_h.h_corr[ind, cycles[0]],\ 'file_ind': np.zeros_like(ind, dtype=int)+count}) els=(temp.x > MOG.x[0]) & (temp.x < MOG.x[-1]) & \ (temp.y > MOG.y[0]) & (temp.y < MOG.y[-1]) xydh.append(temp[els]) except: print("problem with "+file) xydh=pc.data().from_list(xydh) if True: plt.figure() MOG.show(cmap='gray', vmin=14000, vmax=17000) hl=plt.scatter(xydh.x, xydh.y, 3, linewidth=0, \ c=xydh.dh, \ vmin=-1.25, vmax=1.25, cmap='Spectral')
try: this_str += " " + files[count] except Exception: pass count += 1 this_str += "\n" fh.write(this_str) sys.exit() N_max = {'weak': 57. / 2. * 10. * 3., 'strong': 57. / 2. * 10. * 12.} #print(N_max) spot_max = {spot: N_max['weak'] for spot in [2, 4, 6]} spot_max.update({spot: N_max['strong'] for spot in [1, 3, 5]}) for file in files: out_file = out_dir + os.path.basename(file).replace('.h5', '_rh_avg.h5') z, Ntot, Nbar, P, Ps = read_one_hist(file, spot_max) print(out_file) for spot in range(1, 7): try: Ds = { 'z': z, 'Nbar': Nbar[spot], 'P': P[spot], 'P_sigma': Ps[spot] } temp = pc.data().from_dict(Ds).to_h5(out_file, group=f'spot_{spot}', replace=False) except Exception as e: print(e) pass
def read_data(file): groups = [ '10m', '200m', 'D_std_threshold_200m', 'block_std_threshold_200m' ] D = {group: pc.data().from_h5(file, group=group) for group in groups} return D