def make_composite_stats(k0_file, scat_file, impulse_file, res=40, \
t_window=20, dt_fine=0.05):
k0_field_dict={'location':['latitude','longitude', 'elevation'],
'both':['K0']}
catalog=setup(scat_file, impulse_file)
D_in=pc.data().from_h5(k0_file, field_dict=k0_field_dict)
D_in.get_xy(EPSG=3031)
# remove bogus points (lat=lon=0)
D_in.index(np.abs(D_in.x)>0)
pt_dict=pc.bin_rows(np.c_[np.round(D_in.x/res)*res, np.round(D_in.y/res)*res])
D_out=pc.data().from_dict(
{field:np.zeros(len(pt_dict))+np.NaN for field in \
['t_ctr', 't_med','t_sigma', 't_sigma_r','N','x','y', 'z_sigma', 'z_slope_mag']})
TX=resample_wf(catalog[0], dt_fine)
TX_mean, TX_med = composite_stats(TX, t_window)[0:2]
for ii, ctr in enumerate(pt_dict):
N=len(pt_dict[ctr])
if N < 10:
continue
WF0 = make_composite_wf(D_in.K0[pt_dict[ctr]], catalog)[0]
WF0 = resample_wf(WF0, 0.05)
D_out.t_ctr[ii], D_out.t_med[ii], D_out.t_sigma[ii], D_out.t_sigma_r[ii]=composite_stats(WF0, t_window)
D_out.z_sigma[ii], D_out.z_slope_mag[ii] = plane_fit_R(D_in, pt_dict[ctr])
D_out.x[ii] = ctr[0]
D_out.y[ii] = ctr[1]
D_out.N[ii] = len(pt_dict[ctr])
return D_out, TX_mean, TX_med
def get_xovers(self,invalid_to_nan=True):
rgt=self.attrs['ReferenceGroundTrack']
xo={'ref':{},'crossing':{},'both':{}}
n_cycles=self.ROOT.h_corr.shape[1]
zz=np.zeros(n_cycles)
for field in ['delta_time','h_corr','h_corr_sigma','h_corr_sigma_systematic', 'ref_pt','rgt','atl06_quality_summary','latitude','longitude','cycle_number','x_atc','y_atc','fit_quality']:
xo['ref'][field]=[]
xo['crossing'][field]=[]
#if field in ['delta_time','h_corr','h_corr_sigma','h_corr_sigma_systematic']:
# xo['ref'][field].append([])
# xo['ref'][field].append([])
xo['crossing']['along_track_rss']=[]
if hasattr(self,'x'):
for field in ['x','y']:
xo['ref'][field]=[]
for i1, ref_pt in enumerate(self.crossing_track_data.ref_pt):
i0=np.flatnonzero(self.ROOT.ref_pt==ref_pt)[0]
# fill vectors
for field in ['latitude','longitude']:
xo['ref'][field] += [getattr(self.ROOT, field)[i0]+zz]
for field in ['x_atc','y_atc']:
xo['ref'][field] += [getattr(self.ref_surf, field)[i0]+zz]
xo['ref']['ref_pt'] += [self.ROOT.ref_pt[i0]+zz]
xo['ref']['rgt'] += [rgt+zz]
xo['ref']['fit_quality'] += [self.ref_surf.fit_quality[i0]+zz]
for field in ['delta_time','h_corr','h_corr_sigma','h_corr_sigma_systematic', 'ref_pt','rgt','atl06_quality_summary', 'cycle_number', 'along_track_rss' ]:
xo['crossing'][field] += [getattr(self.crossing_track_data, field)[i1]+zz]
# fill vectors for each cycle
for field in ['delta_time', 'h_corr','h_corr_sigma','h_corr_sigma_systematic']: # vars that are N_pts x N_cycles
xo['ref'][field] += [getattr(self.ROOT, field)[i0,:]]
xo['ref']['atl06_quality_summary'] += [self.cycle_stats.atl06_summary_zero_count[i0,:] > 0]
xo['ref']['cycle_number'] += [getattr(self.ROOT,'cycle_number')]
if hasattr(self, 'x'):
for field in ['x','y']:
xo['ref'][field] += [getattr(self, field)[i0]+zz]
xo['crossing']['latitude']=xo['ref']['latitude']
xo['crossing']['longitude']=xo['ref']['longitude']
xo['crossing']['x_atc']=xo['ref']['x_atc']
xo['crossing']['y_atc']=xo['ref']['y_atc']
xo['crossing']['fit_quality'] = xo['ref']['fit_quality']
for field in xo['crossing']:
xo['crossing'][field]=np.concatenate(xo['crossing'][field], axis=0)
for field in xo['ref']:
xo['ref'][field]=np.concatenate(xo['ref'][field], axis=0)
ref=pc.data().from_dict(xo['ref'])
crossing=pc.data().from_dict(xo['crossing'])
delta={}
delta['h_corr']=crossing.h_corr-ref.h_corr
delta['delta_time']=crossing.delta_time-ref.delta_time
delta['h_corr_sigma']=np.sqrt(crossing.h_corr_sigma**2+ref.h_corr_sigma**2)
delta['latitude']=ref.latitude.copy()
delta['longitude']=ref.longitude.copy()
delta=pc.data().from_dict(delta)
return ref, crossing, delta
def save_L_interp(L_interps, file):
D_list = []
for L_interp in L_interps:
pt0 = np.array(list(L_interp.keys()))
D = {}
for field in L_interp[pt0[0]]:
D[field] = np.array([L_interp[pt][field] for pt in pt0])
D_list += [pc.data().from_dict(D)]
pc.data().from_list(D_list).to_h5(file)
def as_points(self, keep_all=False):
"""
Return a pointCollection.data object containing the points in the grid
"""
x,y=np.meshgrid(self.x, self.y)
if keep_all:
result = pc.data(filename=self.filename).\
from_dict({'x':x.ravel(),'y':y.ravel(),'z':self.z.ravel()})
else:
good=np.isfinite(self.z).ravel()
result = pc.data(filename=self.filename).\
from_dict({'x':x.ravel()[good],'y':y.ravel()[good],'z':self.z.ravel()[good]})
if self.time is not None:
result.assign({'time':self.time+np.zeros_like(self.z)})
return result
def read_CS2_data(xy0,
W,
index_files,
apply_filters=True,
DEM_file=None,
dem_tol=50):
if DEM_file is not None:
DEM=pc.grid.data().from_geotif(DEM_file, bounds=[xy0[0]+np.array([-W/2, W/2]), \
xy0[1]+np.array([-W/2, W/2])])
else:
DEM = None
D = []
D = [read_poca_data( xy0, W, file, apply_filters=apply_filters, DEM=DEM) \
for file in index_files['POCA']]
D += [ read_swath_data(xy0, W, file, apply_filters=apply_filters)\
for file in index_files['swath']]
D = pc.data().from_list(D)
if DEM_file is not None:
D.DEM = DEM.interp(D.x, D.y)
D.index(np.abs(D.h - D.DEM) < dem_tol)
if apply_filters:
with open(os.path.join(Path(__file__).parent.absolute(), \
'CS2_known_bad_orbits.json'), 'r') as fh:
bad_orbits = json.load(fh)[1] #['bad orbits']
D.index(~np.in1d(D.abs_orbit.astype(int),
np.array(bad_orbits, dtype='int')))
return D
def read_poca_data(xy0, W, index_file, apply_filters=True, DEM=None):
fields=['x','y','time','h', 'power','coherence','AD','error_composite', \
'ambiguity','abs_orbit','phase','range_surf']
D=pc.geoIndex().from_file(index_file).query_xy_box(xy0[0]+np.array([-W/2, W/2]), \
xy0[1]+np.array([-W/2, W/2]), fields=fields)
D = pc.data().from_list(D)
if D is None:
return D
D.time = matlab_to_year(D.time)
#D.assign({'burst':D.pulse_num,
D.assign({'swath': np.zeros_like(D.x, dtype=bool)})
if DEM is not None:
gx, gy = np.gradient(DEM.z, DEM.x, DEM.y)
temp = DEM.copy()
temp.z = np.abs(gx + 1j * gy)
DEM_slope_mag = temp.interp(D.x, D.y)
D.assign({
'sigma':
np.maximum(
0.5, 50 * DEM_slope_mag +
np.maximum(0, -0.64 * (np.log10(D.power) + 14)))
})
else:
# if no DEM is specified, use a typical value of 0.01 for the slope
D.assign({
'sigma':
50 * 0.01 + np.maximum(0, -0.64 * (np.log10(D.power) + 14))
})
if apply_filters:
D.index((D.power > 1e-16) & (D.error_composite == 0)
& (D.power < 1e-12))
return D
def read_swath_data(xy0, W, index_file, apply_filters=True, baseline_num=None):
fields = ['x','y','time','h', 'power','coherence',\
'R_POCA', 'ambiguity','abs_orbit', 'block_h_spread',\
'count','phase','R_offnadir','range_surf','seg_ind']
D=pc.geoIndex().from_file(index_file).query_xy_box(xy0[0]+np.array([-W/2, W/2]), \
xy0[1]+np.array([-W/2, W/2]), fields=fields)
if D is not None:
D = pc.data().from_list(D)
if D is None:
return D
if baseline_num is not None:
D.assign({'baseline': np.zeros_like(D.x) + baseline_num})
if baseline_num == 2:
D.power /= 2.
D.index(D.power > 0.)
D.time = matlab_to_year(D.time)
#D.assign({'burst':D.pulse_num,
D.assign({'swath': np.ones_like(D.x, dtype=bool)})
D.assign({'sigma':np.minimum(5, np.maximum(1, 0.95 -.4816*(np.log10(D.power)+14.3) +\
1.12*np.sqrt(D.block_h_spread)))})
if apply_filters:
if np.any(D.count > 1):
D.index( (D.power > 1e-17) & (D.power < 1e-13) & \
(D.count > 3) & (D.block_h_spread < 15))
else:
D.index((D.power > 1e-17) & (D.power < 1e-13))
return D
def __init__(self, D, map_file, index_file, datatype):
self.files=[]
self.datatype=datatype
self.map_file=map_file
self.index_file=index_file
print(f"TrackPicker: index_file is {index_file}")
srs_proj4=pc.geoIndex().from_file(index_file).attrs['SRS_proj4']
if datatype == 'ATL06':
for ii, Di in enumerate(D):
Di.get_xy(srs_proj4)
Di.assign({'file_num':np.zeros_like(Di.x)+ii})
self.files += [Di.filename]
self.D_all=pc.data().from_list(D)
elif datatype == 'ATL11':
D_list=[]
for ii, Di in enumerate(D):
self.files += [Di.filename]
Di.get_xy(srs_proj4)
D_list.append(pc.data().from_dict({
'x':Di.x, 'y':Di.y, 'file_num':np.zeros_like(Di.x)+ii,
'h_corr':Di.corrected_h.h_corr[:,-1].ravel()}))
self.D_all=pc.data().from_list(D_list)
else:
self.D_all=pc.data().from_list(D)
self.D=D
XR=[np.nanmin(self.D_all.x), np.nanmax(self.D_all.x)]
YR=[np.nanmin(self.D_all.y), np.nanmax(self.D_all.y)]
self.fig=plt.figure()
if map_file is not None:
get_map(map_file, bounds=[XR, YR])
#for Di in D:
# plt.plot(Di.x, Di.y,'.')
if self.datatype=='ATL06':
plt.scatter(self.D_all.x, self.D_all.y, 6, c=self.D_all.r_eff, linewidth=0, vmax=1, vmin=0)
elif self.datatype=='ATL11':
plt.scatter(self.D_all.x, self.D_all.y, 6, c=self.D_all.h_corr); plt.colorbar()
elif self.datatype=='ATM_waveform_fit':
plt.scatter(self.D_all.x, self.D_all.y, 6, c=np.log10(self.D_all.K0))
elif self.datatype=='CS2':
plt.scatter(self.D_all.x, self.D_all.y, 6, c=self.D_all.h); plt.colorbar()
hax=plt.axes([0.7, 0.05, 0.25, 0.25])
hax.hist((self.D_all.time-730486)/365.25+2000, 100)
self.cid=self.fig.canvas.mpl_connect('button_press_event', self)
plt.show()
def main():
# test code
thefile='Volumes/ice2/ben/ATL14_test/Jako_d2zdt2=5000_d3z=0.00001_d2zdt2=1500_RACMO//centers/E480_N-1200.h5'
D=pc.data().from_h5(thefile, group='/data/')
D.index(D.sensor>=4)
xy0=np.array([ 480000., -1200000.])
best_sensors=subset_DEM_stack(D, xy0, 4.e4)
def calc_cell_area(grid):
xg, yg = np.meshgrid(grid.ctrs[1], grid.ctrs[0])
if grid.srs_proj4 is not None:
lat = pc.data().from_dict({
'x': xg,
'y': yg
}).get_latlon(proj4_string=grid.srs_proj4).latitude
return pc.ps_scale_for_lat(lat)**2 * grid.delta[0] * grid.delta[1]
else:
return np.ones(grid.shape[0:2]) * grid.delta[0] * grid.delta[1]
def main():
import glob
#gI_file='/Volumes/insar10/ben/IS2_tiles/GL/GeoIndex.h5'
gI_file=glob.glob('/Volumes/ice2/ben/scf/GL_06/latest/tiles/*/GeoIndex.h5')[0]
xy0=[-170000.0, -2280000.0]
dI=pc.data().from_list(read_ICESat2(xy0, {'x':4.e4, 'y':4.e4}, gI_file, cplx_accept_threshold=0.25))
import matplotlib.pyplot as plt
plt.scatter(dI.x, dI.y, c=dI.z, linewidth=0); plt.colorbar()
plt.axis('equal')
def main():
import matplotlib.pyplot as plt
import pointCollection as pc
t=np.arange(2002, 2022, 0.1)
xy_jako=(-170000, -2280000)
corrections = ['RACMO_fac', 'RACMO_fac_smb', 'RACMO']
for correction in corrections:
data=pc.data().from_dict({'time':t, 'x':np.ones_like(t)*xy_jako[0], 'y':np.ones_like(t)*xy_jako[1], 'z':np.zeros_like(t)})
assign_firn_correction(data, correction, 1)
plt.plot(data.time, data.z)
plt.legend(corrections)
def apply_bin_fn(D_pt, res, fn=None, field='z'):
pt_dict = bin_rows(np.c_[np.round(D_pt.x / res) * res,
np.round(D_pt.y / res) * res])
keys = list(pt_dict.keys())
z = np.array([fn(D_pt, pt_dict[key]) for key in keys])
N = np.array([len(pt_dict[key]) for key in keys])
keys = np.array(keys)
return pc.data(filename=D_pt.filename)\
.from_dict({'x':keys[:,0], 'y':keys[:,1], field:z,'count':N})
def main():
W = {'x': 1.e4, 'y': 200, 't': 2}
x = np.arange(-W['x'] / 2, W['x'] / 2, 100)
D=pc.data().from_dict({'x':x, 'y':np.zeros_like(x),'z':np.sin(2*np.pi*x/2000.),\
'time':np.zeros_like(x)-0.5, 'sigma':np.zeros_like(x)+0.1})
D1 = D
D1.t = np.ones_like(x)
data = pc.data().from_list(
[D, D.copy().assign({'time': np.zeros_like(x) + 0.5})])
E_d3zdx2dt = 0.0001
E_d2z0dx2 = 0.006
E_d2zdt2 = 5
E_RMS = {
'd2z0_dx2': E_d2z0dx2,
'dz0_dx': E_d2z0dx2 * 1000,
'd3z_dx2dt': E_d3zdx2dt,
'd2z_dxdt': E_d3zdx2dt * 1000,
'd2z_dt2': E_d2zdt2
}
ctr = {'x': 0., 'y': 0., 't': 0.}
SRS_proj4 = '+proj=stere +lat_0=90 +lat_ts=70 +lon_0=-45 +k=1 +x_0=0 +y_0=0 +datum=WGS84 +units=m +no_defs '
spacing = {'z0': 50, 'dz': 50, 'dt': 0.25}
S = smooth_xytb_fit(data=data,
ctr=ctr,
W=W,
spacing=spacing,
E_RMS=E_RMS,
reference_epoch=2,
N_subset=None,
compute_E=False,
max_iterations=2,
srs_proj4=SRS_proj4,
VERBOSE=True,
dzdt_lags=[1])
return S
def write(self,
out_dir,
fields=None,
append=True,
ind_fields=['x', 'y', 'time']):
if self.D is None:
return
out_file = out_dir + '/E%d_N%d.h5' % (self.xy0[0], self.xy0[1])
n_source_files = 0
if fields is None:
field_dict = self.__default_field_dict__()
fields = []
for key in field_dict.keys():
fields += field_dict[key]
if os.path.isfile(out_file):
with h5py.File(out_file, 'r') as h5f:
if 'source_files' in h5f:
grp = h5f['source_files']
n_source_files = len(h5f['source_files'].attrs.keys())
file_dict = {}
for file_num, Di in enumerate(self.D):
if not hasattr(Di, 'x'):
Di.get_xy(self.SRS_proj4)
Di.assign({
'source_file_num':
np.zeros_like(Di.x, dtype=int) + file_num + n_source_files
})
Di.ravel_fields()
Di.index(np.isfinite(getattr(Di, self.z_field)))
file_dict[file_num] = Di.filename
out_fields = list(set(fields + ['x', 'y', 'source_file_num']))
D_all = pc.data(fields=out_fields).from_list(self.D)
if D_all.size == 0:
return
pc.indexedH5.data(filename=out_file, bin_W=self.bin_W).to_file(D_all, \
out_file, time_field=self.time_field, append=append,\
ind_fields=ind_fields)
with h5py.File(out_file, 'r+') as h5f:
if 'source_files' in h5f:
grp = h5f['source_files']
else:
grp = h5f.create_group("source_files")
for key in file_dict:
grp.attrs['file_%d' % key] = file_dict[key]
def from_h5(self, filename, pair=None):
if pair is None:
pair = self.pair
else:
self.pair_name = f'pt{int(self.pair)}'
D_at = pc.ATL11.data().from_h5(filename, pair=pair)
D_xo = pc.data().from_h5(filename,
group=self.pair_name + '/crossing_track_data',
field_dict=self.__default_XO_field_dict__())
D_xo.index(np.isfinite(D_xo.ref_pt) & np.isfinite(D_xo.cycle_number))
with h5py.File(filename, 'r') as h5f:
rgt = int(h5f[self.pair_name].attrs['ReferenceGroundTrack'])
D_at.assign({'rgt': np.zeros_like(D_at.delta_time) + rgt})
n_cycles = D_at.cycle_number[-1, -1]
u_pt_xo = np.unique(D_xo.ref_pt)
D_at.index(np.in1d(D_at.ref_pt[:, 0], u_pt_xo))
theshape = (u_pt_xo.size, n_cycles, 2)
self.assign(
{field: np.zeros(theshape) + np.NaN
for field in D_xo.fields})
row = np.searchsorted(u_pt_xo, D_at.ref_pt.ravel())
col = D_at.cycle_number.ravel().astype(int) - 1
for field in self.fields:
if field not in D_at.fields:
continue
getattr(self, field).flat[np.ravel_multi_index(
(row, col, np.zeros_like(row, dtype=int)),
theshape)] = getattr(D_at, field).ravel()
row = np.searchsorted(u_pt_xo, D_xo.ref_pt)
col = D_xo.cycle_number.astype(int) - 1
for field in self.fields:
getattr(self, field).flat[np.ravel_multi_index(
(row, col, 1 + np.zeros_like(row, dtype=int)),
theshape)] = getattr(D_xo, field)
self.__update_size_and_shape__()
return self
def make_plot(k0_file, scat_file, impulse_file, xy0=None, res=40, t_window=20):
#res_test(impulse_file)
k0_field_dict={'location':['latitude','longitude'],
'both':['K0']}
catalog=setup(scat_file, impulse_file)
D_in=pc.data().from_h5(k0_file, field_dict=k0_field_dict)
D_in.get_xy(EPSG=3031)
pt_dict=pc.bin_rows(np.c_[np.round(D_in.x/res)*res, np.round(D_in.y/res)*res])
xy_fits=np.concatenate([np.array(key).reshape([1,2]) for key in pt_dict], axis=0)
best=np.argmin(np.abs((xy_fits[:,0]-xy0[0])+1j*(xy_fits[:,1]-xy0[1])))
WF_composite, WFs=make_composite_wf(D_in.K0[pt_dict[tuple(xy_fits[best,:])]], catalog)
TX=resample_wf(catalog[0], 0.05)
WF_composite=resample_wf(WF_composite, 0.05)
bar0, med0, sigma0, sigmar_0=composite_stats(TX, t_window)
barc, medc, sigmac, sigmar_c=composite_stats(WF_composite, t_window)
TX_corr_mean_c, TX_corr_med_c=TX_corr(TX, np.max([6*sigmar_c, t_window]), sigmar_c, TX_sigma=sigma0)
TX_corr_mean_0, TX_corr_med_0=TX_corr(TX, np.max([6*sigmar_0, t_window]), sigmar_0, TX_sigma=sigma0)
plt.figure()
z0 = TX.centroid()*-0.15
for WF in WFs:
#WFi = resample_wf(WF, 0.01)
plt.plot(WF.p, WF.t*-.15-z0, linewidth=0.5, color='gray')
plt.plot(TX.p, TX.t*-.15-z0, 'k', linewidth=2)
plt.plot(WF_composite.p.ravel(), WF_composite.t.ravel()*-.15-z0, 'b', linewidth=2)
plt.gca().axhline(-bar0*.15-z0, color='k', linewidth=2, label='TX mean')
plt.gca().axhline(-barc*.15-z0, color='b', linewidth=2, label='RX mean')
plt.gca().axhline(-med0*.15-z0, linestyle='--', color='k', linewidth=2, label='TX med')
plt.gca().axhline(-medc*.15-z0, linestyle='--', color='b', linewidth=2, label='RX med')
plt.legend()
plt.xlabel('power')
plt.ylabel('elevation WRT surface, m')
plt.show()
def make_test_data():
'''
make test data for CS2 inversions
inputs: None (reads test_CS2_data.h5)
outputs:
z: simulated elevation data
bounds: a dict containing the range of the x, y, and time data
z_func: a function that gives heights as a function of x, y, time and
(binary) swath
'''
data = pc.data().from_h5('CS2_test_data.h5', group='/')
data.time = matlab_to_year(data.time)
XR = np.array(
[np.floor(data.x.min() / 1.e4),
np.ceil(data.x.max()) / 1.e4]) * 1.e4
YR = np.array(
[np.floor(data.y.min() / 1.e4),
np.ceil(data.y.max()) / 1.e4]) * 1.e4
TR = np.array([np.floor(data.time.min()), np.ceil(data.time.max())])
def z_func(x, y, t, swath, XR, YR, TR):
if swath is None:
swath = np.zeros_like(x, dtype=bool)
dem = (x - XR[0]) * 0.05 + 20 * np.sin(2 * np.pi * (y - YR[0]) /
(np.diff(YR) / 10))
t0 = TR[0] + np.array([0, 1 / 3, 2 / 3, 1]) * np.diff(TR)
t_func = 3 * np.interp(t, t0, np.array([0, 0, 1, 1]))
delta_z = 2 * t_func * np.sin(2 * np.pi * (x - XR[0]) /
(np.diff(XR) / 4))
swath_bias = 1.5 + np.exp(-((x - np.mean(XR))**2 +
(y - np.mean(YR))**2) / 2 /
(np.diff(XR) / 3)**2)
return dem + delta_z + swath_bias * swath
return data, z_func(data.x, data.y, data.time, data.swath, XR, YR,
TR), [XR, YR, TR], z_func
def append_SMB_ATL11(input_file, base_dir, REGION, MODEL):
# read input file
field_dict = {None: ('delta_time', 'h_corr', 'x', 'y')}
D11 = pc.data().from_h5(input_file, field_dict=field_dict)
# check if running crossover or along-track ATL11
if (D11.h_corr.ndim == 3):
nseg, ncycle, ncross = D11.shape
else:
nseg, ncycle = D11.shape
# get projection of input coordinates
EPSG = set_projection(REGION)
# available models
models = dict(AA={}, GL={})
# MAR
models['GL']['MAR'] = []
# models['GL']['MAR'].append('MARv3.9-ERA')
# models['GL']['MAR'].append('MARv3.10-ERA')
# models['GL']['MAR'].append('MARv3.11-NCEP')
models['GL']['MAR'].append('MARv3.11-ERA')
models['GL']['MAR'].append('MARv3.11.2-ERA-6km')
models['GL']['MAR'].append('MARv3.11.2-ERA-7.5km')
models['GL']['MAR'].append('MARv3.11.2-ERA-10km')
models['GL']['MAR'].append('MARv3.11.2-ERA-15km')
models['GL']['MAR'].append('MARv3.11.2-ERA-20km')
models['GL']['MAR'].append('MARv3.11.2-NCEP-20km')
# RACMO
models['GL']['RACMO'] = []
# models['GL']['RACMO'].append('RACMO2.3-XGRN11')
# models['GL']['RACMO'].append('RACMO2.3p2-XGRN11')
models['GL']['RACMO'].append('RACMO2.3p2-FGRN055')
# MERRA2-hybrid
models['GL']['MERRA2-hybrid'] = []
# models['GL']['MERRA2-hybrid'].append('GSFC-fdm-v0')
# models['GL']['MERRA2-hybrid'].append('GSFC-fdm-v1')
models['GL']['MERRA2-hybrid'].append('GSFC-fdm-v1.1')
models['AA']['MERRA2-hybrid'] = []
# models['AA']['MERRA2-hybrid'].append('GSFC-fdm-v0')
models['AA']['MERRA2-hybrid'].append('GSFC-fdm-v1')
for model_version in models[REGION][MODEL]:
if (MODEL == 'MAR'):
match_object = re.match(r'(MARv\d+\.\d+(.\d+)?)', model_version)
MAR_VERSION = match_object.group(0)
MAR_REGION = dict(GL='Greenland', AA='Antarctic')[REGION]
# model subdirectories
SUBDIRECTORY = dict(AA={}, GL={})
SUBDIRECTORY['GL']['MARv3.9-ERA'] = [
'ERA_1958-2018_10km', 'daily_10km'
]
SUBDIRECTORY['GL']['MARv3.10-ERA'] = [
'ERA_1958-2019-15km', 'daily_15km'
]
SUBDIRECTORY['GL']['MARv3.11-NCEP'] = [
'NCEP1_1948-2020_20km', 'daily_20km'
]
SUBDIRECTORY['GL']['MARv3.11-ERA'] = [
'ERA_1958-2019-15km', 'daily_15km'
]
SUBDIRECTORY['GL']['MARv3.11.2-ERA-6km'] = ['6km_ERA5']
SUBDIRECTORY['GL']['MARv3.11.2-ERA-7.5km'] = ['7.5km_ERA5']
SUBDIRECTORY['GL']['MARv3.11.2-ERA-10km'] = ['10km_ERA5']
SUBDIRECTORY['GL']['MARv3.11.2-ERA-15km'] = ['15km_ERA5']
SUBDIRECTORY['GL']['MARv3.11.2-ERA-20km'] = ['20km_ERA5']
SUBDIRECTORY['GL']['MARv3.11.2-NCEP-20km'] = ['20km_NCEP1']
MAR_MODEL = SUBDIRECTORY[REGION][model_version]
DIRECTORY = os.path.join(base_dir, 'MAR', MAR_VERSION, MAR_REGION,
*MAR_MODEL)
# variable coordinates
KWARGS = dict(AA={}, GL={})
KWARGS['GL']['MARv3.9-ERA'] = dict(XNAME='X10_153',
YNAME='Y21_288')
KWARGS['GL']['MARv3.10-ERA'] = dict(XNAME='X10_105',
YNAME='Y21_199')
KWARGS['GL']['MARv3.11-NCEP'] = dict(XNAME='X12_84',
YNAME='Y21_155')
KWARGS['GL']['MARv3.11-ERA'] = dict(XNAME='X10_105',
YNAME='Y21_199')
KWARGS['GL']['MARv3.11.2-ERA-6km'] = dict(XNAME='X12_251',
YNAME='Y20_465')
KWARGS['GL']['MARv3.11.2-ERA-7.5km'] = dict(XNAME='X12_203',
YNAME='Y20_377')
KWARGS['GL']['MARv3.11.2-ERA-10km'] = dict(XNAME='X10_153',
YNAME='Y21_288')
KWARGS['GL']['MARv3.11.2-ERA-15km'] = dict(XNAME='X10_105',
YNAME='Y21_199')
KWARGS['GL']['MARv3.11.2-ERA-20km'] = dict(XNAME='X12_84',
YNAME='Y21_155')
KWARGS['GL']['MARv3.11.2-NCEP-20km'] = dict(XNAME='X12_84',
YNAME='Y21_155')
MAR_KWARGS = KWARGS[REGION][model_version]
# output variable keys for both direct and derived fields
KEYS = ['zsurf', 'zfirn', 'zmelt', 'zsmb', 'zaccum']
# HDF5 longname attributes for each variable
LONGNAME = {}
LONGNAME['zsurf'] = "Snow Height Change"
LONGNAME['zfirn'] = "Snow Height Change due to Compaction"
LONGNAME['zmelt'] = "Snow Height Change due to Surface Melt"
LONGNAME['zsmb'] = "Snow Height Change due to Surface Mass Balance"
LONGNAME[
'zaccum'] = "Snow Height Change due to Surface Accumulation"
elif (MODEL == 'RACMO'):
RACMO_VERSION, RACMO_MODEL = model_version.split('-')
# output variable keys
KEYS = ['zsurf']
# HDF5 longname attributes for each variable
LONGNAME = {}
LONGNAME['zsurf'] = "Snow Height Change"
elif (MODEL == 'MERRA2-hybrid'):
merra2_regex = re.compile(r'GSFC-fdm-((v\d+)(\.\d+)?)$')
# get MERRA-2 version and major version
MERRA2_VERSION = merra2_regex.match(model_version).group(1)
MERRA2_MAJOR_VERSION = merra2_regex.match(model_version).group(2)
# MERRA-2 hybrid directory
DIRECTORY = os.path.join(base_dir, 'MERRA2_hybrid', MERRA2_VERSION)
MERRA2_REGION = dict(AA='ais', GL='gris')[REGION]
# output variable keys for both direct and derived fields
KEYS = ['zsurf', 'zfirn', 'zsmb']
# HDF5 longname attributes for each variable
LONGNAME = {}
LONGNAME['zsurf'] = "Snow Height Change"
LONGNAME['zfirn'] = "Snow Height Change due to Compaction"
LONGNAME['zsmb'] = "Snow Height Change due to Surface Mass Balance"
# check if running crossover or along track
if (D11.h_corr.ndim == 3):
# allocate for output height for crossover data
OUTPUT = {}
for key in KEYS:
OUTPUT[key] = np.ma.zeros((nseg, ncycle, ncross),
fill_value=np.nan)
OUTPUT[key].mask = np.ones((nseg, ncycle, ncross),
dtype=np.bool)
OUTPUT[key].interpolation = np.zeros((nseg, ncycle, ncross),
dtype=np.uint8)
# for each cycle of ICESat-2 ATL11 data
for c in range(ncycle):
# check that there are valid crossovers
cross = [
xo for xo in range(ncross)
if np.any(np.isfinite(D11.delta_time[:, c, xo]))
]
# for each valid crossing
for xo in cross:
# find valid crossovers
i, = np.nonzero(np.isfinite(D11.delta_time[:, c, xo]))
# convert from delta time to decimal-years
tdec = convert_delta_time(D11.delta_time[i, c,
xo])['decimal']
if (MODEL == 'MAR'):
# read and interpolate daily MAR outputs
ZN4 = SMBcorr.interpolate_mar_daily(DIRECTORY,
EPSG,
MAR_VERSION,
tdec,
D11.x[i, c, xo],
D11.y[i, c, xo],
VARIABLE='ZN4',
SIGMA=1.5,
FILL_VALUE=np.nan,
**MAR_KWARGS)
ZN5 = SMBcorr.interpolate_mar_daily(DIRECTORY,
EPSG,
MAR_VERSION,
tdec,
D11.x[i, c, xo],
D11.y[i, c, xo],
VARIABLE='ZN5',
SIGMA=1.5,
FILL_VALUE=np.nan,
**MAR_KWARGS)
ZN6 = SMBcorr.interpolate_mar_daily(DIRECTORY,
EPSG,
MAR_VERSION,
tdec,
D11.x[i, c, xo],
D11.y[i, c, xo],
VARIABLE='ZN6',
SIGMA=1.5,
FILL_VALUE=np.nan,
**MAR_KWARGS)
# set attributes to output for iteration
OUTPUT['zfirn'].data[i, c, xo] = np.copy(ZN4.data)
OUTPUT['zfirn'].mask[i, c, xo] = np.copy(ZN4.mask)
OUTPUT['zfirn'].interpolation[i, c, xo] = np.copy(
ZN4.interpolation)
OUTPUT['zsurf'].data[i, c, xo] = np.copy(ZN6.data)
OUTPUT['zsurf'].mask[i, c, xo] = np.copy(ZN6.mask)
OUTPUT['zsurf'].interpolation[i, c, xo] = np.copy(
ZN6.interpolation)
OUTPUT['zmelt'].data[i, c, xo] = np.copy(ZN5.data)
OUTPUT['zmelt'].mask[i, c, xo] = np.copy(ZN5.mask)
OUTPUT['zmelt'].interpolation[i, c, xo] = np.copy(
ZN5.interpolation)
# calculate derived fields
OUTPUT['zsmb'].data[i, c, xo] = ZN6.data - ZN4.data
OUTPUT['zsmb'].mask[i, c, xo] = ZN4.mask | ZN6.mask
OUTPUT['zaccum'].data[
i, c, xo] = ZN6.data - ZN4.data - ZN5.data
OUTPUT['zaccum'].mask[
i, c, xo] = ZN4.mask | ZN5.mask | ZN6.mask
elif (MODEL == 'RACMO'):
# read and interpolate daily RACMO outputs
hgtsrf = SMBcorr.interpolate_racmo_daily(
base_dir,
EPSG,
RACMO_MODEL,
tdec,
D11.x[i, c, xo],
D11.y[i, c, xo],
VARIABLE='hgtsrf',
SIGMA=1.5,
FILL_VALUE=np.nan)
# set attributes to output for iteration
OUTPUT['zsurf'].data[i, c, xo] = np.copy(hgtsrf.data)
OUTPUT['zsurf'].mask[i, c, xo] = np.copy(hgtsrf.mask)
OUTPUT['zsurf'].interpolation[i, c, xo] = np.copy(
hgtsrf.interpolation)
elif (MODEL == 'MERRA2-hybrid'):
# read and interpolate 5-day MERRA2-Hybrid outputs
FAC = SMBcorr.interpolate_merra_hybrid(
DIRECTORY,
EPSG,
MERRA2_REGION,
tdec,
D11.x[i, c, xo],
D11.y[i, c, xo],
VERSION=MERRA2_MAJOR_VERSION,
VARIABLE='FAC',
SIGMA=1.5,
FILL_VALUE=np.nan)
smb = SMBcorr.interpolate_merra_hybrid(
DIRECTORY,
EPSG,
MERRA2_REGION,
tdec,
D11.x[i, c, xo],
D11.y[i, c, xo],
VERSION=MERRA2_MAJOR_VERSION,
VARIABLE='cum_smb_anomaly',
SIGMA=1.5,
FILL_VALUE=np.nan)
height = SMBcorr.interpolate_merra_hybrid(
DIRECTORY,
EPSG,
MERRA2_REGION,
tdec,
D11.x[i, c, xo],
D11.y[i, c, xo],
VERSION=MERRA2_MAJOR_VERSION,
VARIABLE='height',
SIGMA=1.5,
FILL_VALUE=np.nan)
# set attributes to output for iteration
OUTPUT['zfirn'].data[i, c, xo] = np.copy(FAC.data)
OUTPUT['zfirn'].mask[i, c, xo] = np.copy(FAC.mask)
OUTPUT['zfirn'].interpolation[i, c, xo] = np.copy(
FAC.interpolation)
OUTPUT['zsurf'].data[i, c, xo] = np.copy(height.data)
OUTPUT['zsurf'].mask[i, c, xo] = np.copy(height.mask)
OUTPUT['zsurf'].interpolation[i, c, xo] = np.copy(
height.interpolation)
OUTPUT['zsmb'].data[i, c, xo] = np.copy(smb.data)
OUTPUT['zsmb'].mask[i, c, xo] = np.copy(smb.mask)
OUTPUT['zsmb'].interpolation[i, c, xo] = np.copy(
smb.interpolation)
else:
# allocate for output height for along-track data
OUTPUT = {}
for key in KEYS:
OUTPUT[key] = np.ma.zeros((nseg, ncycle), fill_value=np.nan)
OUTPUT[key].mask = np.ones((nseg, ncycle), dtype=np.bool)
OUTPUT[key].interpolation = np.zeros((nseg, ncycle),
dtype=np.uint8)
# check that there are valid elevations
cycle = [
c for c in range(ncycle)
if np.any(np.isfinite(D11.delta_time[:, c]))
]
# for each valid cycle of ICESat-2 ATL11 data
for c in cycle:
# find valid elevations
i, = np.nonzero(np.isfinite(D11.delta_time[:, c]))
# convert from delta time to decimal-years
tdec = convert_delta_time(D11.delta_time[i, c])['decimal']
if (MODEL == 'MAR'):
# read and interpolate daily MAR outputs
ZN4 = SMBcorr.interpolate_mar_daily(DIRECTORY,
EPSG,
MAR_VERSION,
tdec,
D11.x[i, c],
D11.y[i, c],
VARIABLE='ZN4',
SIGMA=1.5,
FILL_VALUE=np.nan,
**MAR_KWARGS)
ZN5 = SMBcorr.interpolate_mar_daily(DIRECTORY,
EPSG,
MAR_VERSION,
tdec,
D11.x[i, c],
D11.y[i, c],
VARIABLE='ZN5',
SIGMA=1.5,
FILL_VALUE=np.nan,
**MAR_KWARGS)
ZN6 = SMBcorr.interpolate_mar_daily(DIRECTORY,
EPSG,
MAR_VERSION,
tdec,
D11.x[i, c],
D11.y[i, c],
VARIABLE='ZN6',
SIGMA=1.5,
FILL_VALUE=np.nan,
**MAR_KWARGS)
# set attributes to output for iteration
OUTPUT['zfirn'].data[i, c] = np.copy(ZN4.data)
OUTPUT['zfirn'].mask[i, c] = np.copy(ZN4.mask)
OUTPUT['zfirn'].interpolation[i, c] = np.copy(
ZN4.interpolation)
OUTPUT['zsurf'].data[i, c] = np.copy(ZN6.data)
OUTPUT['zsurf'].mask[i, c] = np.copy(ZN6.mask)
OUTPUT['zsurf'].interpolation[i, c] = np.copy(
ZN6.interpolation)
OUTPUT['zmelt'].data[i, c] = np.copy(ZN5.data)
OUTPUT['zmelt'].mask[i, c] = np.copy(ZN5.mask)
OUTPUT['zmelt'].interpolation[i, c] = np.copy(
ZN5.interpolation)
# calculate derived fields
OUTPUT['zsmb'].data[i, c] = ZN6.data - ZN4.data
OUTPUT['zsmb'].mask[i, c] = ZN4.mask | ZN6.mask
OUTPUT['zaccum'].data[i,
c] = ZN6.data - ZN4.data - ZN5.data
OUTPUT['zaccum'].mask[i,
c] = ZN4.mask | ZN5.mask | ZN6.mask
elif (MODEL == 'RACMO'):
# read and interpolate daily RACMO outputs
hgtsrf = SMBcorr.interpolate_racmo_daily(base_dir,
EPSG,
RACMO_MODEL,
tdec,
D11.x[i, c],
D11.y[i, c],
VARIABLE='hgtsrf',
SIGMA=1.5,
FILL_VALUE=np.nan)
# set attributes to output for iteration
OUTPUT['zsurf'].data[i, c] = np.copy(hgtsrf.data)
OUTPUT['zsurf'].mask[i, c] = np.copy(hgtsrf.mask)
OUTPUT['zsurf'].interpolation[i, c] = np.copy(
hgtsrf.interpolation)
elif (MODEL == 'MERRA2-hybrid'):
# read and interpolate 5-day MERRA2-Hybrid outputs
FAC = SMBcorr.interpolate_merra_hybrid(
DIRECTORY,
EPSG,
MERRA2_REGION,
tdec,
D11.x[i, c],
D11.y[i, c],
VERSION=MERRA2_MAJOR_VERSION,
VARIABLE='FAC',
SIGMA=1.5,
FILL_VALUE=np.nan)
smb = SMBcorr.interpolate_merra_hybrid(
DIRECTORY,
EPSG,
MERRA2_REGION,
tdec,
D11.x[i, c],
D11.y[i, c],
VERSION=MERRA2_MAJOR_VERSION,
VARIABLE='cum_smb_anomaly',
SIGMA=1.5,
FILL_VALUE=np.nan)
height = SMBcorr.interpolate_merra_hybrid(
DIRECTORY,
EPSG,
MERRA2_REGION,
tdec,
D11.x[i, c],
D11.y[i, c],
VERSION=MERRA2_MAJOR_VERSION,
VARIABLE='height',
SIGMA=1.5,
FILL_VALUE=np.nan)
# set attributes to output for iteration
OUTPUT['zfirn'].data[i, c] = np.copy(FAC.data)
OUTPUT['zfirn'].mask[i, c] = np.copy(FAC.mask)
OUTPUT['zfirn'].interpolation[i, c] = np.copy(
FAC.interpolation)
OUTPUT['zsurf'].data[i, c] = np.copy(height.data)
OUTPUT['zsurf'].mask[i, c] = np.copy(height.mask)
OUTPUT['zsurf'].interpolation[i, c] = np.copy(
height.interpolation)
OUTPUT['zsmb'].data[i, c] = np.copy(smb.data)
OUTPUT['zsmb'].mask[i, c] = np.copy(smb.mask)
OUTPUT['zsmb'].interpolation[i, c] = np.copy(
smb.interpolation)
# append input HDF5 file with new firn model outputs
fileID = h5py.File(os.path.expanduser(input_file), 'a')
fileID.create_group(model_version)
h5 = {}
for key in KEYS:
# verify mask values
OUTPUT[key].mask |= (OUTPUT[key].data == OUTPUT[key].fill_value) | \
np.isnan(OUTPUT[key].data)
OUTPUT[key].data[OUTPUT[key].mask] = OUTPUT[key].fill_value
# output variable to HDF5
val = '{0}/{1}'.format(model_version, key)
h5[key] = fileID.create_dataset(val,
OUTPUT[key].shape,
data=OUTPUT[key],
dtype=OUTPUT[key].dtype,
compression='gzip',
fillvalue=OUTPUT[key].fill_value)
h5[key].attrs['units'] = "m"
h5[key].attrs['long_name'] = LONGNAME[key]
h5[key].attrs[
'coordinates'] = "../delta_time ../latitude ../longitude"
h5[key].attrs['model'] = model_version
# close the output HDF5 file
fileID.close()
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Jan 21 15:56:39 2020
@author: ben
"""
import pointCollection as pc
import numpy as np
import sys
import matplotlib.pyplot as plt
plt.figure()
for index_file in sys.argv[1:]:
xy = pc.geoIndex().from_file(index_file).bins_as_array()
plt.plot(xy[0], xy[1], 'o', label=index_file)
D = pc.geoIndex().from_file(index_file).query_xy(
[[xy[0][0]], [xy[1][0]]],
get_data=True,
fields=['x', 'y', 'h_li', 'pair', 'cycle_number'])
D = pc.data().from_list(D)
plt.plot(D.x, D.y, '.', label='first bin of ' + index_file)
plt.axis('equal')
plt.legend()
plt.show()
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--gps', '-G', type=str, help="GPS file to run")
parser.add_argument('--atm', '-A', type=str, help='ATM directory to run')
parser.add_argument('--hemisphere',
'-H',
type=int,
default=-1,
help='hemisphere, must be 1 or -1')
parser.add_argument('--query',
'-Q',
type=float,
default=100,
help='KD-Tree query radius')
parser.add_argument('--median',
'-M',
default=False,
action='store_true',
help='Run block median')
parser.add_argument('--scan',
'-S',
default=False,
action='store_true',
help='Run ATM scan fit')
parser.add_argument('--verbose',
'-v',
default=False,
action='store_true',
help='verbose output of run')
args = parser.parse_args()
if args.hemisphere == 1:
SRS_proj4 = '+proj=stere +lat_0=90 +lat_ts=70 +lon_0=-45 +k=1 +x_0=0 +y_0=0 +datum=WGS84 +units=m +no_defs '
elif args.hemisphere == -1:
SRS_proj4 = '+proj=stere +lat_0=-90 +lat_ts=-71 +lon_0=0 +k=1 +x_0=0 +y_0=0 +datum=WGS84 +units=m +no_defs'
# tilde expansion of file arguments
GPS_file = os.path.expanduser(args.gps)
fileBasename, fileExtension = os.path.splitext(GPS_file)
ATM_dir = os.path.expanduser(args.atm)
print("working on GPS file {0}, ATM directory {1}".format(
GPS_file, ATM_dir)) if args.verbose else None
# find Qfit files within ATM_dir
Qfit_regex = re.compile(
r"ATM1B.*_(\d{4})(\d{2})(\d{2})_(\d{2})(\d{2})(\d{2}).*.h5$")
Qfit_files = [
os.path.join(ATM_dir, f) for f in os.listdir(ATM_dir)
if Qfit_regex.search(f)
]
# output directory
out_dir = os.path.join(ATM_dir, 'xovers')
if not os.path.isdir(out_dir):
os.mkdir(out_dir)
# output file
out_file = 'vs_{0}.h5'.format(os.path.basename(fileBasename))
# check if output file exists
if os.path.isfile(os.path.join(out_dir, out_file)):
print("found: {0}".format(os.path.join(
out_dir, out_file))) if args.verbose else None
# read GPS HDF5 file
GPS_field_dict = {None: ['latitude', 'longitude', 'z']}
GPS = pc.data().from_h5(GPS_file,
field_dict=GPS_field_dict).get_xy(SRS_proj4)
# run block median over GPS data
if args.median:
GPS = blockmedian_for_gps(GPS, 5)
# read all Qfit files within ATM directory
Qlist = list()
for f in sorted(Qfit_files):
Qlist.append(pc.ATM_Qfit.data().from_h5(f))
# merge the list of ATM data and build the search tree
Q_full = pc.data().from_list(Qlist).get_xy(SRS_proj4)
# fit scan parameters to an ATM data structure
if args.scan:
Q_full = fit_ATM_data(Q_full)
# run block median for qsub
if args.median:
Q_full = blockmedian_for_qsub(Q_full, 5)
# construct search tree from ATM Qfit coords
# pickle Qtree to save computational time for future runs
if os.path.isfile(os.path.join(ATM_dir, 'tree.p')):
Qtree = pickle.load(open(os.path.join(ATM_dir, 'tree.p'), 'rb'))
else:
Qtree = KDTree(np.c_[Q_full.x, Q_full.y])
pickle.dump(Qtree, open(os.path.join(ATM_dir, 'tree.p'), 'wb'))
# output fields
out_fields = [
'x', 'y', 'z', 'longitude', 'latitude', 't_qfit', 'h_qfit_50m',
'sigma_qfit_50m', 'dz_50m', 'RDE_50m', 'N_50m', 'hbar_20m',
'h_qfit_10m', 'sigma_qfit_10m', 'dz_10m', 'RDE_10m', 'N_10m',
'x_10m_mean', 'y_10m_mean'
]
# append scan fields to output template
if args.scan:
out_fields.extend(['scan_XT_50m', 'scan_XT_10m'])
out_template = {f: np.NaN for f in out_fields}
out = list()
# query the search tree to find points within query radius
Qquery = Qtree.query_radius(np.c_[GPS.x, GPS.y], args.query)
# indices of GPS points within bin
ind, = np.nonzero([np.any(i) for i in Qquery])
# loop over queries in the GPS data
for i in ind:
GPSsub = GPS.copy_subset(np.array([i]))
# grab the Qfit bins around the GPS bin
Qdata = Q_full.copy_subset(Qquery[i], by_row=True)
Qdata.index(
np.isfinite(Qdata.elevation) & np.isfinite(Qdata.latitude)
& np.isfinite(Qdata.longitude))
# create output dictionary of GPS and plane-fit ATM comparison
this_out = compare_gps_with_qfit(GPSsub, Qdata, out_template)
if this_out is not None:
out.append(this_out)
# if there were overlapping points between the GPS and ATM data
if out:
D = dict()
with h5py.File(os.path.join(out_dir, out_file), 'w') as h5f:
for field in out[0].keys():
D[field] = np.array([ii[field] for ii in out])
print(field, D[field].dtype) if args.verbose else None
h5f.create_dataset(field, data=D[field])
def append_SMB_averages_ATL11(input_file,
base_dir,
REGION,
MODEL,
RANGE=[2000, 2019],
VERBOSE=False):
# read input file
field_dict = {None: ('delta_time', 'h_corr', 'x', 'y')}
D11 = pc.data().from_h5(input_file, field_dict=field_dict)
# check if running crossover or along-track ATL11
if (D11.h_corr.ndim == 3):
nseg, ncycle, ncross = D11.shape
else:
nseg, ncycle = D11.shape
# get projection of input coordinates
EPSG = set_projection(REGION)
# available models
models = dict(AA={}, GL={})
# MAR
models['GL']['MAR'] = []
# models['GL']['MAR'].append('MARv3.9-ERA')
# models['GL']['MAR'].append('MARv3.10-ERA')
# models['GL']['MAR'].append('MARv3.11-NCEP')
# models['GL']['MAR'].append('MARv3.11-ERA')
# models['GL']['MAR'].append('MARv3.11.2-ERA-6km')
# models['GL']['MAR'].append('MARv3.11.2-ERA-7.5km')
# models['GL']['MAR'].append('MARv3.11.2-ERA-10km')
# models['GL']['MAR'].append('MARv3.11.2-ERA-15km')
# models['GL']['MAR'].append('MARv3.11.2-ERA-20km')
# models['GL']['MAR'].append('MARv3.11.2-NCEP-20km')
# models['GL']['MAR'].append('MARv3.11.5-ERA-6km')
models['GL']['MAR'].append('MARv3.11.5-ERA-10km')
models['GL']['MAR'].append('MARv3.11.5-ERA-15km')
models['GL']['MAR'].append('MARv3.11.5-ERA-20km')
# RACMO
models['GL']['RACMO'] = []
# models['GL']['RACMO'].append('RACMO2.3-XGRN11')
# models['GL']['RACMO'].append('RACMO2.3p2-XGRN11')
models['GL']['RACMO'].append('RACMO2.3p2-FGRN055')
# MERRA2-hybrid
models['GL']['MERRA2-hybrid'] = []
# models['GL']['MERRA2-hybrid'].append('GSFC-fdm-v0')
# models['GL']['MERRA2-hybrid'].append('GSFC-fdm-v1')
models['GL']['MERRA2-hybrid'].append('GSFC-fdm-v1.1')
models['AA']['MERRA2-hybrid'] = []
# models['AA']['MERRA2-hybrid'].append('GSFC-fdm-v0')
models['AA']['MERRA2-hybrid'].append('GSFC-fdm-v1')
models['AA']['MERRA2-hybrid'].append('GSFC-fdm-v1.1')
# for each model to append to ATL11
for model_version in models[REGION][MODEL]:
# keyword arguments for all models
# add range to input keyword arguments
KWARGS = dict(SIGMA=1.5, FILL_VALUE=np.nan, RANGE=RANGE)
if (MODEL == 'MAR'):
match_object = re.match(r'(MARv\d+\.\d+(.\d+)?)', model_version)
MAR_VERSION = match_object.group(0)
MAR_REGION = dict(GL='Greenland', AA='Antarctic')[REGION]
# model subdirectories
SUBDIRECTORY = dict(AA={}, GL={})
SUBDIRECTORY['GL']['MARv3.9-ERA'] = [
'ERA_1958-2018_10km', 'daily_10km'
]
SUBDIRECTORY['GL']['MARv3.10-ERA'] = [
'ERA_1958-2019-15km', 'daily_15km'
]
SUBDIRECTORY['GL']['MARv3.11-NCEP'] = [
'NCEP1_1948-2020_20km', 'daily_20km'
]
SUBDIRECTORY['GL']['MARv3.11-ERA'] = [
'ERA_1958-2019-15km', 'daily_15km'
]
SUBDIRECTORY['GL']['MARv3.11.2-ERA-6km'] = ['6km_ERA5']
SUBDIRECTORY['GL']['MARv3.11.2-ERA-7.5km'] = ['7.5km_ERA5']
SUBDIRECTORY['GL']['MARv3.11.2-ERA-10km'] = ['10km_ERA5']
SUBDIRECTORY['GL']['MARv3.11.2-ERA-15km'] = ['15km_ERA5']
SUBDIRECTORY['GL']['MARv3.11.2-ERA-20km'] = ['20km_ERA5']
SUBDIRECTORY['GL']['MARv3.11.2-NCEP-20km'] = ['20km_NCEP1']
SUBDIRECTORY['GL']['MARv3.11.5-ERA-6km'] = ['6km_ERA5']
SUBDIRECTORY['GL']['MARv3.11.5-ERA-10km'] = ['10km_ERA5']
SUBDIRECTORY['GL']['MARv3.11.5-ERA-15km'] = ['15km_ERA5']
SUBDIRECTORY['GL']['MARv3.11.5-ERA-20km'] = ['20km_ERA5']
MAR_MODEL = SUBDIRECTORY[REGION][model_version]
DIRECTORY = os.path.join(base_dir, 'MAR', MAR_VERSION, MAR_REGION,
*MAR_MODEL)
# keyword arguments for variable coordinates
MAR_KWARGS = dict(AA={}, GL={})
MAR_KWARGS['GL']['MARv3.9-ERA'] = dict(XNAME='X10_153',
YNAME='Y21_288')
MAR_KWARGS['GL']['MARv3.10-ERA'] = dict(XNAME='X10_105',
YNAME='Y21_199')
MAR_KWARGS['GL']['MARv3.11-NCEP'] = dict(XNAME='X12_84',
YNAME='Y21_155')
MAR_KWARGS['GL']['MARv3.11-ERA'] = dict(XNAME='X10_105',
YNAME='Y21_199')
MAR_KWARGS['GL']['MARv3.11.2-ERA-6km'] = dict(XNAME='X12_251',
YNAME='Y20_465')
MAR_KWARGS['GL']['MARv3.11.2-ERA-7.5km'] = dict(XNAME='X12_203',
YNAME='Y20_377')
MAR_KWARGS['GL']['MARv3.11.2-ERA-10km'] = dict(XNAME='X10_153',
YNAME='Y21_288')
MAR_KWARGS['GL']['MARv3.11.2-ERA-15km'] = dict(XNAME='X10_105',
YNAME='Y21_199')
MAR_KWARGS['GL']['MARv3.11.2-ERA-20km'] = dict(XNAME='X12_84',
YNAME='Y21_155')
MAR_KWARGS['GL']['MARv3.11.2-NCEP-20km'] = dict(XNAME='X12_84',
YNAME='Y21_155')
MAR_KWARGS['GL']['MARv3.11.5-ERA-6km'] = dict(XNAME='X12_251',
YNAME='Y20_465')
MAR_KWARGS['GL']['MARv3.11.5-ERA-10km'] = dict(XNAME='X10_153',
YNAME='Y21_288')
MAR_KWARGS['GL']['MARv3.11.5-ERA-15km'] = dict(XNAME='X10_105',
YNAME='Y21_199')
MAR_KWARGS['GL']['MARv3.11.5-ERA-20km'] = dict(XNAME='X12_84',
YNAME='Y21_155')
KWARGS.update(MAR_KWARGS[REGION][model_version])
# netCDF4 variable names for direct fields
VARIABLES = ['ZN6', 'ZN4', 'ZN5']
# output variable keys for both direct and derived fields
KEYS = [
'zsurf_ave', 'zfirn_ave', 'zmelt_ave', 'zsmb_ave', 'zaccum_ave'
]
# HDF5 longname attributes for each variable
LONGNAME = {}
LONGNAME['zsurf_ave'] = "Snow Height Change"
LONGNAME['zfirn_ave'] = "Snow Height Change due to Compaction"
LONGNAME['zmelt_ave'] = "Snow Height Change due to Surface Melt"
LONGNAME[
'zsmb_ave'] = "Snow Height Change due to Surface Mass Balance"
LONGNAME[
'zaccum_ave'] = "Snow Height Change due to Surface Accumulation"
elif (MODEL == 'RACMO'):
RACMO_VERSION, RACMO_MODEL = model_version.split('-')
# netCDF4 variable names
VARIABLES = ['hgtsrf']
# output variable keys
KEYS = ['zsurf_ave']
# HDF5 longname attributes for each variable
LONGNAME = {}
LONGNAME['zsurf_ave'] = "Snow Height Change"
elif (MODEL == 'MERRA2-hybrid'):
# regular expression pattern for extracting version
merra2_regex = re.compile(r'GSFC-fdm-((v\d+)(\.\d+)?)$')
# get MERRA-2 version and major version
MERRA2_VERSION = merra2_regex.match(model_version).group(1)
# MERRA-2 hybrid directory
DIRECTORY = os.path.join(base_dir, 'MERRA2_hybrid', MERRA2_VERSION)
# MERRA-2 region name from ATL11 region
MERRA2_REGION = dict(AA='ais', GL='gris')[REGION]
# keyword arguments for MERRA-2 interpolation programs
if MERRA2_VERSION in ('v0', 'v1', 'v1.0'):
KWARGS['VERSION'] = merra2_regex.match(model_version).group(2)
# netCDF4 variable names
VARIABLES = ['FAC', 'cum_smb_anomaly', 'height']
# add additional Greenland variables
if (MERRA2_REGION == 'gris'):
VARIABLES.append('runoff_anomaly')
else:
KWARGS['VERSION'] = MERRA2_VERSION.replace('.', '_')
# netCDF4 variable names
VARIABLES = ['FAC', 'SMB_a', 'h_a']
# add additional Greenland variables
if (MERRA2_REGION == 'gris'):
VARIABLES.append('Me_a')
# output variable keys
KEYS = ['zsurf_ave', 'zfirn_ave', 'zsmb_ave']
# HDF5 longname attributes for each variable
LONGNAME = {}
LONGNAME['zsurf_ave'] = "Snow Height Change"
LONGNAME['zfirn_ave'] = "Snow Height Change due to Compaction"
LONGNAME[
'zsmb_ave'] = "Snow Height Change due to Surface Mass Balance"
# check if running crossover or along track
if (D11.h_corr.ndim == 3):
# allocate for output height for crossover data
OUTPUT = {}
for key in KEYS:
OUTPUT[key] = np.ma.zeros((nseg, ncycle, ncross),
fill_value=np.nan)
OUTPUT[key].mask = np.ones((nseg, ncycle, ncross), dtype=bool)
# for each cycle of ICESat-2 ATL11 data
for c in range(ncycle):
# check that there are valid crossovers
cross = [
xo for xo in range(ncross)
if np.any(np.isfinite(D11.delta_time[:, c, xo]))
]
# for each valid crossing
for xo in cross:
# find valid crossovers
i, = np.nonzero(np.isfinite(D11.delta_time[:, c, xo]))
# convert from delta time to decimal-years
tdec = convert_delta_time(D11.delta_time[i, c,
xo])['decimal']
if (MODEL == 'MAR'):
for key, var in zip(KEYS, VARIABLES):
# read and interpolate daily MAR outputs
OUT = SMBcorr.interpolate_mar_daily(DIRECTORY,
EPSG,
MAR_VERSION,
tdec,
D11.x[i, c,
xo],
D11.y[i, c,
xo],
VARIABLE=var,
**KWARGS)
# set attributes to output for iteration
OUTPUT[key].data[i, c, xo] = np.copy(OUT.data)
OUTPUT[key].mask[i, c, xo] = np.copy(OUT.mask)
OUTPUT[key].interpolation[i, c, xo] = np.copy(
OUT.interpolation)
# calculate derived fields
OUTPUT['zsmb_ave'].data[i,c,xo] = OUTPUT['zsurf_ave'].data[i,c,xo] - \
OUTPUT['zfirn_ave'].data[i,c,xo]
OUTPUT['zsmb_ave'].mask[i,c,xo] = OUTPUT['zsurf_ave'].mask[i,c,xo] | \
OUTPUT['zfirn_ave'].mask[i,c,xo]
OUTPUT['zaccum_ave'].data[i,c,xo] = OUTPUT['zsurf_ave'].data[i,c,xo] - \
OUTPUT['zfirn_ave'].data[i,c,xo] - OUTPUT['zmelt_ave'].data[i,c,xo]
OUTPUT['zaccum_ave'].mask[i,c,xo] = OUTPUT['zsurf_ave'].mask[i,c,xo] | \
OUTPUT['zfirn_ave'].mask[i,c,xo] | OUTPUT['zmelt_ave'].mask[i,c,xo]
elif (MODEL == 'RACMO'):
# read and interpolate daily RACMO outputs
for key, var in zip(KEYS, VARIABLES):
OUT = SMBcorr.interpolate_racmo_daily(base_dir,
EPSG,
RACMO_MODEL,
tdec,
D11.x[i, c,
xo],
D11.y[i, c,
xo],
VARIABLE=var,
**KWARGS)
# set attributes to output for iteration
OUTPUT[key].data[i, c, xo] = np.copy(OUT.data)
OUTPUT[key].mask[i, c, xo] = np.copy(OUT.mask)
OUTPUT[key].interpolation[i, c, xo] = np.copy(
OUT.interpolation)
elif (MODEL == 'MERRA2-hybrid'):
# read and interpolate 5-day MERRA2-Hybrid outputs
for key, var in zip(KEYS, VARIABLES):
OUT = SMBcorr.interpolate_merra_hybrid(
DIRECTORY,
EPSG,
MERRA2_REGION,
tdec,
D11.x[i, c, xo],
D11.y[i, c, xo],
VARIABLE=var,
**KWARGS)
# set attributes to output for iteration
OUTPUT[key].data[i, c, xo] = np.copy(OUT.data)
OUTPUT[key].mask[i, c, xo] = np.copy(OUT.mask)
OUTPUT[key].interpolation[i, c, xo] = np.copy(
OUT.interpolation)
else:
# allocate for output height for along-track data
OUTPUT = {}
for key in KEYS:
OUTPUT[key] = np.ma.zeros((nseg, ncycle), fill_value=np.nan)
OUTPUT[key].mask = np.ones((nseg, ncycle), dtype=bool)
# check that there are valid elevations
cycle = [
c for c in range(ncycle)
if np.any(np.isfinite(D11.delta_time[:, c]))
]
# for each valid cycle of ICESat-2 ATL11 data
for c in cycle:
# find valid elevations
i, = np.nonzero(np.isfinite(D11.delta_time[:, c]))
# convert from delta time to decimal-years
tdec = convert_delta_time(D11.delta_time[i, c])['decimal']
if (MODEL == 'MAR'):
for key, var in zip(KEYS, VARIABLES):
# read and interpolate daily MAR outputs
OUT = SMBcorr.interpolate_mar_daily(DIRECTORY,
EPSG,
MAR_VERSION,
tdec,
D11.x[i, c],
D11.y[i, c],
VARIABLE=var,
**KWARGS)
# set attributes to output for iteration
OUTPUT[key].data[i, c] = np.copy(OUT.data)
OUTPUT[key].mask[i, c] = np.copy(OUT.mask)
OUTPUT[key].interpolation[i, c] = np.copy(
OUT.interpolation)
# calculate derived fields
OUTPUT['zsmb_ave'].data[i,c] = OUTPUT['zsurf_ave'].data[i,c] - \
OUTPUT['zfirn_ave'].data[i,c]
OUTPUT['zsmb_ave'].mask[i,c] = OUTPUT['zsurf_ave'].mask[i,c] | \
OUTPUT['zfirn_ave'].mask[i,c]
OUTPUT['zaccum'].data[i,c] = OUTPUT['zsurf_ave'].data[i,c] - \
OUTPUT['zfirn_ave'].data[i,c] - OUTPUT['zmelt_ave'].data[i,c]
OUTPUT['zaccum'].mask[i,c] = OUTPUT['zsurf_ave'].mask[i,c] | \
OUTPUT['zfirn_ave'].mask[i,c] | OUTPUT['zmelt_ave'].mask[i,c]
elif (MODEL == 'RACMO'):
# read and interpolate daily RACMO outputs
for key, var in zip(KEYS, VARIABLES):
OUT = SMBcorr.interpolate_racmo_daily(base_dir,
EPSG,
RACMO_MODEL,
tdec,
D11.x[i, c],
D11.y[i, c],
VARIABLE=var,
**KWARGS)
# set attributes to output for iteration
OUTPUT[key].data[i, c] = np.copy(OUT.data)
OUTPUT[key].mask[i, c] = np.copy(OUT.mask)
OUTPUT[key].interpolation[i, c] = np.copy(
OUT.interpolation)
elif (MODEL == 'MERRA2-hybrid'):
# read and interpolate 5-day MERRA2-Hybrid outputs
for key, var in zip(KEYS, VARIABLES):
OUT = SMBcorr.interpolate_merra_hybrid(DIRECTORY,
EPSG,
MERRA2_REGION,
tdec,
D11.x[i, c],
D11.y[i, c],
VARIABLE=var,
**KWARGS)
# set attributes to output for iteration
OUTPUT[key].data[i, c] = np.copy(OUT.data)
OUTPUT[key].mask[i, c] = np.copy(OUT.mask)
OUTPUT[key].interpolation[i, c] = np.copy(
OUT.interpolation)
# append input HDF5 file with new firn model outputs
fileID = h5py.File(os.path.expanduser(input_file), 'a')
print(input_file) if VERBOSE else None
# fileID.create_group(model_version) if model_version not in fileID.keys() else None
h5 = {}
for key in KEYS:
print(f'{sys.argv[0]}: writing{key}') if VERBOSE else None
# verify mask values
OUTPUT[key].mask |= (OUTPUT[key].data == OUTPUT[key].fill_value) | \
np.isnan(OUTPUT[key].data)
OUTPUT[key].data[OUTPUT[key].mask] = OUTPUT[key].fill_value
# output variable to HDF5
val = '{0}/{1}'.format(model_version, key)
if val not in fileID:
h5[key] = fileID.create_dataset(
val,
OUTPUT[key].shape,
data=OUTPUT[key],
dtype=OUTPUT[key].dtype,
compression='gzip',
fillvalue=OUTPUT[key].fill_value)
else:
h5[key] = fileID[val]
fileID[val][...] = OUTPUT[key]
h5[key].attrs['units'] = "m"
h5[key].attrs['long_name'] = LONGNAME[key]
h5[key].attrs[
'coordinates'] = "../delta_time ../latitude ../longitude"
h5[key].attrs['model'] = model_version
# close the output HDF5 file
fileID.close()
@author: ben
"""
import matplotlib.pyplot as plt
import pointCollection as pc
import sys
from mpl_toolkits.mplot3d import Axes3D
import numpy as np
D = []
for file in sys.argv[1:]:
D.append({\
'dz':pc.grid.data().from_h5(file, group='dz/', field_mapping={'z':'dz'}),
'count':pc.grid.data().from_h5(file, group='dz/', field_mapping={'z':'count'}),
'z0':pc.grid.data().from_h5(file, group='z0/', field_mapping={'z':'z0'}),
'data':pc.data().from_h5(file, group='data/')})
plt.clf()
hax = []
for ii in range(1, 5):
hax.append(plt.gcf().add_subplot(2, 2, ii))
t_slice = [0, 2]
for Di in D:
dt = Di['dz'].t[t_slice[1]] - Di['dz'].t[t_slice[0]]
h_im = hax[0].imshow(
(Di['dz'].z[:, :, t_slice[1]] - Di['dz'].z[:, :, t_slice[0]]) / dt,
extent=Di['dz'].extent,
cmap='Spectral',
origin='lower',
def read(self,
xy_bin,
fields=['x', 'y', 'time'],
index_range=[[-1], [-1]]):
if isinstance(fields, dict):
field_list = []
for key in fields:
field_list += fields[key]
else:
field_list = fields.copy()
out_data = {field: list() for field in field_list}
with h5py.File(self.filename, 'r') as h5f:
blank_fields = list()
if isinstance(xy_bin, np.ndarray):
xy_bin = [xy_bin[:, 0], xy_bin[:, 1]]
if index_range[0][0] >= 0:
# All the geo bins are together. Use the index_range variable to read
for field in field_list:
if field not in h5f:
blank_fields.append(field)
continue
# make sure the index range gets iterated over properly.
if len(index_range[0]) < 2:
if len(h5f[field].shape) > 1:
out_data[field].append(
np.array(h5f[field][
0, int(index_range[0]):int(index_range[1])]))
else:
out_data[field].append(
np.array(
h5f[field]
[int(index_range[0]):int(index_range[1])]))
else:
for i0_i1 in zip(index_range[0], index_range[1]):
if len(h5f[field].shape) > 1:
out_data[field].append(
np.array(h5f[field][0, i0_i1[0]:i0_i1[1]]))
else:
out_data[field].append(
np.array(h5f[field][i0_i1[0]:i0_i1[1]]))
else:
# this is a file with distinct bins, each with its own set of datasets
for xy in zip(xy_bin[0], xy_bin[1]):
bin_name = '%dE_%dN' % xy
for field in field_list:
if field in h5f:
if bin_name in h5f[field]:
out_data[field].append(
np.array(h5f[field][bin_name]).squeeze())
elif bin_name in h5f:
if field in h5f[bin_name]:
out_data[field].append(
np.array(h5f[bin_name][field]).squeeze())
else:
blank_fields.append(field)
for field in field_list:
if isinstance(out_data[field], list):
if len(out_data[field]) > 1 or isinstance(
out_data[field], list):
try:
temp = list()
for item in out_data[field]:
if item.size > 0 and item.ndim > 0:
temp.append(item)
elif item.size == 1 and item.ndim == 0:
temp.append(np.array([item]))
if len(temp) > 1:
out_data[field] = np.concatenate(temp)
elif len(temp) == 0:
out_data[field] = np.zeros(0)
elif len(temp) == 1:
out_data[field] = temp[0]
except ValueError as e:
print(
"ValueError in read_indexed_h5_file, continuing"
)
print(e)
else:
out_data[field] = np.array(out_data[field])
return pc.data(fields=field_list).from_dict(out_data)
'hbar_20m',
'RDE_50m','t_seg','t_qfit','y_atc', 'x_seg_mean', 'y_seg_mean']
out_template={f:np.NaN for f in out_fields}
out=list()
#READ IN THE ATM DATA HERE AND BUILD THE QTREE
# ALSO RUN BLOCKMEDIAN_FOR_QSUB
#plt.figure(1)
for bin_name in D6_GI.keys():
#plt.clf()
print(bin_name)
bin_xy=[int(coord) for coord in bin_name.split('_')]
# query the Qfit index to get all the data for the current bin. pad=6 means that we read 6 km on either side of the central bin, covering the 10-km ATL06 bin plus 1 km buffer
Qlist=Q_GI.query_xy([[bin_xy[0]], [bin_xy[1]]], pad=6, get_data=True, strict=True, fields=Qfit_fields)
Qsub=pc.data().from_list(Qlist).get_xy(SRS_proj4)
Qsub=blockmedian_for_qsub(Qsub, 5)
#START CUTTING HERE
# the geo index works much better (more selective) if the Qfit data are sorted by geobin
x0=np.round(Qsub.x/100.)*100
y0=np.round(Qsub.x/100.)*100
iB=np.argsort(x0+(y0-y0.min())/(y0.max()-y0.min()))
Qsub.index(iB)
# index the sorted Qfit data
GI_Qsub=pc.geoIndex(delta=[100, 100]).from_xy([Qsub.x, Qsub.y])
# query ATL06 for the current bin, and index it
def find_best_wxx0(D11, W_domain, mask_file, DEBUG=False):
scale_vals = np.array([0.01, 0.03, 0.1, 0.3, 1, 3, 10, 30, 100, 300])
E_d3zdx2dt = 0.0001
E_d2z0dx2 = 0.006
E_d2zdt2 = 5000
data_gap_scale = 2500
# define the domain's width in x, y, and time
W = {'x': W_domain, 'y': 200, 't': .2}
# define the grid center:
XR = np.nanmean(D11.x_atc) + np.array([-1, 1]) * W['x'] / 2
ctr = {'x': XR[0] + W['x'] / 2., 'y': 0., 't': 0.}
# define the grid spacing
spacing = {'z0': 50, 'dz': 100, 'dt': .1}
# reference points are every 60 m, but every third reference point
# is returned.
dN = np.ceil(W['x'] / 20).astype(int)
L_interp = {}
for pt0 in np.arange(D11.ref_pt[0, 0] + dN / 2, D11.ref_pt[-1, 0], dN / 2):
ii = np.flatnonzero(np.abs(D11.ref_pt[:, 0] - pt0) < 3 * dN / 2)
N_good = np.sum(np.isfinite(D11.h_corr[ii, :]), axis=0)
if np.max(N_good) < 0.9 * dN:
continue
max_pts = np.max(N_good)
good_enough = np.argwhere(N_good >= max_pts - 5)
if len(good_enough) == 1:
bc = good_enough[0]
else:
# choose the smoothest cycle that has enough points
sigma_dz = np.nanmedian(np.abs(
np.diff(D11.h_corr[ii, good_enough], axis=1)),
axis=1)
bc = good_enough[np.argmin(sigma_dz)]
nb = N_good[bc]
xy_ctr = [
np.nanmean(D11.x[ii, bc]),
np.nanmean(D11.y[ii, bc]),
np.nanmean(D11.h_corr[ii, bc])
]
D=pc.data().from_dict({'x':D11.x_atc[ii,bc], 'y':np.zeros_like(ii, dtype=float),'z':D11.h_corr[ii,bc],\
'time':np.zeros_like(ii, dtype=float), 'sigma':D11.h_corr_sigma[ii,bc]})
D.index(np.isfinite(D.z) & np.isfinite(D.sigma) & (D.sigma > 0))
S = []
ctr = {'x': np.nanmean(D.x), 'y': 0., 't': 0.}
L_curve = {key: [] for key in ['wzz0', 'sigma_hat_s', 'N']}
for scale_val in scale_vals:
# run the fit
E_RMS = {
'd2z0_dx2': E_d2z0dx2 * scale_val,
'dz0_dx': E_d2z0dx2 * data_gap_scale * scale_val,
'd3z_dx2dt': E_d3zdx2dt,
'd2z_dxdt': E_d3zdx2dt * data_gap_scale,
'd2z_dt2': E_d2zdt2
}
try:
S.append(
smooth_xytb_fit(data=D,
ctr=ctr,
W=W,
spacing=spacing,
E_RMS=E_RMS,
reference_epoch=1,
N_subset=None,
compute_E=False,
max_iterations=5,
VERBOSE=False))
except Exception as e:
S.append(
smooth_xytb_fit(data=D,
ctr=ctr,
W=W,
spacing=spacing,
E_RMS=E_RMS,
reference_epoch=1,
N_subset=None,
compute_E=False,
max_iterations=5,
VERBOSE=False))
d_ed = S[-1]['data']
d_ed.index(d_ed.three_sigma_edit == 1)
L_curve['sigma_hat_s'].append(
LSsurf.RDE((d_ed.z - d_ed.z_est) / d_ed.sigma))
L_curve['wzz0'].append(E_RMS['d2z0_dx2'])
L_curve['N'].append(d_ed.size)
for key in L_curve.keys():
L_curve[key] = np.array(L_curve[key])
try:
N0 = safe_interp(1,
L_curve['sigma_hat_s'],
L_curve['N'],
loglog=True)
if DEBUG and np.nanmin(L_curve['sigma_hat_s'] > 1):
return D, ii, bc, pt0, L_curve
w_for_r_of_1 = safe_interp(1,
L_curve['sigma_hat_s'],
L_curve['wzz0'],
loglog=True)
w_for_r_10pct_above_min = \
safe_interp(1.1*L_curve['sigma_hat_s'].min(), L_curve['sigma_hat_s'], L_curve['wzz0'], loglog=True)
L_interp[pt0] = {
"w_for_r_of_1": w_for_r_of_1,
'w_for_r_10pct_above_min': w_for_r_10pct_above_min,
'sigma_hat_min': np.nanmin(L_curve['sigma_hat_s']),
'F_data_r_of_1': N0 / d_ed.size,
'x': xy_ctr[0],
'y': xy_ctr[1],
'z': xy_ctr[2],
'ref_pt': pt0
}
#print([ L_interp[pt0]['w_for_r_of_1'], np.min(L_curve['sigma_hat_s'])])
except Exception as e:
print(e)
return L_interp
D11.get_xy(EPSG=3413)
ind = np.arange(5, D11.x.shape[0], n_skip)
els=(D11.x[ind,0] > MOG.x[0]) & (D11.x[ind,0] < MOG.x[-1]) & \
(D11.y[ind,0] > MOG.y[0]) & (D11.y[ind,0] < MOG.y[-1])
if els.size > 0:
ind = ind[els]
D11.assign({'file_ind': np.zeros_like(D11.x) + count})
D11.index(ind)
xydh[data_count] = D11
data_count += 1
#except:
# print("problem with "+file)
D = pc.data(columns=D11.shape[1]).from_list(xydh)
D.to_h5('/Volumes/ice2/ben/scf/GL_11/rel007_dump_every_4th.h5')
if False:
D3 = D[np.arange(0, D.shape[0], 3)]
gimp_mask = pc.grid.data().from_geotif(
'/Volumes/ice1/ben/GimpMasks_v1.1/GimpIceMask_1km.tif').interp(
D3.x[:, 0], D3.y[:, 0])
D3 = D3[gimp_mask > 0.1]
fig = plt.figure(4)
plt.clf()
xx = D3.x[:, 0]
yy = D3.y[:, 0]
field_dict={'corrected_h':['h_corr','latitude','longitude','delta_time']}
if True:
MOG=pc.grid.data().from_geotif('/Volumes/ice1/ben/MOG/2005/mog_2005_1km.tif')
thedir='/Volumes/ice2/ben/scf/GL_11_alpha'
files=glob.glob(thedir+'/ATL11*.h5')
xydh=[]
for count, file in enumerate(files):
for pair in [1, 2, 3]:
filePair=(file, pair)
D11 = ATL11.data().from_file(filePair[0], pair=filePair[1], field_dict=field_dict).get_xy(None, EPSG=3413)
try:
ind=np.arange(5, D11.x.size, 10)
temp=pc.data().from_dict({'x':D11.x[ind],'y':D11.y[ind], \
'dh':D11.corrected_h.h_corr[ind, cycles[1]]-D11.corrected_h.h_corr[ind, cycles[0]],\
'file_ind': np.zeros_like(ind, dtype=int)+count})
els=(temp.x > MOG.x[0]) & (temp.x < MOG.x[-1]) & \
(temp.y > MOG.y[0]) & (temp.y < MOG.y[-1])
xydh.append(temp[els])
except:
print("problem with "+file)
xydh=pc.data().from_list(xydh)
if True:
plt.figure()
MOG.show(cmap='gray', vmin=14000, vmax=17000)
hl=plt.scatter(xydh.x, xydh.y, 3, linewidth=0, \
c=xydh.dh, \
vmin=-1.25, vmax=1.25, cmap='Spectral')
try:
this_str += " " + files[count]
except Exception:
pass
count += 1
this_str += "\n"
fh.write(this_str)
sys.exit()
N_max = {'weak': 57. / 2. * 10. * 3., 'strong': 57. / 2. * 10. * 12.}
#print(N_max)
spot_max = {spot: N_max['weak'] for spot in [2, 4, 6]}
spot_max.update({spot: N_max['strong'] for spot in [1, 3, 5]})
for file in files:
out_file = out_dir + os.path.basename(file).replace('.h5', '_rh_avg.h5')
z, Ntot, Nbar, P, Ps = read_one_hist(file, spot_max)
print(out_file)
for spot in range(1, 7):
try:
Ds = {
'z': z,
'Nbar': Nbar[spot],
'P': P[spot],
'P_sigma': Ps[spot]
}
temp = pc.data().from_dict(Ds).to_h5(out_file,
group=f'spot_{spot}',
replace=False)
except Exception as e:
print(e)
pass
def read_data(file):
groups = [
'10m', '200m', 'D_std_threshold_200m', 'block_std_threshold_200m'
]
D = {group: pc.data().from_h5(file, group=group) for group in groups}
return D
