def index_cycle_indices(tile_dir_root, hemisphere):
if hemisphere==1:
SRS_proj4='+proj=stere +lat_0=90 +lat_ts=70 +lon_0=-45 +k=1 +x_0=0 +y_0=0 +datum=WGS84 +units=m +no_defs'
else:
SRS_proj4='+proj=stere +lat_0=-90 +lat_ts=-71 +lon_0=0 +k=1 +x_0=0 +y_0=0 +datum=WGS84 +units=m +no_defs'
files=glob(tile_dir_root+'/cycle*/GeoIndex.h5')
index_list=list()
for sub_index_file in files:
temp=geo_index().from_file(sub_index_file)
xy=temp.bins_as_array()
index_list.append(geo_index(delta=[1.e4, 1.e4]).from_xy(xy, filename=sub_index_file, file_type='h5_geoindex', fake_offset_val=-1))
temp=None
geo_index(delta=[1.e4, 1.e4], SRS_proj4=SRS_proj4).from_list(index_list).to_file(tile_dir_root+'/GeoIndex.h5')
def queryIndex(index_file, demFile, verbose=False):
from PointDatabase.geo_index import geo_index
from IS2_calval.demBounds import demBounds
pointData = dict()
beam_names = ['l', 'r']
field_dict = {
None: ['delta_time', 'h_li', 'h_li_sigma', 'latitude', 'longitude'],
'geophysical': ['dac'],
'fit_statistics': [
'dh_fit_dx', 'h_rms_misfit', 'h_robust_sprd', 'n_fit_photons',
'h_mean', 'signal_selection_source', 'snr_significance',
'w_surface_window_final'
],
'derived': ['valid']
}
gI = geo_index().from_file(index_file)
xr, yr = demBounds(demFile, proj4=gI.attrs['SRS_proj4'])
xr += np.array([-1e4, 1e4])
yr += np.array([-1e4, 1e4])
xy = gI.bins_as_array()
#plt.figure(); plt.plot(xy[0], xy[1],'k.')
#plt.plot(xr[[0, 1, 1, 0, 0]], yr[[0, 0, 1, 1, 0]],'r')
D6es = gI.query_xy_box(xr, yr, get_data=True, fields=field_dict)
if verbose:
temp = gI.query_xy_box(xr + np.array([-1e4, 1e4]),
yr + np.array([-1e4, 1e4]),
get_data=False)
print("list of datasets:")
for key in set(temp.keys()):
print("\t%s" % key)
for D6 in D6es:
f06.phDensityFilter(D6, toNaN=True, subset=True)
if D6.h_li.size == 0:
continue
f06.segDifferenceFilter(D6, tol=2, toNaN=True, subset=True)
if D6.h_li.size == 0:
continue
for beam in [0, 1]:
if np.sum(np.isfinite(D6.h_li[:, beam])) > 100:
first = np.max(np.where(np.isfinite(D6.latitude[:, beam]))[0])
last = np.min(np.where(np.isfinite(D6.latitude[:, beam]))[0])
AD = np.sign(D6.latitude[last, beam] -
D6.latitude[first, beam])
this_name = "%s:gt%d%s" % (os.path.basename(
D6.file), D6.beam_pair, beam_names[beam])
pointData[this_name] = {
'latitude': D6.latitude[:, beam],
'longitude': D6.longitude[:, beam],
'h': D6.h_mean[:, beam] + D6.dac[:, beam],
'delta_time': D6.delta_time[:, beam],
'AD': AD + np.zeros_like(D6.delta_time),
'orbit': D6.orbit + np.zeros_like(D6.delta_time)
}
return pointData
def index_tiles(tile_dir_root, cycle, hemisphere):
"""
Generate a geo_index for the tile files
Arguments:
tile_dir_root: directory under which to search for the tile files
hemisphere: north(1) or south(-1), used to set the projection
"""
if hemisphere==1:
SRS_proj4='+proj=stere +lat_0=90 +lat_ts=70 +lon_0=-45 +k=1 +x_0=0 +y_0=0 +datum=WGS84 +units=m +no_defs'
else:
SRS_proj4='+proj=stere +lat_0=-90 +lat_ts=-71 +lon_0=0 +k=1 +x_0=0 +y_0=0 +datum=WGS84 +units=m +no_defs '
cycle_root=tile_dir_root+('/cycle_%02d' % cycle)
files=glob(cycle_root+'/E*.h5')
print("found files:")
print(files)
iList = index_list_for_files(files, "indexed_h5", [1.e4, 1.e4], SRS_proj4, dir_root=cycle_root)
geo_index(SRS_proj4=SRS_proj4, delta=[1.e4, 1.e4]).from_list(iList, dir_root=tile_dir_root).to_file(cycle_root+'/GeoIndex.h5')
def test_plot(gi_file):
import matplotlib.pyplot as plt
index=geo_index().from_file(gi_file)
dx,dy=np.meshgrid(np.array([-1, 0, 1])*1.e4, np.array([-1, 0, 1])*1.e4)
XX=index.bins_as_array()
D_list=index.query_xy((np.array(XX[0][0])+dx, np.array(XX[1][0])+dy) , fields=['x','y'])
plt.figure();
for D in D_list:
plt.plot(D.x, D.y,'.')
plt.show()
plt.close()
print('Use control-C to close figure.')
def make_tile(args):
xy0 = args['xy0']
SRS_proj4 = args['SRS_proj4']
tile_spacing = args['tile_spacing']
bin_W = args['bin_W']
GI_file = args['GI_file']
out_dir = args['out_dir']
field_dict = args['field_dict']
seg_diff_scale = args['seg_diff_scale']
blockmedian_scale = args['blockmedian_scale']
dxb, dyb = np.meshgrid(
np.arange(-tile_spacing / 2, tile_spacing / 2 + bin_W, bin_W),
np.arange(-tile_spacing / 2, tile_spacing / 2 + bin_W, bin_W))
dxb = dxb.ravel()
dyb = dyb.ravel()
list_of_fields = []
for group in field_dict:
for ds in field_dict[group]:
list_of_fields.append(ds)
gI = geo_index().from_file(GI_file, read_file=False)
out_file = out_dir + ('/E%d_N%d.h5' % (xy0[0] / 1.e3, xy0[1] / 1.e3))
if os.path.isfile(out_file):
os.remove(out_file)
D = gI.query_xy((xy0[0] + dxb, xy0[1] + dyb), fields=field_dict)
if D is None:
return
file_dict = {}
delete_list = []
for file_num, Di in enumerate(D):
Di.get_xy(SRS_proj4)
Di.assign(
{'source_file_num': np.zeros_like(Di.x, dtype=int) + file_num})
if seg_diff_scale is not None:
Di.h_li[Di.atl06_quality_summary == 1] = np.NaN
segDifferenceFilter(Di, setValid=False, toNaN=True)
Di.ravel_fields()
if blockmedian_scale is not None:
Di.index(np.isfinite(Di.h_li) & (Di.atl06_quality_summary == 0))
try:
ind = pt_blockmedian(Di.x,
Di.y,
Di.h_li,
blockmedian_scale,
return_index=True)[3]
except Exception:
delete_list.append(Di)
continue
Di.index(ind[:, 0])
else:
Di.index(np.isfinite(Di.h_li))
file_dict[file_num] = Di.filename
D_all = point_data(list_of_fields=list_of_fields +
['x', 'y', 'source_file_num']).from_list(D)
y_bin_function = np.round(D_all.y / bin_W)
x_bin_function = np.round(D_all.x / bin_W)
x_scale = np.nanmax(x_bin_function) - np.nanmin(x_bin_function)
t_scale = np.nanmax(D_all.delta_time) - np.nanmin(D_all.delta_time)
xy_bin_function = (y_bin_function -
np.nanmin(y_bin_function)) * x_scale + (
x_bin_function - np.nanmin(x_bin_function))
xyt_bin_function = xy_bin_function + (
D_all.delta_time - np.nanmin(D_all.delta_time)) / t_scale
ind = np.argsort(xyt_bin_function)
bin_dict = {}
xy_bin_fn_sort = xy_bin_function[ind]
fn_delta = np.concatenate([[-1],
np.where(np.diff(xy_bin_fn_sort))[0],
[xy_bin_fn_sort.size]])
for ii in range(len(fn_delta) - 1):
this_ind = ind[(fn_delta[ii] + 1):(fn_delta[ii + 1] + 1)]
bin_dict[(x_bin_function[this_ind[0]],
y_bin_function[this_ind[0]])] = this_ind
key_arr = np.array([key for key in bin_dict.keys()])
key_order = np.argsort(key_arr[:, 1] - np.min(key_arr[:, 1]) * x_scale +
(key_arr[:, 0] - np.min(key_arr[:, 0])))
key_arr = key_arr[key_order, :]
for key in key_arr:
this_group = '%dE_%dN' % tuple(key * bin_W)
D_all.subset(bin_dict[tuple(key)]).to_file(out_file,
replace=False,
group=this_group)
with h5py.File(out_file, 'r+') as h5f:
grp = h5f.create_group("source_files")
for key in file_dict:
grp.attrs['file_%d' % key] = file_dict[key]
Qfit_fields = [
'latitude', 'longitude', 'elevation', 'seconds_of_day', 'days_J2K'
]
ps_srs = osr.SpatialReference()
ps_srs.ImportFromProj4(SRS_proj4)
ll_srs = osr.SpatialReference()
ll_srs.ImportFromEPSG(4326)
ll2ps = osr.CoordinateTransformation(ll_srs, ps_srs).TransformPoint
ps2ll = osr.CoordinateTransformation(ps_srs, ll_srs).TransformPoint
WGS84a = 6378137.0
WGS84b = 6356752.31424
d2r = np.pi / 180.
delta = [10000., 10000.]
Q_GI = geo_index(SRS_proj4=SRS_proj4).from_file(Qfit_index)
D6_GI = geo_index(SRS_proj4=SRS_proj4).from_file(ATL06_index)
# the qfit index is at 1 km , the ATL06 index is at 10 km. find the overlap between the two
# Interesect the ATL06 index with the Qfit index
xy_10km_Q = unique_points(Q_GI.bins_as_array())
D6_GI = D6_GI.copy_subset(xyBin=xy_10km_Q)
out_fields = [
'segment_id', 'x', 'y', 'beam', 'beam_pair', 'h_li', 'h_li_sigma',
'atl06_quality_summary', 'dh_fit_dx', 'N_50m', 'N_seg', 'h_qfit_seg',
'dh_qfit_dx', 'dh_qfit_dy', 'h_robust_sprd', 'snr_significance',
'h_qfit_50m', 'sigma_qfit_50m', 'sigma_seg', 'dz_50m', 'E_seg', 'RDE_seg',
'RDE_50m', 't_seg', 't_qfit', 'y_atc', 'x_seg_mean', 'y_seg_mean'
]
out_template = {f: np.NaN for f in out_fields}
out = list()
def glas_fit(xy0=np.array((-150000, -2000000)), W0, D=None, E_RMS=None, gI=None, giFile='/Data/glas/GL/rel_634/GeoIndex.h5'):
if gI is None:
gI=geo_index().from_file(giFile)
#import_D=False; print("WARNING::::::REUSING D")
timing=dict()
xy0=np.array((-150000, -2000000))
E_RMS={'d2z0_dx2':20000./3000/3000, 'd3z_dx2dt':10./3000/3000, 'd2z_dxdt':100/3000, 'd2z_dt2':1}
W={'x':W0, 'y':W0,'t':6}
spacing={'z0':5.e2, 'dzdt':5.e3}
args={'W':W, 'ctr':ctr, 'spacing':spacing, 'E_RMS':E_RMS, 'max_iterations':25}
if D is None:
fields=[ 'IceSVar', 'deltaEllip', 'numPk', 'ocElv', 'reflctUC', 'satElevCorr', 'time', 'x', 'y', 'z']
ctr={'x':xy0[0], 'y':xy0[1], 't':(2003+2009)/2. }
D=gI.query_xy_box(xy0[0]+np.array([-W['x']/2, W['x']/2]), xy0[1]+np.array([-W['y']/2, W['y']/2]), fields=fields)
#plt.plot(xy[0], xy[1],'.')
#plt.plot(xy0[0], xy0[1],'r*')
D.assign({'year': matlabToYear(D.time)})
good=(D.IceSVar < 0.035) & (D.reflctUC >0.05) & (D.satElevCorr < 1) & (D.numPk==1)
D.subset(good, datasets=['x','y','z','year'])
D.assign({'sigma':np.zeros_like(D.x)+0.2, 'time':D.year})
plt.plot(D.x, D.y,'m.')
bds={coord:args['ctr'][coord]+np.array([-0.5, 0.5])*args['W'][coord] for coord in ('x','y')}
grids=dict()
grids['z0']=fd_grid( [bds['y'], bds['x']], args['spacing']['z0']*np.ones(2), name='z0')
grids['dzdt']=fd_grid( [bds['y'], bds['x']], args['spacing']['dzdt']*np.ones(2), \
col_0=grids['z0'].col_N+1, name='dzdt')
valid_z0=grids['z0'].validate_pts((D.coords()[0:2]))
valid_dz=grids['dzdt'].validate_pts((D.coords()))
valid_data=valid_dz & valid_z0
D=D.subset(valid_data)
G_data=lin_op(grids['z0'], name='interp_z').interp_mtx(D.coords()[0:2])
G_dzdt=lin_op(grids['dzdt'], name='dzdt').interp_mtx(D.coords()[0:2])
G_dzdt.v *= (D.year[G_dzdt.r.astype(int)]-ctr['t'])
G_data.add(G_dzdt)
grad2_z0=lin_op(grids['z0'], name='grad2_z0').grad2(DOF='z0')
grad_z0=lin_op(grids['z0'], name='grad_z0').grad(DOF='z0')
grad2_dzdt=lin_op(grids['dzdt'], name='grad2_dzdt').grad2(DOF='dzdt')
grad_dzdt=lin_op(grids['dzdt'], name='grad_dzdt').grad2(DOF='dzdt')
Gc=lin_op(None, name='constraints').vstack((grad2_z0, grad_z0, grad2_dzdt, grad_dzdt))
Ec=np.zeros(Gc.N_eq)
root_delta_A_z0=np.sqrt(np.prod(grids['z0'].delta))
Ec[Gc.TOC['rows']['grad2_z0']]=args['E_RMS']['d2z0_dx2']/root_delta_A_z0
Ec[Gc.TOC['rows']['grad2_dzdt']]=args['E_RMS']['d3z_dx2dt']/root_delta_A_z0
Ec[Gc.TOC['rows']['grad_z0']]=1.e4*args['E_RMS']['d2z0_dx2']/root_delta_A_z0
Ec[Gc.TOC['rows']['grad_dzdt']]=1.e4*args['E_RMS']['d3z_dx2dt']/root_delta_A_z0
Ed=D.sigma.ravel()
N_eq=G_data.N_eq+Gc.N_eq
# calculate the inverse square root of the data covariance matrix
TCinv=sp.dia_matrix((1./np.concatenate((Ed, Ec)), 0), shape=(N_eq, N_eq))
# define the right hand side of the equation
rhs=np.zeros([N_eq])
rhs[0:D.x.size]=D.z.ravel()
# put the fit and constraint matrices together
Gcoo=sp.vstack([G_data.toCSR(), Gc.toCSR()]).tocoo()
cov_rows=G_data.N_eq+np.arange(Gc.N_eq)
# initialize the book-keeping matrices for the inversion
m0=np.zeros(Gcoo.shape[1])
inTSE=np.arange(G_data.N_eq, dtype=int)
for iteration in range(args['max_iterations']):
# build the parsing matrix that removes invalid rows
Ip_r=sp.coo_matrix((np.ones(Gc.N_eq+inTSE.size), (np.arange(Gc.N_eq+inTSE.size), np.concatenate((inTSE, cov_rows)))), shape=(Gc.N_eq+inTSE.size, Gcoo.shape[0])).tocsc()
m0_last=m0
# solve the equations
tic=time();
m0=sparseqr.solve(Ip_r.dot(TCinv.dot(Gcoo)), Ip_r.dot(TCinv.dot(rhs)));
timing['sparseqr_solve']=time()-tic
# quit if the solution is too similar to the previous solution
if np.max(np.abs((m0_last-m0)[Gc.TOC['cols']['dzdt']])) < 0.05:
break
# calculate the full data residual
rs_data=(D.z-G_data.toCSR().dot(m0))/D.sigma
# calculate the robust standard deviation of the scaled residuals for the selected data
sigma_hat=RDE(rs_data[inTSE])
inTSE_last=inTSE
# select the data that are within 3*sigma of the solution
inTSE=np.where(np.abs(rs_data)<3.0*sigma_hat)[0]
print('found %d in TSE, sigma_hat=%3.3f' % (inTSE.size, sigma_hat))
if sigma_hat <= 1 or( inTSE.size == inTSE_last.size and np.all( inTSE_last == inTSE )):
break
m=dict()
m['z0']=m0[Gc.TOC['cols']['z0']].reshape(grids['z0'].shape)
m['dzdt']=m0[Gc.TOC['cols']['dzdt']].reshape(grids['dzdt'].shape)
if DOPLOT:
plt.subplot(121)
plt.imshow(m['z0'])
plt.colorbar()
plt.subplot(122)
plt.imshow(m['dzdt'])
plt.colorbar()
if False:
plt.figure()
Dfinal=D.subset(inTSE)
ii=np.argsort(Dfinal.z)
plt.scatter(Dfinal.x[ii], Dfinal.y[ii], c=Dfinal.z[ii]); plt.colorbar()
return grids, m, D, inTSE, sigma_hat
def fit_GL():
giFile='/Data/glas/GL/rel_634/GeoIndex.h5'
gI=geo_index().from_file(giFile, read_file=False)