def manual_select_pairs_to_remove(File): '''Manually select interferograms to remove''' print '----------------------------------------------------------------------------' print 'Manually select interferograms to remove' print 'Click two dates - points - in the figure to select one pair of interferogram' print 'repeat until you select all pairs you would like to remove' print 'then close the figure to continue the program ...' print '----------------------------------------------------------------------------' # Display the network fig = plt.figure() ax = fig.add_subplot(111) date12_orig = pnet.get_date12_list(File) bperp_list = ut.perp_baseline_ifgram2timeseries(File)[0].tolist() date8_list = ptime.ifgram_date_list(File) ax = pnet.plot_network(ax, date12_orig, date8_list, bperp_list) print 'display the network of interferogram of file: ' + File date6_list = ptime.yymmdd(date8_list) dates_array = np.array(ptime.date_list2vector(date8_list)[0]) dateNum_array = mdates.date2num(dates_array) bperp_array = np.array(bperp_list) date_click = [] date12_click = [] def onclick(event): xClick = event.xdata yClick = event.ydata idx = nearest_neighbor(xClick, yClick, dateNum_array, bperp_array) date6 = date6_list[idx] print 'click at ' + date6 date_click.append(date6) if len(date_click) % 2 == 0 and date_click[-2] != date_click[-1]: [m_date, s_date] = sorted(date_click[-2:]) m_idx = date6_list.index(m_date) s_idx = date6_list.index(s_date) date12 = m_date + '-' + s_date if date12 in date12_orig: print 'select date12: ' + date12 date12_click.append(date12) ax.plot([dateNum_array[m_idx], dateNum_array[s_idx]], [bperp_array[m_idx], bperp_array[s_idx]], 'r', lw=4) else: print date12 + ' is not existed in input file' plt.draw() cid = fig.canvas.mpl_connect('button_press_event', onclick) plt.show() return date12_click
def main(argv): inps = cmdLineParse() if not inps.disp_fig: plt.switch_backend('Agg') if inps.template_file: inps = read_template2inps(inps.template_file, inps) ##### 1. Read Info # Read dateList and bperpList ext = os.path.splitext(inps.file)[1] if ext in ['.h5']: atr = readfile.read_attribute(inps.file) k = atr['FILE_TYPE'] print 'reading date and perpendicular baseline from ' + k + ' file: ' + os.path.basename( inps.file) if not k in multi_group_hdf5_file: raise ValueError('only the following file type are supported:\n' + str(multi_group_hdf5_file)) if not inps.coherence_file and k == 'coherence': inps.coherence_file = inps.file pbase_list = ut.perp_baseline_ifgram2timeseries(inps.file)[0] date8_list = ptime.ifgram_date_list(inps.file) else: print 'reading date and perpendicular baseline from baseline list file: ' + inps.bl_list_file date8_list, pbase_list = pnet.read_baseline_file( inps.bl_list_file)[0:2] print 'number of acquisitions : ' + str(len(date8_list)) # Read Pairs Info print 'reading pairs info from file: ' + inps.file date12_list = pnet.get_date12_list(inps.file) print 'number of interferograms: ' + str(len(date12_list)) # Read drop_ifgram date8_list_drop = [] date12_list_drop = [] if ext in ['.h5', '.he5']: h5 = h5py.File(inps.file, 'r') ifgram_list_all = sorted(h5[k].keys()) ifgram_list_keep = ut.check_drop_ifgram(h5) date12_list_keep = ptime.list_ifgram2date12(ifgram_list_keep) # Get date12_list_drop date12_list_drop = sorted( list(set(date12_list) - set(date12_list_keep))) print 'number of interferograms marked as dropped: ' + str( len(date12_list_drop)) print 'number of interferograms marked as kept : ' + str( len(date12_list_keep)) # Get date_list_drop m_dates = [i.split('-')[0] for i in date12_list_keep] s_dates = [i.split('-')[1] for i in date12_list_keep] date8_list_keep = ptime.yyyymmdd(sorted(list(set(m_dates + s_dates)))) date8_list_drop = sorted(list(set(date8_list) - set(date8_list_keep))) print 'number of acquisitions marked as dropped: ' + str( len(date8_list_drop)) # Read Coherence List inps.coherence_list = None if inps.coherence_file and os.path.isfile(inps.coherence_file): if inps.mask_file and not os.path.isfile(inps.mask_file): inps.mask_file = None inps.coherence_list, inps.coh_date12_list = ut.spatial_average(inps.coherence_file, inps.mask_file, \ saveList=True, checkAoi=False) if all(np.isnan(inps.coherence_list)): print 'WARNING: all coherence value are nan! Do not use this and continue.' inps.coherence_list = None # Check subset of date12 info between input file and coherence file if not set(inps.coh_date12_list) >= set(date12_list): print 'WARNING: not every pair/date12 from input file is in coherence file' print 'turn off the color plotting of interferograms based on coherence' inps.coherence_list = None elif set(inps.coh_date12_list) > set(date12_list): print 'extract coherence value for all pair/date12 in input file' inps.coherence_list = [ inps.coherence_list[inps.coh_date12_list.index(i)] for i in date12_list ] #inps.coh_thres = 0.7 ##### 2. Plot inps.cbar_label = 'Average spatial coherence' # Fig 1 - Baseline History fig = plt.figure() ax = fig.add_subplot(111) ax = pnet.plot_perp_baseline_hist(ax, date8_list, pbase_list, vars(inps), date8_list_drop) figName = 'BperpHistory' + inps.fig_ext if inps.save_fig: fig.savefig(figName, bbox_inches='tight') print 'save figure to ' + figName # Fig 2 - Coherence Matrix if inps.coherence_list: figName = 'CoherenceMatrix' + inps.fig_ext if inps.fig_size: fig = plt.figure(figsize=inps.fig_size) else: fig = plt.figure() ax = fig.add_subplot(111) ax = pnet.plot_coherence_matrix(ax, date12_list, inps.coherence_list,\ date12_list_drop, plot_dict=vars(inps)) if inps.save_fig: fig.savefig(figName, bbox_inches='tight', dpi=150) print 'save figure to ' + figName # Fig 3 - Min/Max Coherence History if inps.coherence_list: figName = 'CoherenceHistory' + inps.fig_ext fig = plt.figure() ax = fig.add_subplot(111) ax = pnet.plot_coherence_history(ax, date12_list, inps.coherence_list) if inps.save_fig: fig.savefig(figName, bbox_inches='tight') print 'save figure to ' + figName # Fig 4 - Interferogram Network if inps.fig_size: fig = plt.figure(figsize=inps.fig_size) else: fig = plt.figure() ax = fig.add_subplot(111) ax = pnet.plot_network(ax, date12_list, date8_list, pbase_list, vars(inps), date12_list_drop) figName = 'Network' + inps.fig_ext if inps.save_fig: fig.savefig(figName, bbox_inches='tight') print 'save figure to ' + figName if inps.save_list: txtFile = os.path.splitext(inps.file)[0] + '_date12_list.txt' np.savetxt(txtFile, date12_list, fmt='%s') print 'save pairs/date12 info to file: ' + txtFile if inps.disp_fig: plt.show()
def main(argv): inps = cmdLineParse() if not inps.disp_fig: plt.switch_backend('Agg') #print '\n******************** Plot Network **********************' ##### 1. Read Info # Read dateList and bperpList ext = os.path.splitext(inps.file)[1] if ext in ['.h5']: atr = readfile.read_attribute(inps.file) k = atr['FILE_TYPE'] print 'reading date and perpendicular baseline from '+k+' file: '+os.path.basename(inps.file) if not k in multi_group_hdf5_file: raise ValueError('only the following file type are supported:\n'+str(multi_group_hdf5_file)) pbase_list = ut.perp_baseline_ifgram2timeseries(inps.file)[0] date8_list = ptime.ifgram_date_list(inps.file) else: print 'reading date and perpendicular baseline from baseline list file: '+inps.bl_list_file date8_list, pbase_list = pnet.read_baseline_file(inps.bl_list_file)[0:2] print 'number of acquisitions : '+str(len(date8_list)) # Read Pairs Info print 'reading pairs info from file: '+inps.file date12_list = pnet.get_date12_list(inps.file) print 'number of interferograms: '+str(len(date12_list)) # Read drop_ifgram date8_list_drop = [] date12_list_drop = [] if ext in ['.h5','.he5']: h5 = h5py.File(inps.file, 'r') ifgram_list_all = sorted(h5[k].keys()) ifgram_list_keep = ut.check_drop_ifgram(h5, atr, ifgram_list_all) date12_list_keep = ptime.list_ifgram2date12(ifgram_list_keep) # Get date12_list_drop date12_list_drop = sorted(list(set(date12_list) - set(date12_list_keep))) print 'number of interferograms marked as dropped: '+str(len(date12_list_drop)) # Get date_list_drop m_dates = [i.split('-')[0] for i in date12_list_keep] s_dates = [i.split('-')[1] for i in date12_list_keep] date8_list_keep = ptime.yyyymmdd(sorted(list(set(m_dates + s_dates)))) date8_list_drop = sorted(list(set(date8_list) - set(date8_list_keep))) print 'number of acquisitions marked as dropped: '+str(len(date8_list_drop)) # Read Coherence List inps.coherence_list = None if inps.coherence_file and os.path.isfile(inps.coherence_file): ext = os.path.splitext(inps.coherence_file)[1] if ext in ['.h5']: listFile = os.path.splitext(inps.coherence_file)[0]+'_spatialAverage.txt' if os.path.isfile(listFile): print 'reading coherence value from existed '+listFile fcoh = np.loadtxt(listFile, dtype=str) inps.coherence_list = [float(i) for i in fcoh[:,1]] inps.coh_date12_list = [i for i in fcoh[:,0]] else: print 'calculating average coherence value from '+inps.coherence_file if inps.mask_file: mask = readfile.read(inps.mask_file)[0] else: mask = None inps.coherence_list = ut.spatial_average(inps.coherence_file, mask, saveList=True) inps.coh_date12_list = pnet.get_date12_list(inps.coherence_file) else: print 'reading coherence value from '+inps.coherence_file fcoh = np.loadtxt(inps.coherence_file, dtype=str) inps.coherence_list = [float(i) for i in fcoh[:,1]] inps.coh_date12_list = [i for i in fcoh[:,0]] # Check length of coherence file and input file if not set(inps.coh_date12_list) == set(date12_list): print 'WARNING: input coherence list has different pairs/date12 from input file' print 'turn off the color plotting of interferograms based on coherence' inps.coherence_list = None #inps.coh_thres = 0.7 ##### 2. Plot # Fig 1 - Baseline History fig = plt.figure() ax = fig.add_subplot(111) ax = pnet.plot_perp_baseline_hist(ax, date8_list, pbase_list, vars(inps), date8_list_drop) figName = 'BperpHistory'+inps.fig_ext if inps.save_fig: fig.savefig(figName,bbox_inches='tight') print 'save figure to '+figName # Fig 2 - Coherence Matrix if inps.coherence_list: figName = 'CoherenceMatrix'+inps.fig_ext if inps.fig_size: fig = plt.figure(figsize=inps.fig_size) else: fig = plt.figure() ax = fig.add_subplot(111) ax = pnet.plot_coherence_matrix(ax, date12_list, inps.coherence_list) if inps.save_fig: fig.savefig(figName, bbox_inches='tight') print 'save figure to '+figName # Fig 3 - Min/Max Coherence History if inps.coherence_list: figName = 'CoherenceHistory'+inps.fig_ext fig = plt.figure() ax = fig.add_subplot(111) ax = pnet.plot_coherence_history(ax, date12_list, inps.coherence_list) if inps.save_fig: fig.savefig(figName, bbox_inches='tight') print 'save figure to '+figName # Fig 4 - Interferogram Network if inps.fig_size: fig = plt.figure(figsize=inps.fig_size) else: fig = plt.figure() ax = fig.add_subplot(111) ax = pnet.plot_network(ax, date12_list, date8_list, pbase_list, vars(inps), date12_list_drop) figName = 'Network'+inps.fig_ext if inps.save_fig: fig.savefig(figName,bbox_inches='tight') print 'save figure to '+figName if inps.save_list: txtFile = os.path.splitext(inps.file)[0]+'_date12_list.txt' np.savetxt(txtFile, date12_list, fmt='%s') print 'save pairs/date12 info to file: '+txtFile if inps.disp_fig: plt.show()
def main(argv): inps = cmdLineParse() suffix = '_demErr' if not inps.outfile: inps.outfile = os.path.splitext( inps.timeseries_file)[0] + suffix + os.path.splitext( inps.timeseries_file)[1] # 1. template_file if inps.template_file: print 'read option from template file: ' + inps.template_file inps = read_template2inps(inps.template_file, inps) # Read Time Series print "loading time series: " + inps.timeseries_file atr = readfile.read_attribute(inps.timeseries_file) length = int(atr['FILE_LENGTH']) width = int(atr['WIDTH']) h5 = h5py.File(inps.timeseries_file) date_list = sorted(h5['timeseries'].keys()) date_num = len(date_list) print 'number of acquisitions: ' + str(date_num) # Exclude date info #inps.ex_date = ['20070115','20100310'] if inps.ex_date: inps = get_exclude_date(inps, date_list) if inps.ex_date: inps.ex_flag = np.array([i not in inps.ex_date for i in date_list]) timeseries = np.zeros((len(date_list), length * width), np.float32) prog_bar = ptime.progress_bar(maxValue=date_num, prefix='loading: ') for i in range(date_num): date = date_list[i] d = h5['timeseries'].get(date)[:] timeseries[i][:] = d.flatten('F') prog_bar.update(i + 1, suffix=date) del d h5.close() prog_bar.close() # Perpendicular Baseline print 'read perpendicular baseline' try: inps.pbase = ut.perp_baseline_timeseries(atr, dimension=0) if inps.pbase.shape[1] > 1: print '\tconsider P_BASELINE variation in azimuth direction' else: pbase = inps.pbase except: print '\tCannot find P_BASELINE_TIMESERIES from timeseries file.' print '\tTrying to calculate it from interferograms file' if inps.ifgram_file: inps.pbase = np.array( ut.perp_baseline_ifgram2timeseries( inps.ifgram_file)[0]).reshape(date_num, 1) else: message = 'No interferogram file input!\n'+\ 'Can not correct for DEM residula without perpendicular base info!' raise Exception(message) # Temporal Baseline print 'read temporal baseline' inps.tbase = np.array(ptime.date_list2tbase(date_list)[0]).reshape( date_num, 1) # Incidence angle (look angle in the paper) if inps.incidence_angle: if os.path.isfile(inps.incidence_angle): print 'reading incidence angle from file: ' + inps.incidence_angle inps.incidence_angle = readfile.read(inps.incidence_angle)[0] else: try: inps.incidence_angle = np.array(float(inps.incidence_angle)) print 'use input incidence angle : ' + str( inps.incidence_angle) except: raise ValueError('Can not read input incidence angle: ' + str(inps.incidence_angle)) else: print 'calculate incidence angle using attributes of time series file' if inps.pbase.shape[1] > 1: inps.incidence_angle = ut.incidence_angle(atr, dimension=2) else: inps.incidence_angle = ut.incidence_angle(atr, dimension=1) inps.incidence_angle *= np.pi / 180.0 # Range distance if inps.range_dis: if os.path.isfile(inps.range_dis): print 'reading range distance from file: ' + inps.range_dis inps.range_dis = readfile.read(inps.range_dis)[0] else: try: inps.range_dis = np.array(float(inps.range_dis)) print 'use input range distance : ' + str(inps.range_dis) except: raise ValueError('Can not read input incidence angle: ' + str(inps.range_dis)) else: print 'calculate range distance using attributes from time series file' if inps.pbase.shape[1] > 1: inps.range_dis = ut.range_distance(atr, dimension=2) else: inps.range_dis = ut.range_distance(atr, dimension=1) # Design matrix - temporal deformation model using tbase print '-------------------------------------------------' if inps.phase_velocity: print 'using phase velocity history' A1 = np.ones((date_num - 1, 1)) A2 = (inps.tbase[1:date_num] + inps.tbase[0:date_num - 1]) / 2.0 A3 = (inps.tbase[1:date_num]**3 - inps.tbase[0:date_num - 1]** 3) / np.diff(inps.tbase, axis=0) / 6.0 #A3 = (inps.tbase[1:date_num]**2 + inps.tbase[1:date_num]*inps.tbase[0:date_num-1] +\ # inps.tbase[0:date_num-1]**2) / 6.0 else: print 'using phase history' A1 = np.hstack((np.ones((date_num, 1)), inps.tbase)) A2 = inps.tbase**2 / 2.0 A3 = inps.tbase**3 / 6.0 # Polynomial order of model print "temporal deformation model's polynomial order = " + str( inps.poly_order) if inps.poly_order == 1: A_def = A1 elif inps.poly_order == 2: A_def = np.hstack((A1, A2)) elif inps.poly_order == 3: A_def = np.hstack((A1, A2, A3)) # step function if inps.step_date: print "temporal deformation model's step function step at " + inps.step_date step_yy = ptime.yyyymmdd2years(inps.step_date) yy_list = ptime.yyyymmdd2years(date_list) flag_array = np.array(yy_list) >= step_yy A_step = np.zeros((date_num, 1)) A_step[flag_array] = 1.0 A_def = np.hstack((A_def, A_step)) # Heresh's original code for phase history approach #A_def = np.hstack((A2,A1,np.ones((date_num,1)))) print '-------------------------------------------------' ##---------------------------------------- Loop for L2-norm inversion -----------------------------------## delta_z_mat = np.zeros([length, width], dtype=np.float32) resid_n = np.zeros([A_def.shape[0], length * width], dtype=np.float32) constC = np.zeros([length, width], dtype=np.float32) #delta_a_mat = np.zeros([length, width]) if inps.incidence_angle.ndim == 2 and inps.range_dis.ndim == 2: print 'inversing using L2-norm minimization (unweighted least squares)'\ ' pixel by pixel: %d loops in total' % (length*width) prog_bar = ptime.progress_bar(maxValue=length * width, prefix='calculating: ') for i in range(length * width): row = i % length col = i / length range_dis = inps.range_dis[row, col] inc_angle = inps.incidence_angle[row, col] # Consider P_BASELINE variation within one interferogram if inps.pbase.shape[1] > 1: pbase = inps.pbase[:, row].reshape(date_num, 1) # Design matrix - DEM error using pbase, range distance and incidence angle A_delta_z = pbase / (range_dis * np.sin(inc_angle)) if inps.phase_velocity: pbase_v = np.diff(pbase, axis=0) / np.diff(inps.tbase, axis=0) A_delta_z_v = pbase_v / (range_dis * np.sin(inc_angle)) A = np.hstack((A_delta_z_v, A_def)) else: A = np.hstack((A_delta_z, A_def)) # L-2 norm inversion if inps.ex_date: A_inv = np.linalg.pinv(A[inps.ex_flag, :]) else: A_inv = np.linalg.pinv(A) # Get unknown parameters X = [delta_z, vel, acc, delta_acc, ...] ts_dis = timeseries[:, i] if inps.phase_velocity: ts_dis = np.diff(ts_dis, axis=0) / np.diff(inps.tbase, axis=0) if inps.ex_date: X = np.dot(A_inv, ts_dis[inps.ex_flag]) else: X = np.dot(A_inv, ts_dis) # Residual vector n resid_n[:, i] = ts_dis - np.dot(A, X) # Update DEM error / timeseries matrix delta_z = X[0] delta_z_mat[row, col] = delta_z if inps.update_timeseries: timeseries[:, i] -= np.dot(A_delta_z, delta_z).flatten() prog_bar.update(i + 1, every=length * width / 100) prog_bar.close() elif inps.incidence_angle.ndim == 1 and inps.range_dis.ndim == 1: print 'inversing using L2-norm minimization (unweighted least squares)'\ ' column by column: %d loops in total' % (width) prog_bar = ptime.progress_bar(maxValue=width, prefix='calculating: ') for i in range(width): range_dis = inps.range_dis[i] inc_angle = inps.incidence_angle[i] # Design matrix - DEM error using pbase, range distance and incidence angle A_delta_z = pbase / (range_dis * np.sin(inc_angle)) if inps.phase_velocity: pbase_v = np.diff(pbase, axis=0) / np.diff(inps.tbase, axis=0) A_delta_z_v = pbase_v / (range_dis * np.sin(inc_angle)) A = np.hstack((A_delta_z_v, A_def)) else: A = np.hstack((A_delta_z, A_def)) # L-2 norm inversion if inps.ex_date: A_inv = np.linalg.pinv(A[inps.ex_flag, :]) else: A_inv = np.linalg.pinv(A) # Get unknown parameters X = [delta_z, vel, acc, delta_acc, ...] ts_dis = timeseries[:, i * length:(i + 1) * length] if inps.phase_velocity: ts_dis = np.diff(ts_dis, axis=0) / np.diff(inps.tbase, axis=0) if inps.ex_date: X = np.dot(A_inv, ts_dis[inps.ex_flag, :]) else: X = np.dot(A_inv, ts_dis) # Residual vector n resid_n[:, i * length:(i + 1) * length] = ts_dis - np.dot(A, X) constC[:, i] = X[1].reshape((1, length)) # Update DEM error / timeseries matrix delta_z = X[0].reshape((1, length)) delta_z_mat[:, i] = delta_z if inps.update_timeseries: timeseries[:, i * length:(i + 1) * length] -= np.dot( A_delta_z, delta_z) prog_bar.update(i + 1, every=width / 100) prog_bar.close() elif inps.incidence_angle.ndim == 0 and inps.range_dis.ndim == 0: print 'inversing using L2-norm minimization (unweighted least squares) for the whole area' # Design matrix - DEM error using pbase, range distance and incidence angle A_delta_z = pbase / (inps.range_dis * np.sin(inps.incidence_angle)) if inps.phase_velocity: pbase_v = np.diff(pbase, axis=0) / np.diff(inps.tbase, axis=0) A_delta_z_v = pbase_v / (inps.range_dis * np.sin(inps.incidence_angle)) A = np.hstack((A_delta_z_v, A_def)) else: A = np.hstack((A_delta_z, A_def)) # L-2 norm inversion if inps.ex_date: A_inv = np.linalg.pinv(A[inps.ex_flag, :]) else: A_inv = np.linalg.pinv(A) # Get unknown parameters X = [delta_z, vel, acc, delta_acc, ...] if inps.phase_velocity: timeseries = np.diff(timeseries, axis=0) / np.diff(inps.tbase, axis=0) if inps.ex_date: X = np.dot(A_inv, timeseries[inps.ex_flag, :]) else: X = np.dot(A_inv, timeseries) # Residual vector n resid_n = ts_dis - np.dot(A, X) # Update DEM error / timeseries matrix delta_z_mat = X[0].reshape((1, length * width)) if inps.update_timeseries: timeseries -= np.dot(A_delta_z, delta_z_mat) delta_z_mat = np.reshape(delta_z_mat, [length, width], order='F') else: print 'ERROR: Script only support same dimension for both incidence angle and range distance matrix.' print 'dimension of incidence angle: ' + str(inps.incidence_angle.ndim) print 'dimension of range distance: ' + str(inps.range_dis.ndim) sys.exit(1) ##------------------------------------------------ Output --------------------------------------------## # DEM error file if 'Y_FIRST' in atr.keys(): dem_error_file = 'demGeo_error.h5' else: dem_error_file = 'demRadar_error.h5' #if inps.phase_velocity: suffix = '_pha_poly'+str(inps.poly_order) #else: suffix = '_vel_poly'+str(inps.poly_order) #dem_error_file = os.path.splitext(dem_error_file)[0]+suffix+os.path.splitext(dem_error_file)[1] print 'writing >>> ' + dem_error_file atr_dem_error = atr.copy() atr_dem_error['FILE_TYPE'] = 'dem' atr_dem_error['UNIT'] = 'm' writefile.write(delta_z_mat, atr_dem_error, dem_error_file) ## Phase Constant C = resid_n[0,:] #atrC = atr.copy() #atrC['FILE_TYPE'] = 'mask' #atrC['UNIT'] = 'm' #writefile.write(constC, atrC, 'constD.h5') ## Corrected DEM file #if inps.dem_file: # inps.dem_outfile = os.path.splitext(inps.dem_file)[0]+suffix+os.path.splitext(inps.dem_file)[1] # print '--------------------------------------' # print 'writing >>> '+inps.dem_outfile # dem, atr_dem = readfile.read(inps.dem_file) # writefile.write(dem+delta_z_mat, atr_dem, inps.dem_outfile) #outfile = 'delta_acc.h5' #print 'writing >>> '+outfile #atr_dem_error = atr.copy() #atr_dem_error['FILE_TYPE'] = 'velocity' #atr_dem_error['UNIT'] = 'm/s' #writefile.write(delta_a_mat, atr_dem_error, outfile) #print '**************************************' # Corrected Time Series if inps.update_timeseries: print 'writing >>> ' + inps.outfile print 'number of dates: ' + str(len(date_list)) h5out = h5py.File(inps.outfile, 'w') group = h5out.create_group('timeseries') prog_bar = ptime.progress_bar(maxValue=date_num, prefix='writing: ') for i in range(date_num): date = date_list[i] d = np.reshape(timeseries[i][:], [length, width], order='F') dset = group.create_dataset(date, data=d, compression='gzip') prog_bar.update(i + 1, suffix=date) prog_bar.close() for key, value in atr.iteritems(): group.attrs[key] = value h5out.close() outFile = os.path.splitext(inps.outfile)[0] + 'InvResid.h5' print 'writing >>> ' + outFile print 'number of dates: ' + str(A_def.shape[0]) h5out = h5py.File(outFile, 'w') group = h5out.create_group('timeseries') prog_bar = ptime.progress_bar(maxValue=A_def.shape[0], prefix='writing: ') for i in range(A_def.shape[0]): date = date_list[i] d = np.reshape(resid_n[i][:], [length, width], order='F') dset = group.create_dataset(date, data=d, compression='gzip') prog_bar.update(i + 1, suffix=date) prog_bar.close() # Attribute for key, value in atr.iteritems(): group.attrs[key] = value if A_def.shape[0] == date_num: group.attrs['UNIT'] = 'm' else: group.attrs['UNIT'] = 'm/yr' h5out.close() return
def ifgram_inversion(ifgramFile='unwrapIfgram.h5', coherenceFile='coherence.h5', meta=None): '''Implementation of the SBAS algorithm. modified from sbas.py written by scott baker, 2012 Inputs: ifgramFile - string, HDF5 file name of the interferograms coherenceFile - string, HDF5 file name of the coherence meta - dict, including the following options: weight_function chunk_size - float, max number of data (ifgram_num*row_num*col_num) to read per loop; to control the memory Output: timeseriesFile - string, HDF5 file name of the output timeseries tempCohFile - string, HDF5 file name of temporal coherence Example: meta = dict() meta['weight_function'] = 'variance' meta['chunk_size'] = 0.5e9 meta['timeseriesFile'] = 'timeseries_var.h5' meta['tempCohFile'] = 'temporalCoherence_var.h5' ifgram_inversion('unwrapIfgram.h5', 'coherence.h5', meta) ''' if 'tempCohFile' not in meta.keys(): meta['tempCohFile'] = 'temporalCoherence.h5' meta['timeseriesStdFile'] = 'timeseriesDecorStd.h5' total = time.time() if not meta: meta = vars(cmdLineParse()) if meta['update_mode'] and not ut.update_file(meta['timeseriesFile'], ifgramFile): return meta['timeseriesFile'], meta['tempCohFile'] ##### Basic Info # length/width atr = readfile.read_attribute(ifgramFile) length = int(atr['FILE_LENGTH']) width = int(atr['WIDTH']) meta['length'] = length meta['width'] = width # ifgram_list h5ifgram = h5py.File(ifgramFile,'r') ifgram_list = sorted(h5ifgram['interferograms'].keys()) #if meta['weight_function'] in ['no','uniform']: # ifgram_list = ut.check_drop_ifgram(h5ifgram) ifgram_list = ut.check_drop_ifgram(h5ifgram) meta['ifgram_list'] = ifgram_list ifgram_num = len(ifgram_list) # date12_list/date8_list/tbase_diff date12_list = ptime.list_ifgram2date12(ifgram_list) m_dates = [i.split('-')[0] for i in date12_list] s_dates = [i.split('-')[1] for i in date12_list] date8_list = ptime.yyyymmdd(sorted(list(set(m_dates + s_dates)))) date_num = len(date8_list) meta['date8_list'] = date8_list meta['date12_list'] = date12_list tbase_list = ptime.date_list2tbase(date8_list)[0] tbase_diff = np.diff(tbase_list).reshape((-1,1)) meta['tbase_diff'] = tbase_diff print 'number of interferograms: %d' % (ifgram_num) print 'number of acquisitions : %d' % (date_num) print 'number of columns: %d' % (width) print 'number of lines : %d' % (length) ##### ref_y/x/value try: ref_x = int(atr['ref_x']) ref_y = int(atr['ref_y']) print 'reference pixel in y/x: [%d, %d]' % (ref_y, ref_x) ref_value = np.zeros((ifgram_num,1), np.float32) for j in range(ifgram_num): ifgram = ifgram_list[j] dset = h5ifgram['interferograms'][ifgram].get(ifgram) ref_value[j] = dset[ref_y,ref_x] meta['ref_y'] = ref_y meta['ref_x'] = ref_x meta['ref_value'] = ref_value except: if meta['skip_ref']: meta['ref_value'] = 0.0 print 'skip checking reference pixel info - This is for SIMULATION ONLY.' else: print 'ERROR: No ref_x/y found! Can not invert interferograms without reference in space.' print 'run seed_data.py '+ifgramFile+' --mark-attribute for a quick referencing.' sys.exit(1) h5ifgram.close() ##### Rank of Design matrix for weighted inversion A, B = ut.design_matrix(ifgramFile, date12_list) print '-------------------------------------------------------------------------------' if meta['weight_function'] in ['no','uniform']: print 'generic least square inversion with min-norm phase velocity' print ' based on Berardino et al. (2002, IEEE-TGRS)' print ' OLS for pixels with fully connected network' print ' SVD for pixels with partially connected network' if np.linalg.matrix_rank(A) < date_num-1: print 'WARNING: singular design matrix! Inversion result can be biased!' print 'continue using its SVD solution on all pixels' else: print 'weighted least square (WLS) inversion with min-norm phase, pixelwise' if np.linalg.matrix_rank(A) < date_num-1: print 'ERROR: singular design matrix!' print ' Input network of interferograms is not fully connected!' print ' Can not invert the weighted least square solution.' print 'You could try:' print ' 1) Add more interferograms to make the network fully connected:' print ' a.k.a., no multiple subsets nor network islands' print " 2) Use '-w no' option for non-weighted SVD solution." sys.exit(-1) print '-------------------------------------------------------------------------------' ##### Invert time-series phase ##Check parallel environment if meta['weight_function'] in ['no','uniform']: meta['parallel'] = False if meta['parallel']: num_cores, meta['parallel'], Parallel, delayed = ut.check_parallel(1000, print_msg=False) ##Split into chunks to reduce memory usage r_step = meta['chunk_size']/ifgram_num/width #split in lines if meta['weight_function'] not in ['no','uniform']: #more memory usage (coherence) for WLS r_step /= 2.0 if meta['parallel']: r_step /= num_cores r_step = int(ceil_to_1(r_step)) meta['row_step'] = r_step chunk_num = int((length-1)/r_step)+1 if chunk_num > 1: print 'maximum chunk size: %.1E' % (meta['chunk_size']) print 'split %d lines into %d patches for processing' % (length, chunk_num) print ' with each patch up to %d lines' % (r_step) if meta['parallel']: print 'parallel processing using %d cores ...' % (min([num_cores,chunk_num])) ##Computing the inversion box_list = [] for i in range(chunk_num): r0 = i*r_step r1 = min([length, r0+r_step]) box = (0,r0,width,r1) box_list.append(box) box_num = len(box_list) if not meta['parallel']: timeseries = np.zeros((date_num, length, width), np.float32) timeseriesStd = np.zeros((date_num, length, width), np.float32) tempCoh = np.zeros((length, width), np.float32) for i in range(box_num): if box_num > 1: print '\n------- Processing Patch %d out of %d --------------' % (i+1, box_num) box = box_list[i] ts, tcoh, tsStd = ifgram_inversion_patch(ifgramFile, coherenceFile, meta, box) tempCoh[box[1]:box[3],box[0]:box[2]] = tcoh timeseries[:,box[1]:box[3],box[0]:box[2]] = ts timeseriesStd[:,box[1]:box[3],box[0]:box[2]] = tsStd else: ##Temp file list meta['ftemp_base'] = 'timeseries_temp_' temp_file_list = [meta['ftemp_base']+str(i)+'.h5' for i in range(chunk_num)] ##Computation Parallel(n_jobs=num_cores)(delayed(ifgram_inversion_patch)\ (ifgramFile, coherenceFile, meta, box) for box in box_list) ##Concatenate temp files print 'concatenating temporary timeseries files ...' timeseries = np.zeros((date_num, length, width), np.float32) tempCoh = np.zeros((length, width), np.float32) rmCmd = 'rm' for i in range(chunk_num): fname = temp_file_list[i] box = box_list[i] print 'reading '+fname h5temp = h5py.File(fname, 'r') dset = h5temp['timeseries'].get('timeseries') timeseries[:,box[1]:box[3],box[0]:box[2]] = dset[0:-1,:,:] tempCoh[box[1]:box[3],box[0]:box[2]] = dset[-1,:,:] h5temp.close() rmCmd += ' '+fname print rmCmd os.system(rmCmd) print 'converting phase to range' phase2range = -1*float(atr['WAVELENGTH'])/(4.*np.pi) timeseries *= phase2range timeseriesStd *= abs(phase2range) ##### Calculate time-series attributes print 'calculating perpendicular baseline timeseries' pbase, pbase_top, pbase_bottom = ut.perp_baseline_ifgram2timeseries(ifgramFile, ifgram_list) pbase = str(pbase.tolist()).translate(None,'[],') # convert np.array into string separated by white space pbase_top = str(pbase_top.tolist()).translate(None,'[],') pbase_bottom = str(pbase_bottom.tolist()).translate(None,'[],') atr['P_BASELINE_TIMESERIES'] = pbase atr['P_BASELINE_TOP_TIMESERIES'] = pbase_top atr['P_BASELINE_BOTTOM_TIMESERIES'] = pbase_bottom atr['ref_date'] = date8_list[0] atr['FILE_TYPE'] = 'timeseries' atr['UNIT'] = 'm' ##### Output ## 1. Write time-series file meta['timeseriesFile'] = write_timeseries_hdf5_file(timeseries, date8_list, atr,\ timeseriesFile=meta['timeseriesFile']) if not np.all(timeseriesStd == 0.): meta['timeseriesStdFile'] = write_timeseries_hdf5_file(timeseriesStd, date8_list, atr,\ timeseriesFile=meta['timeseriesStdFile']) ## 2. Write Temporal Coherence File print 'writing >>> '+meta['tempCohFile'] atr['FILE_TYPE'] = 'temporal_coherence' atr['UNIT'] = '1' meta['tempCohFile'] = writefile.write(tempCoh, atr, meta['tempCohFile']) print 'Time series inversion took ' + str(time.time()-total) +' secs\nDone.' return meta['timeseriesFile'], meta['tempCohFile']
def timeseries_inversion(ifgramFile='unwrapIfgram.h5', coherenceFile='coherence.h5', inps_dict=None): '''Implementation of the SBAS algorithm. modified from sbas.py written by scott baker, 2012 Inputs: ifgramFile - string, HDF5 file name of the interferograms coherenceFile - string, HDF5 file name of the coherence inps_dict - dict, including the following options: weight_function min_coherence max_coherence Output: timeseriesFile - string, HDF5 file name of the output timeseries tempCohFile - string, HDF5 file name of temporal coherence ''' total = time.time() if not inps_dict: inps_dict = vars(cmdLineParse()) weight_func = inps_dict['weight_function'] min_coh = inps_dict['min_coherence'] max_coh = inps_dict['max_coherence'] # Basic Info atr = readfile.read_attribute(ifgramFile) length = int(atr['FILE_LENGTH']) width = int(atr['WIDTH']) pixel_num = length * width h5ifgram = h5py.File(ifgramFile, 'r') ifgram_list = sorted(h5ifgram['interferograms'].keys()) if inps_dict['weight_function'] == 'no': ifgram_list = ut.check_drop_ifgram(h5ifgram, atr, ifgram_list) ifgram_num = len(ifgram_list) # Convert ifgram_list to date12/8_list date12_list = ptime.list_ifgram2date12(ifgram_list) m_dates = [i.split('-')[0] for i in date12_list] s_dates = [i.split('-')[1] for i in date12_list] date8_list = ptime.yyyymmdd(sorted(list(set(m_dates + s_dates)))) date_num = len(date8_list) tbase_list = ptime.date_list2tbase(date8_list)[0] tbase_diff = np.diff(tbase_list).reshape((date_num - 1, 1)) print 'number of interferograms: ' + str(ifgram_num) print 'number of acquisitions : ' + str(date_num) print 'number of pixels: ' + str(pixel_num) # Reference pixel in space try: ref_x = int(atr['ref_x']) ref_y = int(atr['ref_y']) print 'reference pixel in y/x: [%d, %d]' % (ref_y, ref_x) except: print 'ERROR: No ref_x/y found! Can not inverse interferograms without reference in space.' print 'run seed_data.py ' + ifgramFile + ' --mark-attribute for a quick referencing.' sys.exit(1) ##### Read Interferograms print 'reading interferograms ...' ifgram_data = np.zeros((ifgram_num, pixel_num), np.float32) prog_bar = ptime.progress_bar(maxValue=ifgram_num) for j in range(ifgram_num): ifgram = ifgram_list[j] d = h5ifgram['interferograms'][ifgram].get(ifgram)[:] #d[d != 0.] -= d[ref_y, ref_x] d -= d[ref_y, ref_x] ifgram_data[j] = d.flatten() prog_bar.update(j + 1, suffix=date12_list[j]) h5ifgram.close() prog_bar.close() #####---------------------- Inversion ----------------------##### # Design matrix A, B = ut.design_matrix(ifgramFile, date12_list) if weight_func == 'no': print 'generalized inversion using SVD (Berardino et al., 2002, IEEE-TGRS)' print 'inversing time series ...' B_inv = np.array(np.linalg.pinv(B), np.float32) ts_rate = np.dot(B_inv, ifgram_data) ts1 = ts_rate * np.tile(tbase_diff, (1, pixel_num)) ts0 = np.array([0.] * pixel_num, np.float32) ts_data = np.vstack((ts0, np.cumsum(ts1, axis=0))) del ts_rate, ts0, ts1 # Temporal coherence print 'calculating temporal coherence (Tizzani et al., 2007, RSE)' temp_coh = np.zeros((1, pixel_num), np.float32) + 0j prog_bar = ptime.progress_bar(maxValue=ifgram_num) for i in range(ifgram_num): ifgram_est = np.dot(A[i, :], ts_data[1:, :]) ifgram_diff = ifgram_data[i, :] - ifgram_est temp_coh += np.exp(1j * ifgram_diff) prog_bar.update(i + 1, suffix=date12_list[i]) prog_bar.close() del ifgram_data, ifgram_est, ifgram_diff temp_coh = np.array((np.absolute(temp_coh) / ifgram_num).reshape( (length, width)), dtype=np.float32) else: print 'weighted least square (WLS) inversion using coherence pixel by pixel' if np.linalg.matrix_rank(A) < date_num - 1: print 'ERROR: singular design matrix!' print ' Input network of interferograms is not fully connected!' print ' Can not inverse the weighted least square solution.' print 'You could try:' print ' 1) Add more interferograms to make the network fully connected:' print ' a.k.a., no multiple subsets nor network islands' print " 2) Use '-w no' option for non-weighted SVD solution." sys.exit(-1) pixel_mask = np.ones(pixel_num, np.bool_) print 'reading coherence: ' + os.path.basename(coherenceFile) h5coh = h5py.File(coherenceFile, 'r') coh_list = sorted(h5coh['coherence'].keys()) coh_data = np.zeros((ifgram_num, pixel_num), np.float32) prog_bar = ptime.progress_bar(maxValue=ifgram_num) for j in range(ifgram_num): ifgram = coh_list[j] d = h5coh['coherence'][ifgram].get(ifgram)[:].flatten() d[np.isnan(d)] = 0. pixel_mask[d == 0.] = 0 coh_data[j] = d prog_bar.update(j + 1, suffix=date12_list[j]) h5coh.close() prog_bar.close() # Get mask of valid pixels to inverse print 'skip pixels with zero coherence in at least one interferogram' print 'skip pixels with zero phase in all interferograms' ifgram_stack = ut.get_file_stack(ifgramFile).flatten() pixel_mask[ifgram_stack == 0.] = 0 pixel_num2inv = np.sum(pixel_mask) pixel_idx2inv = np.where(pixel_mask)[0] ifgram_data = ifgram_data[:, pixel_mask] coh_data = coh_data[:, pixel_mask] print 'number of pixels to inverse: %d' % (pixel_num2inv) ##### Calculate Weight matrix weight = coh_data if weight_func.startswith('var'): print 'convert coherence to weight using inverse of variance: x**2/(1-x**2) from Hanssen (2001, for 4.2.32)' weight[weight > 0.999] = 0.999 if weight_func == 'variance-max-coherence': print 'constrain the max coherence to %f' % max_coh weight[weight > max_coh] = max_coh weight = np.square(weight) weight *= 1. / (1. - weight) if weight_func == 'variance-log': print 'use log(1/variance)+1 as weight' weight = np.log(weight + 1) elif weight_func.startswith('lin'): print 'use coherence as weight directly (Tong et al., 2016, RSE)' elif weight_func.startswith('norm'): print 'convert coherence to weight using CDF of normal distribution: N(%f, %f)' % ( mu, std) mu = (min_coh + max_coh) / 2.0 std = (max_coh - min_coh) / 6.0 chunk_size = 1000 chunk_num = int(pixel_num2inv / chunk_size) + 1 prog_bar = ptime.progress_bar(maxValue=chunk_num) for i in range(chunk_num): i0 = (i - 1) * chunk_size i1 = min([pixel_num2inv, i0 + chunk_size]) weight[:, i0:i1] = norm.cdf(weight[:, i0:i1], mu, std) prog_bar.update(i + 1, every=10) prog_bar.close() #weight = norm.cdf(weight, mu, std) else: print 'Un-recognized weight function: %s' % weight_func sys.exit(-1) ##### Weighted Inversion pixel by pixel print 'inversing time series ...' ts_data = np.zeros((date_num, pixel_num), np.float32) temp_coh = np.zeros(pixel_num, np.float32) prog_bar = ptime.progress_bar(maxValue=pixel_num2inv) for i in range(pixel_num2inv): # Inverse timeseries ifgram_pixel = ifgram_data[:, i] weight_pixel = weight[:, i] W = np.diag(weight_pixel) ts = np.linalg.inv(A.T.dot(W).dot(A)).dot( A.T).dot(W).dot(ifgram_pixel) ts_data[1:, pixel_idx2inv[i]] = ts # Calculate weighted temporal coherence ifgram_diff = ifgram_pixel - np.dot(A, ts) temp_coh_pixel = np.abs( np.sum(np.multiply(weight_pixel, np.exp(1j * ifgram_diff)), axis=0)) / np.sum(weight_pixel) temp_coh[pixel_idx2inv[i]] = temp_coh_pixel prog_bar.update(i + 1, every=2000, suffix=str(i + 1) + ' pixels') prog_bar.close() del ifgram_data, weight #####---------------------- Outputs ----------------------##### ## 1.1 Convert time-series phase to displacement print 'converting phase to range' phase2range = -1 * float(atr['WAVELENGTH']) / (4. * np.pi) ts_data *= phase2range ## 1.2 Write time-series data matrix timeseriesFile = 'timeseries.h5' print 'writing >>> ' + timeseriesFile print 'number of acquisitions: ' + str(date_num) h5timeseries = h5py.File(timeseriesFile, 'w') group = h5timeseries.create_group('timeseries') prog_bar = ptime.progress_bar(maxValue=date_num) for i in range(date_num): date = date8_list[i] dset = group.create_dataset(date, data=ts_data[i].reshape(length, width), compression='gzip') prog_bar.update(i + 1, suffix=date) prog_bar.close() ## 1.3 Write time-series attributes print 'calculating perpendicular baseline timeseries' pbase, pbase_top, pbase_bottom = ut.perp_baseline_ifgram2timeseries( ifgramFile, ifgram_list) pbase = str(pbase.tolist()).translate( None, '[],') # convert np.array into string separated by white space pbase_top = str(pbase_top.tolist()).translate(None, '[],') pbase_bottom = str(pbase_bottom.tolist()).translate(None, '[],') atr['P_BASELINE_TIMESERIES'] = pbase atr['P_BASELINE_TOP_TIMESERIES'] = pbase_top atr['P_BASELINE_BOTTOM_TIMESERIES'] = pbase_bottom atr['ref_date'] = date8_list[0] atr['FILE_TYPE'] = 'timeseries' atr['UNIT'] = 'm' for key, value in atr.iteritems(): group.attrs[key] = value h5timeseries.close() del ts_data ## 2. Write Temporal Coherence File tempCohFile = 'temporalCoherence.h5' print 'writing >>> ' + tempCohFile atr['FILE_TYPE'] = 'temporal_coherence' atr['UNIT'] = '1' writefile.write(temp_coh.reshape(length, width), atr, tempCohFile) print 'Time series inversion took ' + str(time.time() - total) + ' secs\nDone.' return timeseriesFile, tempCohFile