def reconstruct_igrams_from_timeseries(h5timeseries,h5igrams): dateList = h5timeseries['timeseries'].keys() dateIndex={} for ni in range(len(dateList)): dateIndex[dateList[ni]]=ni dset = h5timeseries['timeseries'].get(h5timeseries['timeseries'].keys()[0]) nrows,ncols=shape(dset) timeseries = zeros((len(h5timeseries['timeseries'].keys()),shape(dset)[0]*shape(dset)[1]),float32) for date in dateList: dset = h5timeseries['timeseries'].get(date) d = dset[0:dset.shape[0],0:dset.shape[1]] timeseries[dateIndex[date]][:]=d.flatten(0) del d lt,numpixels=shape(timeseries) A,B = ut.design_matrix(h5igrams) lam = float(h5timeseries['timeseries'].attrs['WAVELENGTH']) range2phase=-4*pi/lam timeseries=range2phase*timeseries p=-1*ones([A.shape[0],1]) Ap=hstack((p,A)) estData=dot(Ap,timeseries) return estData,nrows,ncols
def temporal_coherence(timeseriesFile, ifgramFile): '''Calculate temporal coherence based on input timeseries file and interferograms file Inputs: timeseriesFile - string, path of time series file ifgramFile - string, path of interferograms file Output: temp_coh - 2D np.array, temporal coherence in float32 ''' # Basic Info atr_ts = readfile.read_attribute(timeseriesFile) length = int(atr_ts['FILE_LENGTH']) width = int(atr_ts['WIDTH']) pixel_num = length * width # Read time series data h5timeseries = h5py.File(timeseriesFile, 'r') date_list = sorted(h5timeseries['timeseries'].keys()) date_num = len(date_list) print "load time series: " + timeseriesFile print 'number of acquisitions: ' + str(date_num) timeseries = np.zeros((date_num, pixel_num), np.float32) prog_bar = ptime.progress_bar(maxValue=date_num, prefix='loading: ') for i in range(date_num): date = date_list[i] d = h5timeseries['timeseries'].get(date)[:] timeseries[i][:] = d.flatten(0) prog_bar.update(i + 1, suffix=date) prog_bar.close() h5timeseries.close() # Convert displacement from meter to radian range2phase = -4 * np.pi / float(atr_ts['WAVELENGTH']) timeseries *= range2phase # interferograms data print "interferograms file: " + ifgramFile atr_ifgram = readfile.read_attribute(ifgramFile) h5ifgram = h5py.File(ifgramFile, 'r') ifgram_list = sorted(h5ifgram['interferograms'].keys()) ifgram_list = ut.check_drop_ifgram(h5ifgram, atr_ifgram, ifgram_list) ifgram_num = len(ifgram_list) # Design matrix date12_list = ptime.list_ifgram2date12(ifgram_list) A1, B = ut.design_matrix(ifgramFile, date12_list) A0 = -1 * np.ones([ifgram_num, 1]) A = np.hstack((A0, A1)) # Get reference pixel try: ref_x = int(atr_ts['ref_x']) ref_y = int(atr_ts['ref_y']) print 'find reference pixel in y/x: [%d, %d]' % (ref_y, ref_x) except ValueError: print 'No ref_x/y found! Can not calculate temporal coherence without it.' print 'calculating temporal coherence interferogram by interferogram ...' print 'number of interferograms: ' + str(ifgram_num) temp_coh = np.zeros(pixel_num, dtype=np.float32) + 0j prog_bar = ptime.progress_bar(maxValue=ifgram_num, prefix='calculating: ') for i in range(ifgram_num): ifgram = ifgram_list[i] # read interferogram data = h5ifgram['interferograms'][ifgram].get(ifgram)[:] data -= data[ref_y, ref_x] data = data.flatten(0) # calculate difference between observed and estimated data dataEst = np.dot(A[i, :], timeseries) dataDiff = data - dataEst temp_coh += np.exp(1j * dataDiff) prog_bar.update(i + 1, suffix=date12_list[i]) prog_bar.close() del timeseries, data, dataEst, dataDiff h5ifgram.close() temp_coh = np.array((np.absolute(temp_coh) / ifgram_num).reshape( (length, width)), dtype=np.float32) return temp_coh
def main(argv): method = 'triangular_consistency' ## or 'bonding_point' ramp_type = 'plane' save_rampCor = 'yes' plot_bonding_points = 'yes' ##### Check Inputs if len(sys.argv) > 2: try: opts, args = getopt.getopt(argv, 'h:f:m:x:y:o:t:', ['ramp=', 'no-ramp-save']) except getopt.GetoptError: print 'Error while getting args' usage() sys.exit(1) for opt, arg in opts: if opt in ['-h', '--help']: usage() sys.exit() elif opt in '-f': File = arg elif opt in '-m': maskFile = arg elif opt in '-o': outName = arg elif opt in '-x': x = [int(i) for i in arg.split(',')] method = 'bonding_point' elif opt in '-y': y = [int(i) for i in arg.split(',')] method = 'bonding_point' elif opt in '-t': templateFile = arg elif opt in '--ramp': ramp_type = arg.lower() elif opt in '--no-ramp-save': save_rampCor = 'no' elif len(sys.argv) == 2: if argv[0] in ['-h', '--help']: usage() sys.exit() elif os.path.isfile(argv[0]): File = argv[0] maskFile = argv[1] else: print 'Input file does not existed: ' + argv[0] sys.exit(1) else: usage() sys.exit(1) ##### Check template file try: templateFile templateContents = readfile.read_template(templateFile) except: pass try: yx = [ int(i) for i in templateContents['pysar.unwrapError.yx'].split(',') ] x = yx[1::2] y = yx[0::2] method = 'bonding_point' except: pass ##### Read Mask File ## Priority: ## Input mask file > pysar.mask.file > existed Modified_Mask.h5 > existed Mask.h5 try: maskFile except: try: maskFile = templateContents['pysar.mask.file'] except: if os.path.isfile('Modified_Mask.h5'): maskFile = 'Modified_Mask.h5' elif os.path.isfile('Mask.h5'): maskFile = 'Mask.h5' else: print 'No mask found!' sys.exit(1) try: Mask, Matr = readfile.read(maskFile) print 'mask: ' + maskFile except: print 'Can not open mask file: ' + maskFile sys.exit(1) ##### Output file name ext = os.path.splitext(File)[1] try: outName except: outName = File.split('.')[0] + '_unwCor' + ext print '\n**************** Unwrapping Error Correction ******************' #################### Triangular Consistency (Phase Closure) #################### if method == 'triangular_consistency': print 'Phase unwrapping error correction using Triangular Consistency / Phase Closure' h5file = h5py.File(File) ifgramList = h5file['interferograms'].keys() sx = int(h5file['interferograms'][ifgramList[0]].attrs['WIDTH']) sy = int(h5file['interferograms'][ifgramList[0]].attrs['FILE_LENGTH']) curls, Triangles, C = ut.get_triangles(h5file) A, B = ut.design_matrix(h5file) ligram, lv = np.shape(B) lcurls = np.shape(curls)[0] print 'Number of all triangles: ' + str(lcurls) print 'Number of interferograms: ' + str(ligram) #print curls curlfile = 'curls.h5' if not os.path.isfile(curlfile): ut.generate_curls(curlfile, h5file, Triangles, curls) thr = 0.50 curls = np.array(curls) n1 = curls[:, 0] n2 = curls[:, 1] n3 = curls[:, 2] numPixels = sy * sx print 'reading interferograms...' data = np.zeros((ligram, numPixels), np.float32) for ni in range(ligram): dset = h5file['interferograms'][ifgramList[ni]].get(ifgramList[ni]) d = dset[0:dset.shape[0], 0:dset.shape[1]] data[ni] = d.flatten(1) print np.shape(data) print 'reading curls ...' h5curl = h5py.File(curlfile) curlList = h5curl['interferograms'].keys() curlData = np.zeros((lcurls, numPixels), np.float32) for ni in range(lcurls): dset = h5curl['interferograms'][curlList[ni]].get(curlList[ni]) d = dset[0:dset.shape[0], 0:dset.shape[1]] curlData[ni] = d.flatten(1) pi = np.pi EstUnwrap = np.zeros((ligram, numPixels), np.float32) #try: # maskFile=argv[1] # h5Mask=h5py.File(maskFile) # dset = h5Mask['mask'].get('mask') # Mask=dset[0:dset.shape[0],0:dset.shape[1]] #except: # dset = h5file['mask'].get('mask') # Mask=dset[0:dset.shape[0],0:dset.shape[1]] Mask = Mask.flatten(1) for ni in range(numPixels): #dU = np.zeros([ligram,1]) #print np.shape(dU) #print np.shape(data[:,ni]) if Mask[ni] == 1: dU = data[:, ni] #nan_ndx = dataPoint == 0. unwCurl = np.array(curlData[:, ni]) #print unwCurl ind = np.abs(unwCurl) >= thr N1 = n1[ind] N2 = n2[ind] N3 = n3[ind] indC = np.abs(unwCurl) < thr Nc1 = n1[indC] Nc2 = n2[indC] Nc3 = n3[indC] N = np.hstack([N1, N2, N3]) UniN = np.unique(N) Nc = np.hstack([Nc1, Nc2, Nc3]) UniNc = np.unique(Nc) inter = list(set(UniNc) & set(UniN)) # intersetion UniNc = list(UniNc) for x in inter: UniNc.remove(x) D = np.zeros([len(UniNc), ligram]) for i in range(len(UniNc)): D[i, UniNc[i]] = 1 AAA = np.vstack([-2 * pi * C, D]) #AAA1=np.hstack([AAA,np.zeros([AAA.shape[0],lv])]) #AAA2=np.hstack([-2*pi*np.eye(ligram),B]) #AAAA=np.vstack([AAA1,AAA2]) AAAA = np.vstack([AAA, 0.25 * np.eye(ligram)]) #print '************************' #print np.linalg.matrix_rank(C) #print np.linalg.matrix_rank(AAA) #print np.linalg.matrix_rank(AAAA) #print '************************' #LLL=list(np.dot(C,dU)) + list(np.zeros(np.shape(UniNc)[0]))# + list(dU) #ind=np.isnan(AAA) #M1=pinv(AAA) #M=np.dot(M1,LLL) #EstUnwrap[:,ni]=np.round(M[0:ligram])*2.0*np.pi ########## # with Tikhonov regularization: AAAA = np.vstack([AAA, 0.25 * np.eye(ligram)]) LLL = list(np.dot(C, dU)) + list(np.zeros( np.shape(UniNc)[0])) + list(np.zeros(ligram)) ind = np.isnan(AAAA) M1 = pinv(AAAA) M = np.dot(M1, LLL) EstUnwrap[:, ni] = np.round(M[0:ligram]) * 2.0 * np.pi #print M[0:ligram] #print np.round(M[0:ligram]) else: EstUnwrap[:, ni] = np.zeros([ligram]) if not np.remainder(ni, 10000): print 'Processing point: %7d of %7d ' % (ni, numPixels) ##### Output dataCor = data + EstUnwrap unwCorFile = File.replace('.h5', '') + '_unwCor.h5' print 'writing >>> ' + unwCorFile h5unwCor = h5py.File(unwCorFile, 'w') gg = h5unwCor.create_group('interferograms') for i in range(ligram): group = gg.create_group(ifgramList[i]) dset = group.create_dataset(ifgramList[i], data=np.reshape( dataCor[i, :], [sx, sy]).T, compression='gzip') for key, value in h5file['interferograms'][ ifgramList[i]].attrs.iteritems(): group.attrs[key] = value try: MASK = h5file['mask'].get('mask') gm = h5unwCor.create_group('mask') dset = gm.create_dataset('mask', data=MASK, compression='gzip') except: pass h5unwCor.close() h5file.close() h5curl.close() #################### Bonding Points (Spatial Continuity) #################### elif method == 'bonding_point': print 'Phase unwrapping error correction using Bonding Points / Spatial Continuity' ##### Read Bridge Points Info try: x y if len(x) != len(y) or np.mod(len(x), 2) != 0: print 'Wrong number of bridge points input: ' + str( len(x)) + ' for x, ' + str(len(y)) + ' for y' usage() sys.exit(1) except: print 'Error in reading bridge points info!' usage() sys.exit(1) for i in range(0, len(x)): if Mask[y[i], x[i]] == 0: print '\nERROR: Connecting point (' + str(y[i]) + ',' + str( x[i]) + ') is out of masked area! Select them again!\n' sys.exit(1) print 'Number of bonding point pairs: ' + str(len(x) / 2) print 'Bonding points coordinates:\nx: ' + str(x) + '\ny: ' + str(y) ## Plot Connecting Pair of Points if plot_bonding_points == 'yes': point_yx = '' line_yx = '' n_bridge = len(x) / 2 for i in range(n_bridge): pair_yx = str(y[2 * i]) + ',' + str(x[2 * i]) + ',' + str( y[2 * i + 1]) + ',' + str(x[2 * i + 1]) if not i == n_bridge - 1: point_yx += pair_yx + ',' line_yx += pair_yx + ';' else: point_yx += pair_yx line_yx += pair_yx try: plot_cmd = 'view.py --point-yx="'+point_yx+'" --line-yx="'+line_yx+\ '" --nodisplay -o bonding_points.png -f '+maskFile print plot_cmd os.system(plot_cmd) except: pass ##### Ramp Info ramp_mask = Mask == 1 print 'estimate phase ramp during the correction' print 'ramp type: ' + ramp_type if save_rampCor == 'yes': outName_ramp = os.path.basename(outName).split( ext)[0] + '_' + ramp_type + ext ########## PySAR ########## if ext == '.h5': ##### Read try: h5file = h5py.File(File, 'r') except: print 'ERROR: Cannot open input file: ' + File sys.exit(1) k = h5file.keys() if 'interferograms' in k: k[0] = 'interferograms' print 'Input file is ' + k[0] else: print 'Input file - ' + File + ' - is not interferograms.' usage() sys.exit(1) igramList = sorted(h5file[k[0]].keys()) #### Write h5out = h5py.File(outName, 'w') gg = h5out.create_group(k[0]) print 'writing >>> ' + outName if save_rampCor == 'yes': h5out_ramp = h5py.File(outName_ramp, 'w') gg_ramp = h5out_ramp.create_group(k[0]) print 'writing >>> ' + outName_ramp ##### Loop print 'Number of interferograms: ' + str(len(igramList)) for igram in igramList: print igram data = h5file[k[0]][igram].get(igram)[:] data_ramp, ramp = rm.remove_data_surface( data, ramp_mask, ramp_type) #ramp = data_ramp - data data_rampCor = phase_bonding(data_ramp, Mask, x, y) dataCor = data_rampCor - ramp group = gg.create_group(igram) dset = group.create_dataset(igram, data=dataCor, compression='gzip') for key, value in h5file[k[0]][igram].attrs.iteritems(): group.attrs[key] = value if save_rampCor == 'yes': group_ramp = gg_ramp.create_group(igram) dset = group_ramp.create_dataset(igram, data=data_rampCor, compression='gzip') for key, value in h5file[k[0]][igram].attrs.iteritems(): group_ramp.attrs[key] = value try: mask = h5file['mask'].get('mask') gm = h5out.create_group('mask') dset = gm.create_dataset('mask', data=mask[0:mask.shape[0], 0:mask.shape[1]], compression='gzip') except: print 'no mask group found.' h5file.close() h5out.close() if save_rampCor == 'yes': h5out_ramp.close() ########## ROI_PAC ########## elif ext == '.unw': print 'Input file is ' + ext a, data, atr = readfile.read_float32(File) data_ramp, ramp = rm.remove_data_surface(data, ramp_mask, ramp_type) #ramp = data_ramp - data data_rampCor = phase_bonding(data_ramp, Mask, x, y) dataCor = data_rampCor - ramp writefile.write(dataCor, atr, outName) if save_rampCor == 'yes': writefile.write(data_rampCor, atr, outName_ramp) else: print 'Un-supported file type: ' + ext usage() sys.exit(1)
def ifgram_inversion(ifgramFile='unwrapIfgram.h5', coherenceFile='coherence.h5', meta=None): '''Implementation of the SBAS algorithm. modified from sbas.py written by scott baker, 2012 Inputs: ifgramFile - string, HDF5 file name of the interferograms coherenceFile - string, HDF5 file name of the coherence meta - dict, including the following options: weight_function chunk_size - float, max number of data (ifgram_num*row_num*col_num) to read per loop; to control the memory Output: timeseriesFile - string, HDF5 file name of the output timeseries tempCohFile - string, HDF5 file name of temporal coherence Example: meta = dict() meta['weight_function'] = 'variance' meta['chunk_size'] = 0.5e9 meta['timeseriesFile'] = 'timeseries_var.h5' meta['tempCohFile'] = 'temporalCoherence_var.h5' ifgram_inversion('unwrapIfgram.h5', 'coherence.h5', meta) ''' if 'tempCohFile' not in meta.keys(): meta['tempCohFile'] = 'temporalCoherence.h5' meta['timeseriesStdFile'] = 'timeseriesDecorStd.h5' total = time.time() if not meta: meta = vars(cmdLineParse()) if meta['update_mode'] and not ut.update_file(meta['timeseriesFile'], ifgramFile): return meta['timeseriesFile'], meta['tempCohFile'] ##### Basic Info # length/width atr = readfile.read_attribute(ifgramFile) length = int(atr['FILE_LENGTH']) width = int(atr['WIDTH']) meta['length'] = length meta['width'] = width # ifgram_list h5ifgram = h5py.File(ifgramFile,'r') ifgram_list = sorted(h5ifgram['interferograms'].keys()) #if meta['weight_function'] in ['no','uniform']: # ifgram_list = ut.check_drop_ifgram(h5ifgram) ifgram_list = ut.check_drop_ifgram(h5ifgram) meta['ifgram_list'] = ifgram_list ifgram_num = len(ifgram_list) # date12_list/date8_list/tbase_diff date12_list = ptime.list_ifgram2date12(ifgram_list) m_dates = [i.split('-')[0] for i in date12_list] s_dates = [i.split('-')[1] for i in date12_list] date8_list = ptime.yyyymmdd(sorted(list(set(m_dates + s_dates)))) date_num = len(date8_list) meta['date8_list'] = date8_list meta['date12_list'] = date12_list tbase_list = ptime.date_list2tbase(date8_list)[0] tbase_diff = np.diff(tbase_list).reshape((-1,1)) meta['tbase_diff'] = tbase_diff print 'number of interferograms: %d' % (ifgram_num) print 'number of acquisitions : %d' % (date_num) print 'number of columns: %d' % (width) print 'number of lines : %d' % (length) ##### ref_y/x/value try: ref_x = int(atr['ref_x']) ref_y = int(atr['ref_y']) print 'reference pixel in y/x: [%d, %d]' % (ref_y, ref_x) ref_value = np.zeros((ifgram_num,1), np.float32) for j in range(ifgram_num): ifgram = ifgram_list[j] dset = h5ifgram['interferograms'][ifgram].get(ifgram) ref_value[j] = dset[ref_y,ref_x] meta['ref_y'] = ref_y meta['ref_x'] = ref_x meta['ref_value'] = ref_value except: if meta['skip_ref']: meta['ref_value'] = 0.0 print 'skip checking reference pixel info - This is for SIMULATION ONLY.' else: print 'ERROR: No ref_x/y found! Can not invert interferograms without reference in space.' print 'run seed_data.py '+ifgramFile+' --mark-attribute for a quick referencing.' sys.exit(1) h5ifgram.close() ##### Rank of Design matrix for weighted inversion A, B = ut.design_matrix(ifgramFile, date12_list) print '-------------------------------------------------------------------------------' if meta['weight_function'] in ['no','uniform']: print 'generic least square inversion with min-norm phase velocity' print ' based on Berardino et al. (2002, IEEE-TGRS)' print ' OLS for pixels with fully connected network' print ' SVD for pixels with partially connected network' if np.linalg.matrix_rank(A) < date_num-1: print 'WARNING: singular design matrix! Inversion result can be biased!' print 'continue using its SVD solution on all pixels' else: print 'weighted least square (WLS) inversion with min-norm phase, pixelwise' if np.linalg.matrix_rank(A) < date_num-1: print 'ERROR: singular design matrix!' print ' Input network of interferograms is not fully connected!' print ' Can not invert the weighted least square solution.' print 'You could try:' print ' 1) Add more interferograms to make the network fully connected:' print ' a.k.a., no multiple subsets nor network islands' print " 2) Use '-w no' option for non-weighted SVD solution." sys.exit(-1) print '-------------------------------------------------------------------------------' ##### Invert time-series phase ##Check parallel environment if meta['weight_function'] in ['no','uniform']: meta['parallel'] = False if meta['parallel']: num_cores, meta['parallel'], Parallel, delayed = ut.check_parallel(1000, print_msg=False) ##Split into chunks to reduce memory usage r_step = meta['chunk_size']/ifgram_num/width #split in lines if meta['weight_function'] not in ['no','uniform']: #more memory usage (coherence) for WLS r_step /= 2.0 if meta['parallel']: r_step /= num_cores r_step = int(ceil_to_1(r_step)) meta['row_step'] = r_step chunk_num = int((length-1)/r_step)+1 if chunk_num > 1: print 'maximum chunk size: %.1E' % (meta['chunk_size']) print 'split %d lines into %d patches for processing' % (length, chunk_num) print ' with each patch up to %d lines' % (r_step) if meta['parallel']: print 'parallel processing using %d cores ...' % (min([num_cores,chunk_num])) ##Computing the inversion box_list = [] for i in range(chunk_num): r0 = i*r_step r1 = min([length, r0+r_step]) box = (0,r0,width,r1) box_list.append(box) box_num = len(box_list) if not meta['parallel']: timeseries = np.zeros((date_num, length, width), np.float32) timeseriesStd = np.zeros((date_num, length, width), np.float32) tempCoh = np.zeros((length, width), np.float32) for i in range(box_num): if box_num > 1: print '\n------- Processing Patch %d out of %d --------------' % (i+1, box_num) box = box_list[i] ts, tcoh, tsStd = ifgram_inversion_patch(ifgramFile, coherenceFile, meta, box) tempCoh[box[1]:box[3],box[0]:box[2]] = tcoh timeseries[:,box[1]:box[3],box[0]:box[2]] = ts timeseriesStd[:,box[1]:box[3],box[0]:box[2]] = tsStd else: ##Temp file list meta['ftemp_base'] = 'timeseries_temp_' temp_file_list = [meta['ftemp_base']+str(i)+'.h5' for i in range(chunk_num)] ##Computation Parallel(n_jobs=num_cores)(delayed(ifgram_inversion_patch)\ (ifgramFile, coherenceFile, meta, box) for box in box_list) ##Concatenate temp files print 'concatenating temporary timeseries files ...' timeseries = np.zeros((date_num, length, width), np.float32) tempCoh = np.zeros((length, width), np.float32) rmCmd = 'rm' for i in range(chunk_num): fname = temp_file_list[i] box = box_list[i] print 'reading '+fname h5temp = h5py.File(fname, 'r') dset = h5temp['timeseries'].get('timeseries') timeseries[:,box[1]:box[3],box[0]:box[2]] = dset[0:-1,:,:] tempCoh[box[1]:box[3],box[0]:box[2]] = dset[-1,:,:] h5temp.close() rmCmd += ' '+fname print rmCmd os.system(rmCmd) print 'converting phase to range' phase2range = -1*float(atr['WAVELENGTH'])/(4.*np.pi) timeseries *= phase2range timeseriesStd *= abs(phase2range) ##### Calculate time-series attributes print 'calculating perpendicular baseline timeseries' pbase, pbase_top, pbase_bottom = ut.perp_baseline_ifgram2timeseries(ifgramFile, ifgram_list) pbase = str(pbase.tolist()).translate(None,'[],') # convert np.array into string separated by white space pbase_top = str(pbase_top.tolist()).translate(None,'[],') pbase_bottom = str(pbase_bottom.tolist()).translate(None,'[],') atr['P_BASELINE_TIMESERIES'] = pbase atr['P_BASELINE_TOP_TIMESERIES'] = pbase_top atr['P_BASELINE_BOTTOM_TIMESERIES'] = pbase_bottom atr['ref_date'] = date8_list[0] atr['FILE_TYPE'] = 'timeseries' atr['UNIT'] = 'm' ##### Output ## 1. Write time-series file meta['timeseriesFile'] = write_timeseries_hdf5_file(timeseries, date8_list, atr,\ timeseriesFile=meta['timeseriesFile']) if not np.all(timeseriesStd == 0.): meta['timeseriesStdFile'] = write_timeseries_hdf5_file(timeseriesStd, date8_list, atr,\ timeseriesFile=meta['timeseriesStdFile']) ## 2. Write Temporal Coherence File print 'writing >>> '+meta['tempCohFile'] atr['FILE_TYPE'] = 'temporal_coherence' atr['UNIT'] = '1' meta['tempCohFile'] = writefile.write(tempCoh, atr, meta['tempCohFile']) print 'Time series inversion took ' + str(time.time()-total) +' secs\nDone.' return meta['timeseriesFile'], meta['tempCohFile']
def ifgram_inversion_patch(ifgramFile, coherenceFile, meta, box=None): ''' Inputs: ifgramFile - string, interferograms hdf5 file coherenceFile - string, coherence hdf5 file box - 4-tuple, left, upper, right, and lower pixel coordinate of area of interest meta - dict, including the following attributes: #Interferograms length/width - int, file size for each interferogram ifgram_list - list of string, interferogram dataset name date12_list - list of string, YYMMDD-YYMMDD ref_value - np.array in size of (ifgram_num, 1) reference pixel coordinate in row/column number ref_y/x - int, reference pixel coordinate in row/column number #Time-series date8_list - list of string in YYYYMMDD tbase_diff - np.array in size of (date_num-1, 1), differential temporal baseline #Inversion weight_function - no, fim, var, coh Outputs: ts - 3D np.array in size of (date_num, row_num, col_num) temp_coh - 2D np.array in size of (row_num, col_num) tsStd - 3D np.array in size of (date_num, row_num, col_num) ''' ##### Get patch size/index if not box: box = (0,0,meta['width'],meta['length']) c0,r0,c1,r1 = box print 'processing %8d/%d lines ...' % (r1, meta['length']) ## Initiate output data matrixs row_num = r1-r0 col_num = c1-c0 pixel_num = row_num * col_num date_num = len(meta['date8_list']) ts = np.zeros((date_num, pixel_num), np.float32) tsStd = np.zeros((date_num, pixel_num), np.float32) temp_coh = np.zeros(pixel_num, np.float32) ##### Mask for pixels to invert mask = np.ones(pixel_num, np.bool_) ## 1 - Water Mask if meta['water_mask_file']: print 'skip pixels on water with mask from file: %s' % (os.path.basename(meta['water_mask_file'])) try: waterMask = readfile.read(meta['water_mask_file'], epoch='waterMask')[0][r0:r1,c0:c1].flatten() except: waterMask = readfile.read(meta['water_mask_file'], epoch='mask')[0][r0:r1,c0:c1].flatten() mask *= np.array(waterMask, np.bool_) ## 2 - Mask for Zero Phase in ALL ifgrams print 'skip pixels with zero/nan value in all interferograms' ifgram_stack = ut.get_file_stack(ifgramFile)[r0:r1,c0:c1].flatten() mask *= ~np.isnan(ifgram_stack) mask *= ifgram_stack != 0. ## Invert pixels on mask 1+2 pixel_num2inv = np.sum(mask) pixel_idx2inv = np.where(mask)[0] print 'number of pixels to invert: %s out of %s' % (pixel_num2inv, pixel_num) if pixel_num2inv < 1: ts = ts.reshape(date_num, row_num, col_num) temp_coh = temp_coh.reshape(row_num, col_num) tsStd = tsStd.reshape(date_num, row_num, col_num) return ts, temp_coh, tsStd ##### Read interferograms ifgram_num = len(meta['ifgram_list']) ifgram_data = np.zeros((ifgram_num, pixel_num), np.float32) date12_list = meta['date12_list'] if meta['skip_zero_phase']: print 'skip zero phase value (masked out and filled during phase unwrapping)' atr = readfile.read_attribute(ifgramFile) h5ifgram = h5py.File(ifgramFile,'r') for j in range(ifgram_num): ifgram = meta['ifgram_list'][j] d = h5ifgram['interferograms'][ifgram].get(ifgram)[r0:r1,c0:c1].flatten() if meta['skip_zero_phase']: d[d != 0.] -= meta['ref_value'][j] else: d -= meta['ref_value'][j] ifgram_data[j] = d sys.stdout.write('\rreading interferograms %s/%s ...' % (j+1, ifgram_num)) sys.stdout.flush() print ' ' h5ifgram.close() #ifgram_data -= meta['ref_value'] ## 3 - Mask for Non-Zero Phase in ALL ifgrams (share one B in sbas inversion) maskAllNet = np.all(ifgram_data, axis=0) maskAllNet *= mask maskPartNet = mask ^ maskAllNet ##### Design matrix A,B = ut.design_matrix(ifgramFile, date12_list) try: ref_date = str(np.loadtxt('reference_date.txt', dtype=str)) except: ref_date = meta['date8_list'][0] #print 'calculate decorrelation noise covariance with reference date = %s' % (ref_date) refIdx = meta['date8_list'].index(ref_date) timeIdx = [i for i in range(date_num)] timeIdx.remove(refIdx) Astd = ut.design_matrix(ifgramFile, date12_list, referenceDate=ref_date)[0] ##### Inversion if meta['weight_function'] in ['no','uniform']: if np.sum(maskAllNet) > 0: print 'inverting pixels with valid phase in all ifgrams with OLS (%.0f pixels) ...' % (np.sum(maskAllNet)) ts1, tempCoh1 = network_inversion_sbas(B, ifgram_data[:,maskAllNet], meta['tbase_diff'], skipZeroPhase=False) ts[1:,maskAllNet] = ts1 temp_coh[maskAllNet] = tempCoh1 if np.sum(maskPartNet) > 0: print 'inverting pixels with valid phase in part of ifgrams with SVD ...' pixel_num2inv = np.sum(maskPartNet) pixel_idx2inv = np.where(maskPartNet)[0] prog_bar = ptime.progress_bar(maxValue=pixel_num2inv) for i in range(pixel_num2inv): idx = pixel_idx2inv[i] ts1, tempCoh1 = network_inversion_sbas(B, ifgram_data[:,idx], meta['tbase_diff'], meta['skip_zero_phase']) ts[1:, idx] = ts1.flatten() temp_coh[idx] = tempCoh1 prog_bar.update(i+1, every=100, suffix=str(i+1)+'/'+str(pixel_num2inv)+' pixels') prog_bar.close() else: ##### Read coherence coh_data = np.zeros((ifgram_num, pixel_num), np.float32) h5coh = h5py.File(coherenceFile,'r') coh_list = sorted(h5coh['coherence'].keys()) coh_list = ut.check_drop_ifgram(h5coh) for j in range(ifgram_num): ifgram = coh_list[j] d = h5coh['coherence'][ifgram].get(ifgram)[r0:r1,c0:c1] d[np.isnan(d)] = 0. coh_data[j] = d.flatten() sys.stdout.write('\rreading coherence %s/%s ...' % (j+1, ifgram_num)) sys.stdout.flush() print ' ' h5coh.close() ##### Calculate Weight matrix weight = np.array(coh_data, np.float64) L = int(atr['ALOOKS']) * int(atr['RLOOKS']) epsilon = 1e-4 if meta['weight_function'].startswith('var'): print 'convert coherence to weight using inverse of phase variance' print ' with phase PDF for distributed scatterers from Tough et al. (1995)' weight = 1.0 / coherence2phase_variance_ds(weight, L, print_msg=True) elif meta['weight_function'].startswith(('lin','coh','cor')): print 'use coherence as weight directly (Perissin & Wang, 2012; Tong et al., 2016)' weight[weight < epsilon] = epsilon elif meta['weight_function'].startswith(('fim','fisher')): print 'convert coherence to weight using Fisher Information Index (Seymour & Cumming, 1994)' weight = coherence2fisher_info_index(weight, L) else: print 'Un-recognized weight function: %s' % meta['weight_function'] sys.exit(-1) ##### Weighted Inversion pixel by pixel print 'inverting time series ...' prog_bar = ptime.progress_bar(maxValue=pixel_num2inv) for i in range(pixel_num2inv): idx = pixel_idx2inv[i] ts1, tempCoh1, tsStd1 = network_inversion_wls(A, ifgram_data[:,idx], weight[:,idx], Astd=Astd,\ skipZeroPhase=meta['skip_zero_phase']) ts[1:, idx] = ts1.flatten() temp_coh[idx] = tempCoh1 tsStd[timeIdx, idx] = tsStd1.flatten() prog_bar.update(i+1, every=100, suffix=str(i+1)+'/'+str(pixel_num2inv)+' pixels') prog_bar.close() ts = ts.reshape(date_num, row_num, col_num) temp_coh = temp_coh.reshape(row_num, col_num) tsStd = tsStd.reshape(date_num, row_num, col_num) ##Write to temp hdf5 files for parallel processing if meta['parallel']: fname = meta['ftemp_base']+str(int(r0/meta['row_step']))+'.h5' print 'writing >>> '+fname h5temp = h5py.File(fname, 'w') group = h5temp.create_group('timeseries') dset = group.create_dataset('timeseries', shape=(date_num+1, row_num, col_num), dtype=np.float32) dset[0:-1,:,:] = ts dset[1,:,:] = temp_coh h5temp.close() return else: return ts, temp_coh, tsStd
def main(argv): method = 'triangular_consistency' ## or 'bonding_point' ramp_type = 'plane' save_rampCor = 'yes' plot_bonding_points = 'yes' ##### Check Inputs if len(sys.argv)>2: try: opts, args = getopt.getopt(argv,'h:f:m:x:y:o:t:',['ramp=','no-ramp-save']) except getopt.GetoptError: print 'Error while getting args'; Usage(); sys.exit(1) for opt,arg in opts: if opt in ['-h','--help']: Usage(); sys.exit() elif opt in '-f': File = arg elif opt in '-m': maskFile = arg elif opt in '-o': outName = arg elif opt in '-x': x = [int(i) for i in arg.split(',')]; method = 'bonding_point' elif opt in '-y': y = [int(i) for i in arg.split(',')]; method = 'bonding_point' elif opt in '-t': templateFile = arg elif opt in '--ramp' : ramp_type = arg.lower() elif opt in '--no-ramp-save' : save_rampCor = 'no' elif len(sys.argv)==2: if argv[0] in ['-h','--help']: Usage(); sys.exit() elif os.path.isfile(argv[0]): File = argv[0]; maskFile = argv[1] else: print 'Input file does not existed: '+argv[0]; sys.exit(1) else: Usage(); sys.exit(1) ##### Check template file try: templateFile templateContents = readfile.read_template(templateFile) except: pass try: yx = [int(i) for i in templateContents['pysar.unwrapError.yx'].split(',')] x = yx[1::2] y = yx[0::2] method = 'bonding_point' except: pass ##### Read Mask File ## Priority: ## Input mask file > pysar.mask.file > existed Modified_Mask.h5 > existed Mask.h5 try: maskFile except: try: maskFile = templateContents['pysar.mask.file'] except: if os.path.isfile('Modified_Mask.h5'): maskFile = 'Modified_Mask.h5' elif os.path.isfile('Mask.h5'): maskFile = 'Mask.h5' else: print 'No mask found!'; sys.exit(1) try: Mask,Matr = readfile.read(maskFile); print 'mask: '+maskFile except: print 'Can not open mask file: '+maskFile; sys.exit(1) ##### Output file name ext = os.path.splitext(File)[1] try: outName except: outName = File.split('.')[0]+'_unwCor'+ext print '\n**************** Unwrapping Error Correction ******************' #################### Triangular Consistency (Phase Closure) #################### if method == 'triangular_consistency': print 'Phase unwrapping error correction using Triangular Consistency / Phase Closure' h5file=h5py.File(File) ifgramList = h5file['interferograms'].keys() sx = int(h5file['interferograms'][ifgramList[0]].attrs['WIDTH']) sy = int(h5file['interferograms'][ifgramList[0]].attrs['FILE_LENGTH']) curls,Triangles,C=ut.get_triangles(h5file) A,B = ut.design_matrix(h5file) ligram,lv=np.shape(B) lcurls=np.shape(curls)[0] print 'Number of all triangles: '+ str(lcurls) print 'Number of interferograms: '+ str(ligram) #print curls curlfile='curls.h5' if not os.path.isfile(curlfile): ut.generate_curls(curlfile,h5file,Triangles,curls) thr=0.50 curls=np.array(curls); n1=curls[:,0]; n2=curls[:,1]; n3=curls[:,2] numPixels=sy*sx print 'reading interferograms...' data = np.zeros((ligram,numPixels),np.float32) for ni in range(ligram): dset=h5file['interferograms'][ifgramList[ni]].get(ifgramList[ni]) d = dset[0:dset.shape[0],0:dset.shape[1]] data[ni] = d.flatten(1) print np.shape(data) print 'reading curls ...' h5curl=h5py.File(curlfile) curlList=h5curl['interferograms'].keys() curlData = np.zeros((lcurls,numPixels),np.float32) for ni in range(lcurls): dset=h5curl['interferograms'][curlList[ni]].get(curlList[ni]) d = dset[0:dset.shape[0],0:dset.shape[1]] curlData[ni] = d.flatten(1) pi=np.pi EstUnwrap=np.zeros((ligram,numPixels),np.float32) #try: # maskFile=argv[1] # h5Mask=h5py.File(maskFile) # dset = h5Mask['mask'].get('mask') # Mask=dset[0:dset.shape[0],0:dset.shape[1]] #except: # dset = h5file['mask'].get('mask') # Mask=dset[0:dset.shape[0],0:dset.shape[1]] Mask=Mask.flatten(1) from scipy.linalg import pinv as pinv for ni in range(numPixels): #dU = np.zeros([ligram,1]) #print np.shape(dU) #print np.shape(data[:,ni]) if Mask[ni]==1: dU = data[:,ni] #nan_ndx = dataPoint == 0. unwCurl = np.array(curlData[:,ni]) #print unwCurl ind = np.abs(unwCurl)>=thr; N1 =n1[ind]; N2 =n2[ind]; N3 =n3[ind] indC = np.abs(unwCurl)< thr; Nc1=n1[indC]; Nc2=n2[indC]; Nc3=n3[indC] N =np.hstack([N1, N2, N3]); UniN =np.unique(N) Nc=np.hstack([Nc1,Nc2,Nc3]); UniNc=np.unique(Nc) inter=list(set(UniNc) & set(UniN)) # intersetion UniNc= list(UniNc) for x in inter: UniNc.remove(x) D=np.zeros([len(UniNc),ligram]) for i in range(len(UniNc)): D[i,UniNc[i]]=1 AAA=np.vstack([-2*pi*C,D]) #AAA1=np.hstack([AAA,np.zeros([AAA.shape[0],lv])]) #AAA2=np.hstack([-2*pi*np.eye(ligram),B]) #AAAA=np.vstack([AAA1,AAA2]) AAAA=np.vstack([AAA,0.25*np.eye(ligram)]) #print '************************' #print np.linalg.matrix_rank(C) #print np.linalg.matrix_rank(AAA) #print np.linalg.matrix_rank(AAAA) #print '************************' #LLL=list(np.dot(C,dU)) + list(np.zeros(np.shape(UniNc)[0]))# + list(dU) #ind=np.isnan(AAA) #M1=pinv(AAA) #M=np.dot(M1,LLL) #EstUnwrap[:,ni]=np.round(M[0:ligram])*2.0*np.pi ########## # with Tikhonov regularization: AAAA=np.vstack([AAA,0.25*np.eye(ligram)]) LLL=list(np.dot(C,dU)) + list(np.zeros(np.shape(UniNc)[0])) + list(np.zeros(ligram)) ind=np.isnan(AAAA) M1=pinv(AAAA) M=np.dot(M1,LLL) EstUnwrap[:,ni]=np.round(M[0:ligram])*2.0*np.pi #print M[0:ligram] #print np.round(M[0:ligram]) else: EstUnwrap[:,ni]=np.zeros([ligram]) if not np.remainder(ni,10000): print 'Processing point: %7d of %7d ' % (ni,numPixels) ##### Output dataCor = data+EstUnwrap unwCorFile=File.replace('.h5','')+'_unwCor.h5'; print 'writing >>> '+unwCorFile h5unwCor=h5py.File(unwCorFile,'w') gg = h5unwCor.create_group('interferograms') for i in range(ligram): group = gg.create_group(ifgramList[i]) dset = group.create_dataset(ifgramList[i], data=np.reshape(dataCor[i,:],[sx,sy]).T, compression='gzip') for key, value in h5file['interferograms'][ifgramList[i]].attrs.iteritems(): group.attrs[key] = value try: MASK=h5file['mask'].get('mask') gm = h5unwCor.create_group('mask') dset = gm.create_dataset('mask', data=MASK, compression='gzip') except: pass h5unwCor.close() h5file.close() h5curl.close() #################### Bonding Points (Spatial Continuity) #################### elif method == 'bonding_point': print 'Phase unwrapping error correction using Bonding Points / Spatial Continuity' ##### Read Bridge Points Info try: x y if len(x) != len(y) or np.mod(len(x),2) != 0: print 'Wrong number of bridge points input: '+str(len(x))+' for x, '+str(len(y))+' for y' Usage(); sys.exit(1) except: print 'Error in reading bridge points info!'; Usage(); sys.exit(1) for i in range(0,len(x)): if Mask[y[i],x[i]] == 0: print '\nERROR: Connecting point ('+str(y[i])+','+str(x[i])+') is out of masked area! Select them again!\n' sys.exit(1) print 'Number of bonding point pairs: '+str(len(x)/2) print 'Bonding points coordinates:\nx: '+str(x)+'\ny: '+str(y) ## Plot Connecting Pair of Points if plot_bonding_points == 'yes': point_yx = '' line_yx = '' n_bridge = len(x)/2 for i in range(n_bridge): pair_yx = str(y[2*i])+','+str(x[2*i])+','+str(y[2*i+1])+','+str(x[2*i+1]) if not i == n_bridge-1: point_yx += pair_yx+',' line_yx += pair_yx+';' else: point_yx += pair_yx line_yx += pair_yx try: plot_cmd = 'view.py --point-yx="'+point_yx+'" --line-yx="'+line_yx+\ '" --nodisplay -o bonding_points.png -f '+maskFile print plot_cmd os.system(plot_cmd) except: pass ##### Ramp Info ramp_mask = Mask==1 print 'estimate phase ramp during the correction' print 'ramp type: '+ramp_type if save_rampCor == 'yes': outName_ramp = os.path.basename(outName).split(ext)[0]+'_'+ramp_type+ext ########## PySAR ########## if ext == '.h5': ##### Read try: h5file=h5py.File(File,'r') except: print 'ERROR: Cannot open input file: '+File; sys.exit(1) k=h5file.keys() if 'interferograms' in k: k[0] = 'interferograms'; print 'Input file is '+k[0] else: print 'Input file - '+File+' - is not interferograms.'; Usage(); sys.exit(1) igramList = h5file[k[0]].keys() igramList = sorted(igramList) #### Write h5out = h5py.File(outName,'w') gg = h5out.create_group(k[0]) print 'writing >>> '+outName if save_rampCor == 'yes': h5out_ramp = h5py.File(outName_ramp,'w') gg_ramp = h5out_ramp.create_group(k[0]) print 'writing >>> '+outName_ramp ##### Loop print 'Number of interferograms: '+str(len(igramList)) for igram in igramList: print igram data = h5file[k[0]][igram].get(igram)[:] data_ramp,ramp = rm.remove_data_surface(data,ramp_mask,ramp_type) #ramp = data_ramp - data data_rampCor = phase_bonding(data_ramp,Mask,x,y) dataCor = data_rampCor - ramp group = gg.create_group(igram) dset = group.create_dataset(igram, data=dataCor, compression='gzip') for key, value in h5file[k[0]][igram].attrs.iteritems(): group.attrs[key]=value if save_rampCor == 'yes': group_ramp = gg_ramp.create_group(igram) dset = group_ramp.create_dataset(igram, data=data_rampCor, compression='gzip') for key, value in h5file[k[0]][igram].attrs.iteritems(): group_ramp.attrs[key]=value try: mask = h5file['mask'].get('mask'); gm = h5out.create_group('mask') dset = gm.create_dataset('mask', data=mask[0:mask.shape[0],0:mask.shape[1]], compression='gzip') except: print 'no mask group found.' h5file.close() h5out.close() if save_rampCor == 'yes': h5out_ramp.close() ########## ROI_PAC ########## elif ext == '.unw': print 'Input file is '+ext a,data,atr = readfile.read_float32(File); data_ramp,ramp = rm.remove_data_surface(data,ramp_mask,ramp_type) #ramp = data_ramp - data data_rampCor = phase_bonding(data_ramp,Mask,x,y) dataCor = data_rampCor - ramp writefile.write(dataCor, atr, outName) if save_rampCor == 'yes': writefile.write(data_rampCor,atr,outName_ramp) else: print 'Un-supported file type: '+ext; Usage(); sys.exit(1)
def main(argv): ##### Inputs try: ifgram_file = argv[0] timeseries_file = argv[1] except: usage(); sys.exit(1) try: outfile = argv[2] except: outfile = 'reconstructed_'+ifgram_file atr = readfile.read_attribute(timeseries_file) length = int(atr['FILE_LENGTH']) width = int(atr['WIDTH']) ##### Read time-series file print 'loading timeseries ...' h5ts = h5py.File(timeseries_file, 'r') date_list = sorted(h5ts['timeseries'].keys()) date_num = len(date_list) timeseries = np.zeros((date_num, length*width)) print 'number of acquisitions: '+str(date_num) prog_bar = ptime.progress_bar(maxValue=date_num) for i in range(date_num): date = date_list[i] d = h5ts['timeseries'].get(date)[:] timeseries[i,:] = d.flatten(0) prog_bar.update(i+1, suffix=date) prog_bar.close() h5ts.close() del d range2phase = -4*np.pi/float(atr['WAVELENGTH']) timeseries = range2phase*timeseries ##### Estimate interferograms from timeseries print 'estimating interferograms from timeseries using design matrix from input interferograms' A,B = ut.design_matrix(ifgram_file) p = -1*np.ones([A.shape[0],1]) Ap = np.hstack((p,A)) estData = np.dot(Ap, timeseries) del timeseries ##### Write interferograms file print 'writing >>> '+outfile h5 = h5py.File(ifgram_file,'r') ifgram_list = sorted(h5['interferograms'].keys()) ifgram_num = len(ifgram_list) date12_list = ptime.list_ifgram2date12(ifgram_list) h5out = h5py.File(outfile,'w') group = h5out.create_group('interferograms') print 'number of interferograms: '+str(ifgram_num) prog_bar = ptime.progress_bar(maxValue=ifgram_num) for i in range(ifgram_num): ifgram = ifgram_list[i] data = np.reshape(estData[i,:],(length, width)) gg = group.create_group(ifgram) dset = gg.create_dataset(ifgram, data=data, compression='gzip') for key, value in h5['interferograms'][ifgram].attrs.iteritems(): gg.attrs[key] = value prog_bar.update(i+1, suffix=date12_list[i]) prog_bar.close() h5.close() h5out.close() print 'Done.' return outfile
def timeseries_inversion(ifgramFile='unwrapIfgram.h5', coherenceFile='coherence.h5', inps_dict=None): '''Implementation of the SBAS algorithm. modified from sbas.py written by scott baker, 2012 Inputs: ifgramFile - string, HDF5 file name of the interferograms coherenceFile - string, HDF5 file name of the coherence inps_dict - dict, including the following options: weight_function min_coherence max_coherence Output: timeseriesFile - string, HDF5 file name of the output timeseries tempCohFile - string, HDF5 file name of temporal coherence ''' total = time.time() if not inps_dict: inps_dict = vars(cmdLineParse()) weight_func = inps_dict['weight_function'] min_coh = inps_dict['min_coherence'] max_coh = inps_dict['max_coherence'] # Basic Info atr = readfile.read_attribute(ifgramFile) length = int(atr['FILE_LENGTH']) width = int(atr['WIDTH']) pixel_num = length * width h5ifgram = h5py.File(ifgramFile, 'r') ifgram_list = sorted(h5ifgram['interferograms'].keys()) if inps_dict['weight_function'] == 'no': ifgram_list = ut.check_drop_ifgram(h5ifgram, atr, ifgram_list) ifgram_num = len(ifgram_list) # Convert ifgram_list to date12/8_list date12_list = ptime.list_ifgram2date12(ifgram_list) m_dates = [i.split('-')[0] for i in date12_list] s_dates = [i.split('-')[1] for i in date12_list] date8_list = ptime.yyyymmdd(sorted(list(set(m_dates + s_dates)))) date_num = len(date8_list) tbase_list = ptime.date_list2tbase(date8_list)[0] tbase_diff = np.diff(tbase_list).reshape((date_num - 1, 1)) print 'number of interferograms: ' + str(ifgram_num) print 'number of acquisitions : ' + str(date_num) print 'number of pixels: ' + str(pixel_num) # Reference pixel in space try: ref_x = int(atr['ref_x']) ref_y = int(atr['ref_y']) print 'reference pixel in y/x: [%d, %d]' % (ref_y, ref_x) except: print 'ERROR: No ref_x/y found! Can not inverse interferograms without reference in space.' print 'run seed_data.py ' + ifgramFile + ' --mark-attribute for a quick referencing.' sys.exit(1) ##### Read Interferograms print 'reading interferograms ...' ifgram_data = np.zeros((ifgram_num, pixel_num), np.float32) prog_bar = ptime.progress_bar(maxValue=ifgram_num) for j in range(ifgram_num): ifgram = ifgram_list[j] d = h5ifgram['interferograms'][ifgram].get(ifgram)[:] #d[d != 0.] -= d[ref_y, ref_x] d -= d[ref_y, ref_x] ifgram_data[j] = d.flatten() prog_bar.update(j + 1, suffix=date12_list[j]) h5ifgram.close() prog_bar.close() #####---------------------- Inversion ----------------------##### # Design matrix A, B = ut.design_matrix(ifgramFile, date12_list) if weight_func == 'no': print 'generalized inversion using SVD (Berardino et al., 2002, IEEE-TGRS)' print 'inversing time series ...' B_inv = np.array(np.linalg.pinv(B), np.float32) ts_rate = np.dot(B_inv, ifgram_data) ts1 = ts_rate * np.tile(tbase_diff, (1, pixel_num)) ts0 = np.array([0.] * pixel_num, np.float32) ts_data = np.vstack((ts0, np.cumsum(ts1, axis=0))) del ts_rate, ts0, ts1 # Temporal coherence print 'calculating temporal coherence (Tizzani et al., 2007, RSE)' temp_coh = np.zeros((1, pixel_num), np.float32) + 0j prog_bar = ptime.progress_bar(maxValue=ifgram_num) for i in range(ifgram_num): ifgram_est = np.dot(A[i, :], ts_data[1:, :]) ifgram_diff = ifgram_data[i, :] - ifgram_est temp_coh += np.exp(1j * ifgram_diff) prog_bar.update(i + 1, suffix=date12_list[i]) prog_bar.close() del ifgram_data, ifgram_est, ifgram_diff temp_coh = np.array((np.absolute(temp_coh) / ifgram_num).reshape( (length, width)), dtype=np.float32) else: print 'weighted least square (WLS) inversion using coherence pixel by pixel' if np.linalg.matrix_rank(A) < date_num - 1: print 'ERROR: singular design matrix!' print ' Input network of interferograms is not fully connected!' print ' Can not inverse the weighted least square solution.' print 'You could try:' print ' 1) Add more interferograms to make the network fully connected:' print ' a.k.a., no multiple subsets nor network islands' print " 2) Use '-w no' option for non-weighted SVD solution." sys.exit(-1) pixel_mask = np.ones(pixel_num, np.bool_) print 'reading coherence: ' + os.path.basename(coherenceFile) h5coh = h5py.File(coherenceFile, 'r') coh_list = sorted(h5coh['coherence'].keys()) coh_data = np.zeros((ifgram_num, pixel_num), np.float32) prog_bar = ptime.progress_bar(maxValue=ifgram_num) for j in range(ifgram_num): ifgram = coh_list[j] d = h5coh['coherence'][ifgram].get(ifgram)[:].flatten() d[np.isnan(d)] = 0. pixel_mask[d == 0.] = 0 coh_data[j] = d prog_bar.update(j + 1, suffix=date12_list[j]) h5coh.close() prog_bar.close() # Get mask of valid pixels to inverse print 'skip pixels with zero coherence in at least one interferogram' print 'skip pixels with zero phase in all interferograms' ifgram_stack = ut.get_file_stack(ifgramFile).flatten() pixel_mask[ifgram_stack == 0.] = 0 pixel_num2inv = np.sum(pixel_mask) pixel_idx2inv = np.where(pixel_mask)[0] ifgram_data = ifgram_data[:, pixel_mask] coh_data = coh_data[:, pixel_mask] print 'number of pixels to inverse: %d' % (pixel_num2inv) ##### Calculate Weight matrix weight = coh_data if weight_func.startswith('var'): print 'convert coherence to weight using inverse of variance: x**2/(1-x**2) from Hanssen (2001, for 4.2.32)' weight[weight > 0.999] = 0.999 if weight_func == 'variance-max-coherence': print 'constrain the max coherence to %f' % max_coh weight[weight > max_coh] = max_coh weight = np.square(weight) weight *= 1. / (1. - weight) if weight_func == 'variance-log': print 'use log(1/variance)+1 as weight' weight = np.log(weight + 1) elif weight_func.startswith('lin'): print 'use coherence as weight directly (Tong et al., 2016, RSE)' elif weight_func.startswith('norm'): print 'convert coherence to weight using CDF of normal distribution: N(%f, %f)' % ( mu, std) mu = (min_coh + max_coh) / 2.0 std = (max_coh - min_coh) / 6.0 chunk_size = 1000 chunk_num = int(pixel_num2inv / chunk_size) + 1 prog_bar = ptime.progress_bar(maxValue=chunk_num) for i in range(chunk_num): i0 = (i - 1) * chunk_size i1 = min([pixel_num2inv, i0 + chunk_size]) weight[:, i0:i1] = norm.cdf(weight[:, i0:i1], mu, std) prog_bar.update(i + 1, every=10) prog_bar.close() #weight = norm.cdf(weight, mu, std) else: print 'Un-recognized weight function: %s' % weight_func sys.exit(-1) ##### Weighted Inversion pixel by pixel print 'inversing time series ...' ts_data = np.zeros((date_num, pixel_num), np.float32) temp_coh = np.zeros(pixel_num, np.float32) prog_bar = ptime.progress_bar(maxValue=pixel_num2inv) for i in range(pixel_num2inv): # Inverse timeseries ifgram_pixel = ifgram_data[:, i] weight_pixel = weight[:, i] W = np.diag(weight_pixel) ts = np.linalg.inv(A.T.dot(W).dot(A)).dot( A.T).dot(W).dot(ifgram_pixel) ts_data[1:, pixel_idx2inv[i]] = ts # Calculate weighted temporal coherence ifgram_diff = ifgram_pixel - np.dot(A, ts) temp_coh_pixel = np.abs( np.sum(np.multiply(weight_pixel, np.exp(1j * ifgram_diff)), axis=0)) / np.sum(weight_pixel) temp_coh[pixel_idx2inv[i]] = temp_coh_pixel prog_bar.update(i + 1, every=2000, suffix=str(i + 1) + ' pixels') prog_bar.close() del ifgram_data, weight #####---------------------- Outputs ----------------------##### ## 1.1 Convert time-series phase to displacement print 'converting phase to range' phase2range = -1 * float(atr['WAVELENGTH']) / (4. * np.pi) ts_data *= phase2range ## 1.2 Write time-series data matrix timeseriesFile = 'timeseries.h5' print 'writing >>> ' + timeseriesFile print 'number of acquisitions: ' + str(date_num) h5timeseries = h5py.File(timeseriesFile, 'w') group = h5timeseries.create_group('timeseries') prog_bar = ptime.progress_bar(maxValue=date_num) for i in range(date_num): date = date8_list[i] dset = group.create_dataset(date, data=ts_data[i].reshape(length, width), compression='gzip') prog_bar.update(i + 1, suffix=date) prog_bar.close() ## 1.3 Write time-series attributes print 'calculating perpendicular baseline timeseries' pbase, pbase_top, pbase_bottom = ut.perp_baseline_ifgram2timeseries( ifgramFile, ifgram_list) pbase = str(pbase.tolist()).translate( None, '[],') # convert np.array into string separated by white space pbase_top = str(pbase_top.tolist()).translate(None, '[],') pbase_bottom = str(pbase_bottom.tolist()).translate(None, '[],') atr['P_BASELINE_TIMESERIES'] = pbase atr['P_BASELINE_TOP_TIMESERIES'] = pbase_top atr['P_BASELINE_BOTTOM_TIMESERIES'] = pbase_bottom atr['ref_date'] = date8_list[0] atr['FILE_TYPE'] = 'timeseries' atr['UNIT'] = 'm' for key, value in atr.iteritems(): group.attrs[key] = value h5timeseries.close() del ts_data ## 2. Write Temporal Coherence File tempCohFile = 'temporalCoherence.h5' print 'writing >>> ' + tempCohFile atr['FILE_TYPE'] = 'temporal_coherence' atr['UNIT'] = '1' writefile.write(temp_coh.reshape(length, width), atr, tempCohFile) print 'Time series inversion took ' + str(time.time() - total) + ' secs\nDone.' return timeseriesFile, tempCohFile
def ifgram_inversion_patch(ifgramFile, coherenceFile, meta, box=None): ''' Inputs: ifgramFile - string, interferograms hdf5 file coherenceFile - string, coherence hdf5 file box - 4-tuple, left, upper, right, and lower pixel coordinate of area of interest meta - dict, including the following attributes: #Interferograms length/width - int, file size for each interferogram ifgram_list - list of string, interferogram dataset name date12_list - list of string, YYMMDD-YYMMDD ref_value - np.array in size of (ifgram_num, 1) reference pixel coordinate in row/column number ref_y/x - int, reference pixel coordinate in row/column number #Time-series date8_list - list of string in YYYYMMDD tbase_diff - np.array in size of (date_num-1, 1), differential temporal baseline #Inversion weight_function - ''' ##### Get patch size/index if not box: box = (0, 0, meta['width'], meta['length']) c0, r0, c1, r1 = box print 'processing %8d/%d lines ...' % (r1, meta['length']) ## Initiate output data matrixs row_num = r1 - r0 col_num = c1 - c0 pixel_num = row_num * col_num date_num = len(meta['date8_list']) ts = np.zeros((date_num, pixel_num), np.float32) temp_coh = np.zeros(pixel_num, np.float32) ##### Get mask of non-zero pixels print 'skip pixels with zero/nan value in all interferograms' ifgram_stack = ut.get_file_stack(ifgramFile)[r0:r1, c0:c1].flatten() mask = ~np.isnan(ifgram_stack) mask[ifgram_stack == 0.] = 0 pixel_num2inv = np.sum(mask) pixel_idx2inv = np.where(mask)[0] print 'number of pixels to inverse: %d' % (pixel_num2inv) if pixel_num2inv < 1: ts = ts.reshape(date_num, row_num, col_num) temp_coh = temp_coh.reshape(row_num, col_num) return ts, temp_coh ##### Read interferograms ifgram_num = len(meta['ifgram_list']) ifgram_data = np.zeros((ifgram_num, pixel_num2inv), np.float32) date12_list = meta['date12_list'] atr = readfile.read_attribute(ifgramFile) h5ifgram = h5py.File(ifgramFile, 'r') for j in range(ifgram_num): ifgram = meta['ifgram_list'][j] d = h5ifgram['interferograms'][ifgram].get(ifgram)[r0:r1, c0:c1] ifgram_data[j] = d.flatten()[mask] sys.stdout.write('\rreading interferograms %s/%s ...' % (j + 1, ifgram_num)) sys.stdout.flush() print ' ' h5ifgram.close() ifgram_data -= meta['ref_value'] ##### Design matrix A, B = ut.design_matrix(ifgramFile, date12_list) B_inv = np.array(np.linalg.pinv(B), np.float32) ##### Inverse if meta['weight_function'] in ['no', 'uniform']: print 'inversing time-series ...' ts[1:, mask] = network_inversion_sbas(B, ifgram_data, meta['tbase_diff'], B_inv=B_inv) print 'calculating temporal coherence ...' temp_coh[mask] = temporal_coherence(A, ts[1:, mask], ifgram_data) else: ##### Read coherence coh_data = np.zeros((ifgram_num, pixel_num2inv), np.float32) h5coh = h5py.File(coherenceFile, 'r') coh_list = sorted(h5coh['coherence'].keys()) coh_list = ut.check_drop_ifgram(h5coh) for j in range(ifgram_num): ifgram = coh_list[j] d = h5coh['coherence'][ifgram].get(ifgram)[r0:r1, c0:c1] d[np.isnan(d)] = 0. coh_data[j] = d.flatten()[mask] sys.stdout.write('\rreading coherence %s/%s ...' % (j + 1, ifgram_num)) sys.stdout.flush() print ' ' h5coh.close() ##### Calculate Weight matrix weight = coh_data if meta['weight_function'].startswith('var'): print 'convert coherence to weight using inverse of phase variance' print ' with phase PDF for distributed scatterers from Tough et al. (1995)' L = int(atr['ALOOKS']) * int(atr['RLOOKS']) lineStr = ' number of multilooks L=%d' % L if L > 80: L = 80 lineStr += ', use L=80 to avoid dividing by 0 in calculation with Negligible effect' print lineStr weight = 1.0 / coherence2phase_variance_ds(weight, L) elif meta['weight_function'].startswith(('lin', 'coh', 'cor')): print 'use coherence as weight directly (Perissin & Wang, 2012; Tong et al., 2016)' epsilon = 1e-4 weight[weight < epsilon] = epsilon else: print 'Un-recognized weight function: %s' % meta['weight_function'] sys.exit(-1) ##### Weighted Inversion pixel by pixel print 'inversing time series ...' prog_bar = ptime.progress_bar(maxValue=pixel_num2inv) for i in range(pixel_num2inv): ts[1:, pixel_idx2inv[i]] = network_inversion_wls( A, ifgram_data[:, i], weight[:, i])[0].flatten() prog_bar.update(i + 1, every=100, suffix=str(i + 1) + '/' + str(pixel_num2inv) + ' pixels') prog_bar.close() print 'calculating temporal coherence ...' #temp_coh[mask] = temporal_coherence(A, ts[1:,mask], ifgram_data, weight) temp_coh[mask] = temporal_coherence(A, ts[1:, mask], ifgram_data) ts = ts.reshape(date_num, row_num, col_num) temp_coh = temp_coh.reshape(row_num, col_num) ##Write to temp hdf5 files for parallel processing if meta['parallel']: fname = meta['ftemp_base'] + str(int(r0 / meta['row_step'])) + '.h5' print 'writing >>> ' + fname h5temp = h5py.File(fname, 'w') group = h5temp.create_group('timeseries') dset = group.create_dataset('timeseries', shape=(date_num + 1, row_num, col_num), dtype=np.float32) dset[0:-1, :, :] = ts dset[1, :, :] = temp_coh h5temp.close() return else: return ts, temp_coh